@littlebearapps/platform-admin-sdk 1.5.0 → 2.0.0

package/templates/shared/dashboard/src/pages/api/usage/projects.ts

@@ -0,0 +1,51 @@
+import type { APIRoute } from 'astro';
+
+interface Env {
+  PLATFORM_DB?: D1Database;
+}
+
+export const GET: APIRoute = async ({ locals }) => {
+  const env = locals.runtime?.env as Env | undefined;
+  const db = env?.PLATFORM_DB;
+
+  const projects: Array<{
+    project: string;
+    total_cost: number;
+    d1_writes: number;
+    worker_requests: number;
+    latest_date: string;
+  }> = [];
+
+  if (db) {
+    try {
+      const monthStart = new Date();
+      monthStart.setDate(1);
+      const cutoff = monthStart.toISOString().slice(0, 10);
+
+      const result = await db
+        .prepare(
+          `SELECT project,
+                  SUM(total_cost_usd) as total_cost,
+                  SUM(d1_writes) as d1_writes,
+                  SUM(worker_requests) as worker_requests,
+                  MAX(snapshot_date) as latest_date
+           FROM daily_usage_rollups
+           WHERE project != 'all' AND snapshot_date >= ?
+           GROUP BY project
+           ORDER BY total_cost DESC
+           LIMIT 20`
+        )
+        .bind(cutoff)
+        .all();
+      if (result.results) {
+        projects.push(...(result.results as typeof projects));
+      }
+    } catch {
+      // Table may not exist
+    }
+  }
+
+  return new Response(JSON.stringify({ projects }), {
+    headers: { 'Content-Type': 'application/json', 'Cache-Control': 'max-age=300' },
+  });
+};

package/templates/shared/dashboard/src/pages/api/user/identity.ts

@@ -0,0 +1,11 @@
+import type { APIRoute } from 'astro';
+
+export const GET: APIRoute = async ({ request }) => {
+  // Extract identity from CF Access JWT headers
+  const email = request.headers.get('cf-access-authenticated-user-email') ?? 'unknown';
+  const name = email.split('@')[0] ?? 'User';
+
+  return new Response(JSON.stringify({ email, name }), {
+    headers: { 'Content-Type': 'application/json', 'Cache-Control': 'max-age=3600' },
+  });
+};

package/templates/shared/dashboard/src/pages/settings/notifications.astro

@@ -0,0 +1,34 @@
+---
+import DashboardLayout from '../../layouts/DashboardLayout.astro';
+import { SettingsCard } from '../../components/settings/SettingsCard';
+---
+
+<DashboardLayout title="Notification Settings">
+  <div class="max-w-3xl mx-auto space-y-4">
+    <h2 class="text-xl font-bold text-gray-900 dark:text-white mb-4">Notification Settings</h2>
+    <SettingsCard
+      client:load
+      label="Slack Alerts"
+      description="Send alerts to a Slack channel via webhook URL."
+      value="Not configured"
+    />
+    <SettingsCard
+      client:load
+      label="Email Alerts"
+      description="Send critical alerts via email."
+      value="Disabled"
+    />
+    <SettingsCard
+      client:load
+      label="Budget Warnings"
+      description="Notify at 70% and 90% budget thresholds."
+      value="Enabled"
+    />
+    <SettingsCard
+      client:load
+      label="Circuit Breaker Trips"
+      description="Immediate notification when a circuit breaker trips."
+      value="Enabled"
+    />
+  </div>
+</DashboardLayout>

package/templates/shared/dashboard/src/pages/settings/thresholds.astro

@@ -0,0 +1,39 @@
+---
+import DashboardLayout from '../../layouts/DashboardLayout.astro';
+import { SettingsCard } from '../../components/settings/SettingsCard';
+---
+
+<DashboardLayout title="Alert Thresholds">
+  <div class="max-w-3xl mx-auto space-y-4">
+    <h2 class="text-xl font-bold text-gray-900 dark:text-white mb-4">Alert Thresholds</h2>
+    <p class="text-sm text-gray-500 dark:text-gray-400 mb-4">
+      Configure when budget warnings fire. These values are defined in
+      <code class="text-xs bg-gray-100 dark:bg-gray-700 px-1 py-0.5 rounded">budgets.yaml</code>
+      and synced via <code class="text-xs bg-gray-100 dark:bg-gray-700 px-1 py-0.5 rounded">npm run sync:config</code>.
+    </p>
+    <SettingsCard
+      client:load
+      label="Warning Threshold"
+      description="Alert when feature budget usage reaches this percentage."
+      value="70%"
+    />
+    <SettingsCard
+      client:load
+      label="Critical Threshold"
+      description="Escalate alert severity at this percentage."
+      value="90%"
+    />
+    <SettingsCard
+      client:load
+      label="Circuit Breaker Trip"
+      description="Automatically pause feature at this percentage."
+      value="100%"
+    />
+    <SettingsCard
+      client:load
+      label="Anomaly Sensitivity"
+      description="Deviation percentage to flag as anomaly."
+      value="200%"
+    />
+  </div>
+</DashboardLayout>

package/templates/shared/dashboard/src/pages/settings/usage.astro

@@ -0,0 +1,28 @@
+---
+import DashboardLayout from '../../layouts/DashboardLayout.astro';
+import { SettingsCard } from '../../components/settings/SettingsCard';
+---
+
+<DashboardLayout title="Usage Settings">
+  <div class="max-w-3xl mx-auto space-y-4">
+    <h2 class="text-xl font-bold text-gray-900 dark:text-white mb-4">Usage Display Settings</h2>
+    <SettingsCard
+      client:load
+      label="Default Period"
+      description="Default time range for usage charts."
+      value="24 hours"
+    />
+    <SettingsCard
+      client:load
+      label="Cost Display"
+      description="Show costs as net (after allowances) or gross."
+      value="Net"
+    />
+    <SettingsCard
+      client:load
+      label="Data Refresh"
+      description="How often dashboard polls for new data."
+      value="60 seconds"
+    />
+  </div>
+</DashboardLayout>

package/templates/shared/docs/architecture.md

@@ -0,0 +1,89 @@
+# Architecture Overview
+
+## System Design
+
+The platform follows a hub-and-spoke model:
+
+```
+Producer Projects (Scout, Brand Copilot, etc.)
+    │
+    ▼ SDK telemetry via Queue
+┌────────────────────────────┐
+│  platform-usage worker     │ ← Central data warehouse
+│  (cron + queue consumer)   │
+└────────────┬───────────────┘
+             │
+    ┌────────┼────────────┐
+    ▼        ▼            ▼
+  D1 DB    KV Cache   Analytics Engine
+    │        │
+    ▼        ▼
+  Dashboard (Astro SSR + CF Pages)
+```
+
+## Data Flow
+
+### Collection Pipeline
+
+1. **SDK Telemetry**: Producer projects send telemetry via `platform-telemetry` queue
+2. **External APIs**: Cron jobs query Cloudflare GraphQL, GitHub, Stripe, AI providers
+3. **Queue Consumer**: Processes messages, enforces budgets, writes to D1 + Analytics Engine
+4. **Failed messages** route to `platform-telemetry-dlq` for manual inspection
+
+### Storage Tiers
+
+| Storage | Use Case | Cost Profile |
+|---------|----------|--------------|
+| D1 | Historical data, rollups, settings | Writes: $1/M rows |
+| KV | Circuit breakers, budgets, cache | Reads: $0.50/M, Writes: $5/M |
+| Analytics Engine | High-cardinality metrics | Free tier generous |
+
+### Budget Enforcement
+
+```
+Telemetry arrives → Check feature budget → Check project budget → Check global budget
+    │                    │                      │                      │
+    │               70% → Slack warn       90% → Slack critical    100% → CB trip
+    │                                                                     │
+    └─────────────────────────────────────────────────────────────────────┘
+                              Write to D1 + Analytics Engine
+```
+
+## Worker Topology
+
+### Minimal Tier
+- **platform-usage**: Data warehouse, cron scheduler, queue consumer
+
+### Standard Tier (adds)
+- **error-collector**: Tail worker → error fingerprinting → GitHub issues
+- **platform-sentinel**: Gap detection, cost monitoring
+- **platform-mapper**: Infrastructure discovery, attribution
+
+### Full Tier (adds)
+- **pattern-discovery**: AI-assisted transient error pattern detection
+- **platform-alert-router**: Unified alert normalisation
+- **platform-notifications**: In-app notification API
+- **platform-search**: Full-text search (FTS5)
+- **platform-settings**: Settings management API
+- **platform-auditor**: SDK integration auditor + AI Judge
+
+## Dashboard Architecture
+
+Astro SSR deployed on Cloudflare Pages with:
+- Service bindings to backend workers
+- D1/KV direct bindings for read-heavy pages
+- CF Access for authentication
+- React islands for interactive components
+
+## Configuration
+
+All configuration lives in Git and syncs to runtime:
+
+```
+platform/config/
+├── services.yaml     ← Project registry, feature IDs
+├── budgets.yaml      ← Limits, thresholds, CB config
+└── observability.yaml ← Monitoring standards
+```
+
+Sync via: `npm run sync:config` → D1 tables + KV keys

package/templates/shared/docs/post-deploy-runbook.md

@@ -0,0 +1,126 @@
+# Post-Deploy Runbook
+
+## After Initial Scaffold
+
+### 1. Configure Cloudflare Resources
+
+```bash
+# Create D1 database
+direnv exec . wrangler d1 create platform-metrics
+
+# Update database IDs in all wrangler.*.jsonc files
+# (use the ID returned from the create command)
+
+# Run migrations
+direnv exec . wrangler d1 migrations apply platform-metrics --remote
+
+# Create KV namespace
+direnv exec . wrangler kv namespace create PLATFORM_CACHE
+# Update KV namespace IDs in wrangler configs
+
+# Create telemetry queue
+direnv exec . wrangler queues create platform-telemetry
+direnv exec . wrangler queues create platform-telemetry-dlq
+```
+
+### 2. Sync Configuration
+
+```bash
+# Push services.yaml + budgets.yaml to D1/KV
+npm run sync:config
+```
+
+### 3. Deploy Workers
+
+```bash
+# Deploy in order: usage first (it's the foundation)
+npm run deploy:usage
+
+# Standard tier: error collector + sentinel
+direnv exec . wrangler deploy -c wrangler.*-error-collector.jsonc
+direnv exec . wrangler deploy -c wrangler.*-sentinel.jsonc
+
+# Full tier: remaining workers
+direnv exec . wrangler deploy -c wrangler.*-pattern-discovery.jsonc
+# ... etc
+```
+
+### 4. Deploy Dashboard
+
+```bash
+cd dashboard
+npm install && npm run build
+direnv exec . npx wrangler pages deploy dist
+```
+
+### 5. Verify
+
+```bash
+# Check worker is responding
+direnv exec . wrangler tail {worker-name} --format json
+
+# Run validation
+npm run validate:pipeline
+npm run validate:controls
+
+# Check dashboard loads
+# https://your-dashboard-url.pages.dev
+```
+
+## After Code Changes
+
+### Worker Changes
+
+1. Run tests: `npm test`
+2. Run typecheck: `npm run typecheck`
+3. Check D1 migrations: any new migrations need `wrangler d1 migrations apply`
+4. Deploy affected worker(s)
+5. Watch logs for 30 seconds: `direnv exec . wrangler tail {worker} --format json`
+
+### Budget/Config Changes
+
+1. Run `npm run validate:controls --strict`
+2. Run `npm run sync:config`
+3. Verify KV keys updated correctly
+
+### Dashboard Changes
+
+1. Build locally: `cd dashboard && npm run build`
+2. Deploy: `direnv exec . npx wrangler pages deploy dist`
+3. Verify pages load, API routes return data
+
+## Backfill After Data Gaps
+
+If usage collection missed data (e.g., worker was down):
+
+```bash
+# 1. Backfill hourly data from Cloudflare GraphQL
+npm run backfill -- --start 2026-01-01 --end 2026-01-31
+
+# 2. Roll up to daily
+npm run backfill:daily -- --start 2026-01-01 --end 2026-01-31
+
+# 3. Roll up to monthly
+npm run backfill:monthly -- --start 2026-01 --end 2026-01
+
+# Always dry-run first:
+npm run backfill:daily -- --dry-run --start 2026-01-01 --end 2026-01-31
+```
+
+## Emergency: Circuit Breaker Trip
+
+If a circuit breaker trips unexpectedly:
+
+```bash
+# 1. Check what tripped
+# Dashboard → Circuit Breakers page
+
+# 2. Investigate root cause
+direnv exec . wrangler tail {worker} --format json
+
+# 3. If safe to reset
+npm run reset-cb
+
+# 4. Monitor after reset
+direnv exec . wrangler tail {worker} --format json
+```

package/templates/shared/docs/troubleshooting.md

@@ -0,0 +1,91 @@
+# Troubleshooting Guide
+
+## Common Issues
+
+### Workers Not Deploying
+
+**Symptom**: `wrangler deploy` fails or worker doesn't respond.
+
+**Check**:
+1. Verify wrangler config references correct `main` entry point
+2. Check `compatibility_date` is recent
+3. Ensure all D1 bindings use correct database IDs
+4. Run `direnv exec . wrangler whoami` to verify credentials
+
+### D1 Write Errors
+
+**Symptom**: `D1_ERROR: too many requests` or budget circuit breaker trips.
+
+**Fix**:
+1. Check `budgets.yaml` — limits may be too low for current usage
+2. Run `npm run validate:controls` to verify budget configuration
+3. Check circuit breaker state: `cb:global`, `cb:project:{name}`, `cb:feature:{id}`
+4. Reset with `npm run reset-cb` if needed
+
+### Queue Messages Stuck in DLQ
+
+**Symptom**: DLQ depth increasing, messages not processing.
+
+**Check**:
+1. View DLQ status in dashboard Health tab
+2. Check queue consumer logs: `direnv exec . wrangler tail {worker-name}`
+3. Common cause: malformed telemetry envelopes — validate with `npm run validate:schemas`
+4. Retry DLQ messages via dashboard or API
+
+### Dashboard Not Loading Data
+
+**Symptom**: Dashboard shows empty states or loading spinners.
+
+**Check**:
+1. Verify D1 database has data: `direnv exec . wrangler d1 execute platform-metrics --command "SELECT COUNT(*) FROM hourly_usage_snapshots"`
+2. Check KV cache: service registry may be stale
+3. Run `npm run sync:config` to refresh D1/KV from YAML
+4. Check CF Access — dashboard requires authentication
+
+### Budget Warnings Not Firing
+
+**Symptom**: Usage exceeds thresholds but no Slack alerts.
+
+**Check**:
+1. Verify `SLACK_WEBHOOK_URL` is set in wrangler config
+2. Check KV dedup keys: `BUDGET_WARN:{feature}` (1hr TTL)
+3. Ensure `budgets.yaml` has correct feature keys matching `services.yaml`
+4. Run `npm run validate:controls --strict` for cross-reference check
+
+### Backfill Scripts Failing
+
+**Symptom**: `backfill:daily` or `backfill:monthly` errors.
+
+**Check**:
+1. Required env vars: `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, `D1_DATABASE_ID`
+2. API token needs D1:Write permissions
+3. Run with `--dry-run` first to verify date ranges
+4. Rate limit: scripts include 200ms delay between queries
+
+## Diagnostic Commands
+
+```bash
+# Check worker health
+direnv exec . wrangler tail {worker-name} --format json
+
+# Verify D1 schema
+direnv exec . wrangler d1 migrations list platform-metrics --remote
+
+# Test config sync
+npm run sync:config -- --dry-run
+
+# Validate all controls
+npm run validate:controls --strict
+
+# Run schema validation
+npm run validate:schemas
+
+# Check pipeline health
+npm run validate:pipeline
+```
+
+## Getting Help
+
+1. Check the [KV Key Patterns](./kv-key-patterns.md) doc for key prefix reference
+2. Review circuit breaker states in the dashboard Circuit Breakers page
+3. Run `npm test` to verify worker logic locally

package/templates/shared/package.json.hbs

@@ -8,10 +8,14 @@
     "sync:config": "npx tsx scripts/sync-config.ts",
     "deploy:usage": "wrangler deploy -c wrangler.{{projectSlug}}-usage.jsonc",
     "backfill": "npx tsx scripts/ops/backfill-cloudflare-hourly.ts",
+    "backfill:daily": "npx tsx scripts/ops/backfill-cloudflare-daily.ts",
+    "backfill:monthly": "npx tsx scripts/ops/backfill-monthly-rollups.ts",
+    "validate:controls": "node scripts/ops/validate-controls.js",
     "reset-cb": "npx tsx scripts/ops/reset-budget-state.ts",
     "verify": "npx tsx scripts/ops/verify-account-completeness.ts",
     "validate:pipeline": "npx tsx scripts/ops/validate-pipeline.ts",
     "validate:schemas": "node scripts/validate-schemas.js",
+    "test": "vitest run",
     "deploy:auditor": "wrangler deploy -c wrangler.{{projectSlug}}-auditor.jsonc",
     "deploy:mapper": "wrangler deploy -c wrangler.{{projectSlug}}-mapper.jsonc",
     "deploy:test-client": "wrangler deploy -c wrangler.{{projectSlug}}-sdk-test-client.jsonc",
@@ -27,6 +31,7 @@
     "ajv-formats": "^3.0.0",
     "tsx": "^4.19.0",
     "typescript": "^5.7.3",
+    "vitest": "^3.0.5",
     "wrangler": "^3.100.0"
   }
 }

package/templates/shared/scripts/ops/backfill-cloudflare-daily.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env npx tsx
+/**
+ * Cloudflare Daily Rollup Backfill Script
+ *
+ * Aggregates hourly_usage_snapshots into daily_usage_rollups via the D1 REST API.
+ * Queries existing hourly data and rolls it up to daily granularity.
+ *
+ * Prerequisites:
+ *   CLOUDFLARE_API_TOKEN  — API token with D1:Write permissions
+ *   CLOUDFLARE_ACCOUNT_ID — Your Cloudflare account ID
+ *   D1_DATABASE_ID        — Your platform-metrics D1 database ID
+ *
+ * Usage:
+ *   npx tsx scripts/ops/backfill-cloudflare-daily.ts
+ *   npx tsx scripts/ops/backfill-cloudflare-daily.ts --dry-run
+ *   npx tsx scripts/ops/backfill-cloudflare-daily.ts --start 2026-02-01 --end 2026-02-28
+ *   npx tsx scripts/ops/backfill-cloudflare-daily.ts --limit 30
+ */
+
+const REST_API_BASE = 'https://api.cloudflare.com/client/v4';
+const RATE_LIMIT_MS = 200;
+
+interface Args {
+  start?: string;
+  end?: string;
+  dryRun: boolean;
+  limit: number;
+}
+
+function parseArgs(): Args {
+  const args = process.argv.slice(2);
+  const result: Args = { dryRun: false, limit: 90 };
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--start' && args[i + 1]) result.start = args[++i];
+    else if (args[i] === '--end' && args[i + 1]) result.end = args[++i];
+    else if (args[i] === '--limit' && args[i + 1]) result.limit = Number(args[++i]);
+    else if (args[i] === '--dry-run') result.dryRun = true;
+  }
+
+  if (!result.start) {
+    const d = new Date();
+    d.setDate(d.getDate() - 30);
+    result.start = d.toISOString().slice(0, 10);
+  }
+  if (!result.end) {
+    result.end = new Date().toISOString().slice(0, 10);
+  }
+
+  return result;
+}
+
+function getEnvOrThrow(key: string): string {
+  const val = process.env[key];
+  if (!val) throw new Error(`Missing required env var: ${key}`);
+  return val;
+}
+
+async function d1Query(accountId: string, dbId: string, token: string, sql: string, params: unknown[] = []) {
+  const res = await fetch(`${REST_API_BASE}/accounts/${accountId}/d1/database/${dbId}/query`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ sql, params }),
+  });
+  if (!res.ok) {
+    const text = await res.text();
+    throw new Error(`D1 query failed (${res.status}): ${text}`);
+  }
+  return res.json() as Promise<{ result: Array<{ results: unknown[] }> }>;
+}
+
+async function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function main() {
+  const args = parseArgs();
+  const token = getEnvOrThrow('CLOUDFLARE_API_TOKEN');
+  const accountId = getEnvOrThrow('CLOUDFLARE_ACCOUNT_ID');
+  const dbId = getEnvOrThrow('D1_DATABASE_ID');
+
+  console.log(`Backfilling daily rollups: ${args.start} → ${args.end} (limit: ${args.limit}, dry-run: ${args.dryRun})`);
+
+  // Find dates that have hourly data but no daily rollup
+  const missingDays = await d1Query(accountId, dbId, token, `
+    SELECT DISTINCT DATE(snapshot_hour) as snapshot_date
+    FROM hourly_usage_snapshots
+    WHERE snapshot_hour >= ? AND snapshot_hour < ? AND project = 'all'
+    AND DATE(snapshot_hour) NOT IN (
+      SELECT snapshot_date FROM daily_usage_rollups WHERE project = 'all'
+    )
+    ORDER BY snapshot_date ASC
+    LIMIT ?
+  `, [args.start + ' 00:00:00', args.end + ' 23:59:59', args.limit]);
+
+  const dates = (missingDays.result?.[0]?.results ?? []) as Array<{ snapshot_date: string }>;
+  console.log(`Found ${dates.length} dates needing daily rollups`);
+
+  let inserted = 0;
+  for (const { snapshot_date } of dates) {
+    const nextDate = new Date(snapshot_date);
+    nextDate.setDate(nextDate.getDate() + 1);
+    const nextDateStr = nextDate.toISOString().slice(0, 10);
+
+    if (args.dryRun) {
+      console.log(`[DRY-RUN] Would roll up ${snapshot_date}`);
+      continue;
+    }
+
+    await d1Query(accountId, dbId, token, `
+      INSERT INTO daily_usage_rollups (
+        project, snapshot_date,
+        d1_reads, d1_writes, kv_reads, kv_writes,
+        r2_reads, r2_writes, worker_requests, total_cost_usd
+      )
+      SELECT
+        project, DATE(snapshot_hour) as snapshot_date,
+        SUM(d1_reads), SUM(d1_writes), SUM(kv_reads), SUM(kv_writes),
+        SUM(r2_reads), SUM(r2_writes), SUM(worker_requests), SUM(total_cost_usd)
+      FROM hourly_usage_snapshots
+      WHERE snapshot_hour >= ? AND snapshot_hour < ? AND project = 'all'
+      GROUP BY project, DATE(snapshot_hour)
+      ON CONFLICT (project, snapshot_date) DO UPDATE SET
+        d1_reads = excluded.d1_reads,
+        d1_writes = excluded.d1_writes,
+        kv_reads = excluded.kv_reads,
+        kv_writes = excluded.kv_writes,
+        r2_reads = excluded.r2_reads,
+        r2_writes = excluded.r2_writes,
+        worker_requests = excluded.worker_requests,
+        total_cost_usd = excluded.total_cost_usd
+    `, [snapshot_date + ' 00:00:00', nextDateStr + ' 00:00:00']);
+
+    inserted++;
+    console.log(`  Rolled up ${snapshot_date} (${inserted}/${dates.length})`);
+    await sleep(RATE_LIMIT_MS);
+  }
+
+  console.log(`Done. Inserted ${inserted} daily rollup rows.`);
+}
+
+main().catch((err) => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});