@littlebearapps/platform-admin-sdk 2.1.0 → 2.2.0

package/templates/shared/scripts/test-telemetry-flow.ts

@@ -0,0 +1,464 @@
+/**
+ * Verification Script: Platform SDK Telemetry Flow
+ *
+ * This script tests the Platform SDK telemetry flow including:
+ * - TelemetryMessage format validation
+ * - MetricsAccumulator initialization
+ * - Circuit breaker error handling
+ * - Feature ID parsing
+ * - End-to-end flow simulation
+ *
+ * Run with: npx tsx scripts/test-telemetry-flow.ts
+ *
+ * @task SDK pilot integration verification
+ */
+
+import {
+  CircuitBreakerError,
+  createMetricsAccumulator,
+  CIRCUIT_STATUS,
+  KV_KEYS,
+  type TelemetryMessage,
+  type MetricsAccumulator,
+  type FeatureMetrics,
+} from '@littlebearapps/platform-consumer-sdk';
+
+// ============================================================================
+// Utility Functions
+// ============================================================================
+
+function parseFeatureId(featureId: string): {
+  project: string;
+  category: string;
+  feature: string;
+} {
+  const parts = featureId.split(':');
+  if (parts.length !== 3) {
+    throw new Error(
+      `Invalid featureId format: "${featureId}". Expected "project:category:feature"`
+    );
+  }
+  return {
+    project: parts[0],
+    category: parts[1],
+    feature: parts[2],
+  };
+}
+
+function createTelemetryMessage(featureId: string, metrics: FeatureMetrics): TelemetryMessage {
+  const { project, category, feature } = parseFeatureId(featureId);
+  return {
+    feature_key: featureId,
+    project,
+    category,
+    feature,
+    metrics,
+    timestamp: Date.now(),
+  };
+}
+
+// ============================================================================
+// Test Cases: TelemetryMessage Format
+// ============================================================================
+
+console.log('='.repeat(70));
+console.log('Platform SDK Telemetry Flow Verification');
+console.log('='.repeat(70));
+console.log();
+
+let passed = 0;
+let failed = 0;
+
+// Test 1: TelemetryMessage creation
+console.log('Test 1: TelemetryMessage Creation');
+console.log('-'.repeat(50));
+
+const testFeatureId = 'scout:ocr:process';
+const testMetrics: FeatureMetrics = {
+  d1Reads: 5,
+  d1Writes: 2,
+  d1RowsRead: 50,
+  d1RowsWritten: 10,
+  kvReads: 3,
+  kvWrites: 1,
+};
+
+try {
+  const message = createTelemetryMessage(testFeatureId, testMetrics);
+
+  const checks = [
+    { name: 'feature_key matches', pass: message.feature_key === testFeatureId },
+    { name: 'project parsed correctly', pass: message.project === 'scout' },
+    { name: 'category parsed correctly', pass: message.category === 'ocr' },
+    { name: 'feature parsed correctly', pass: message.feature === 'process' },
+    { name: 'timestamp is number', pass: typeof message.timestamp === 'number' },
+    { name: 'timestamp is recent', pass: Date.now() - message.timestamp < 1000 },
+    { name: 'metrics.d1Reads preserved', pass: message.metrics.d1Reads === 5 },
+    { name: 'metrics.d1Writes preserved', pass: message.metrics.d1Writes === 2 },
+    { name: 'metrics.kvReads preserved', pass: message.metrics.kvReads === 3 },
+  ];
+
+  for (const check of checks) {
+    if (check.pass) {
+      console.log(`  ✓ ${check.name}`);
+      passed++;
+    } else {
+      console.log(`  ✗ ${check.name}`);
+      failed++;
+    }
+  }
+} catch (e) {
+  console.log(`  ✗ TelemetryMessage creation failed: ${e}`);
+  failed++;
+}
+
+console.log();
+
+// ============================================================================
+// Test Cases: MetricsAccumulator
+// ============================================================================
+
+console.log('Test 2: MetricsAccumulator Initialization');
+console.log('-'.repeat(50));
+
+const accumulator = createMetricsAccumulator();
+
+const accumulatorChecks: Array<{ name: string; field: keyof MetricsAccumulator }> = [
+  { name: 'd1Writes starts at 0', field: 'd1Writes' },
+  { name: 'd1Reads starts at 0', field: 'd1Reads' },
+  { name: 'd1RowsRead starts at 0', field: 'd1RowsRead' },
+  { name: 'd1RowsWritten starts at 0', field: 'd1RowsWritten' },
+  { name: 'kvReads starts at 0', field: 'kvReads' },
+  { name: 'kvWrites starts at 0', field: 'kvWrites' },
+  { name: 'kvDeletes starts at 0', field: 'kvDeletes' },
+  { name: 'kvLists starts at 0', field: 'kvLists' },
+  { name: 'aiRequests starts at 0', field: 'aiRequests' },
+  { name: 'aiNeurons starts at 0', field: 'aiNeurons' },
+  { name: 'vectorizeQueries starts at 0', field: 'vectorizeQueries' },
+  { name: 'vectorizeInserts starts at 0', field: 'vectorizeInserts' },
+  // vectorizeDeletes removed - Analytics Engine 20 double limit
+];
+
+for (const check of accumulatorChecks) {
+  if (accumulator[check.field] === 0) {
+    console.log(`  ✓ ${check.name}`);
+    passed++;
+  } else {
+    console.log(`  ✗ ${check.name} (got ${accumulator[check.field]})`);
+    failed++;
+  }
+}
+
+// Test accumulator mutation
+accumulator.d1Writes += 5;
+accumulator.kvReads += 3;
+
+if (accumulator.d1Writes === 5 && accumulator.kvReads === 3) {
+  console.log('  ✓ Accumulator is mutable');
+  passed++;
+} else {
+  console.log('  ✗ Accumulator mutation failed');
+  failed++;
+}
+
+console.log();
+
+// ============================================================================
+// Test Cases: CircuitBreakerError
+// ============================================================================
+
+console.log('Test 3: CircuitBreakerError');
+console.log('-'.repeat(50));
+
+const cbError = new CircuitBreakerError('scout:ocr:process', 'feature', 'Budget exceeded');
+
+const errorChecks = [
+  { name: 'is Error instance', pass: cbError instanceof Error },
+  { name: 'is CircuitBreakerError instance', pass: cbError instanceof CircuitBreakerError },
+  { name: 'name is CircuitBreakerError', pass: cbError.name === 'CircuitBreakerError' },
+  { name: 'featureId preserved', pass: cbError.featureId === 'scout:ocr:process' },
+  { name: 'level is feature', pass: cbError.level === 'feature' },
+  { name: 'reason preserved', pass: cbError.reason === 'Budget exceeded' },
+  { name: 'message contains featureId', pass: cbError.message.includes('scout:ocr:process') },
+  { name: 'message contains reason', pass: cbError.message.includes('Budget exceeded') },
+];
+
+for (const check of errorChecks) {
+  if (check.pass) {
+    console.log(`  ✓ ${check.name}`);
+    passed++;
+  } else {
+    console.log(`  ✗ ${check.name}`);
+    failed++;
+  }
+}
+
+// Test error without reason
+const cbErrorNoReason = new CircuitBreakerError('scout:ocr:process', 'project');
+if (cbErrorNoReason.reason === undefined) {
+  console.log('  ✓ Error without reason has undefined reason');
+  passed++;
+} else {
+  console.log('  ✗ Error without reason should have undefined reason');
+  failed++;
+}
+
+// Test different levels
+const levelTests: Array<{ level: 'feature' | 'project' | 'global' }> = [
+  { level: 'feature' },
+  { level: 'project' },
+  { level: 'global' },
+];
+
+for (const test of levelTests) {
+  const err = new CircuitBreakerError('test:a:b', test.level);
+  if (err.level === test.level) {
+    console.log(`  ✓ Level ${test.level} works correctly`);
+    passed++;
+  } else {
+    console.log(`  ✗ Level ${test.level} failed`);
+    failed++;
+  }
+}
+
+console.log();
+
+// ============================================================================
+// Test Cases: KV Key Patterns
+// ============================================================================
+
+console.log('Test 4: KV Key Patterns');
+console.log('-'.repeat(50));
+
+const kvKeyTests = [
+  {
+    name: 'featureStatus',
+    fn: () => KV_KEYS.featureStatus('scout:ocr:process'),
+    expected: 'CONFIG:FEATURE:scout:ocr:process:STATUS',
+  },
+  {
+    name: 'projectStatus',
+    fn: () => KV_KEYS.projectStatus('scout'),
+    expected: 'CONFIG:PROJECT:scout:STATUS',
+  },
+  {
+    name: 'globalStatus',
+    fn: () => KV_KEYS.globalStatus(),
+    expected: 'CONFIG:GLOBAL:STATUS',
+  },
+  {
+    name: 'featureReason',
+    fn: () => KV_KEYS.featureReason('scout:ocr:process'),
+    expected: 'CONFIG:FEATURE:scout:ocr:process:REASON',
+  },
+  {
+    name: 'featureDisabledAt',
+    fn: () => KV_KEYS.featureDisabledAt('scout:ocr:process'),
+    expected: 'CONFIG:FEATURE:scout:ocr:process:DISABLED_AT',
+  },
+  {
+    name: 'featureAutoResetAt',
+    fn: () => KV_KEYS.featureAutoResetAt('scout:ocr:process'),
+    expected: 'CONFIG:FEATURE:scout:ocr:process:AUTO_RESET_AT',
+  },
+  {
+    name: 'legacy.enabled',
+    fn: () => KV_KEYS.legacy.enabled('scout:ocr:process'),
+    expected: 'FEATURE:scout:ocr:process:enabled',
+  },
+];
+
+for (const test of kvKeyTests) {
+  const result = test.fn();
+  if (result === test.expected) {
+    console.log(`  ✓ ${test.name}: ${result}`);
+    passed++;
+  } else {
+    console.log(`  ✗ ${test.name}: expected "${test.expected}", got "${result}"`);
+    failed++;
+  }
+}
+
+console.log();
+
+// ============================================================================
+// Test Cases: Circuit Status Constants
+// ============================================================================
+
+console.log('Test 5: Circuit Status Constants');
+console.log('-'.repeat(50));
+
+if (CIRCUIT_STATUS.GO === 'GO') {
+  console.log('  ✓ CIRCUIT_STATUS.GO = "GO"');
+  passed++;
+} else {
+  console.log('  ✗ CIRCUIT_STATUS.GO should be "GO"');
+  failed++;
+}
+
+if (CIRCUIT_STATUS.STOP === 'STOP') {
+  console.log('  ✓ CIRCUIT_STATUS.STOP = "STOP"');
+  passed++;
+} else {
+  console.log('  ✗ CIRCUIT_STATUS.STOP should be "STOP"');
+  failed++;
+}
+
+console.log();
+
+// ============================================================================
+// Test Cases: Feature ID Parsing
+// ============================================================================
+
+console.log('Test 6: Feature ID Parsing');
+console.log('-'.repeat(50));
+
+const validFeatureIds = [
+  { id: 'scout:ocr:process', project: 'scout', category: 'ocr', feature: 'process' },
+  {
+    id: 'brand-copilot:scanner:github',
+    project: 'brand-copilot',
+    category: 'scanner',
+    feature: 'github',
+  },
+  { id: 'platform:usage:scheduled', project: 'platform', category: 'usage', feature: 'scheduled' },
+  { id: 'a:b:c', project: 'a', category: 'b', feature: 'c' },
+];
+
+for (const test of validFeatureIds) {
+  try {
+    const parsed = parseFeatureId(test.id);
+    if (
+      parsed.project === test.project &&
+      parsed.category === test.category &&
+      parsed.feature === test.feature
+    ) {
+      console.log(`  ✓ Parsed "${test.id}" correctly`);
+      passed++;
+    } else {
+      console.log(`  ✗ Parsed "${test.id}" incorrectly`);
+      failed++;
+    }
+  } catch {
+    console.log(`  ✗ Failed to parse valid ID "${test.id}"`);
+    failed++;
+  }
+}
+
+// Note: '::' and 'a:b:' technically parse to 3 parts (empty strings are valid parts)
+// The SDK doesn't validate empty parts - only that there are exactly 3 parts
+const invalidFeatureIds = ['scout', 'scout:ocr', 'scout:ocr:process:extra', ''];
+
+for (const id of invalidFeatureIds) {
+  try {
+    parseFeatureId(id);
+    console.log(`  ✗ Should have rejected invalid ID "${id}"`);
+    failed++;
+  } catch {
+    console.log(`  ✓ Correctly rejected invalid ID "${id}"`);
+    passed++;
+  }
+}
+
+console.log();
+
+// ============================================================================
+// Test Cases: End-to-End Flow Simulation
+// ============================================================================
+
+console.log('Test 7: End-to-End Flow Simulation');
+console.log('-'.repeat(50));
+
+// Simulate a complete telemetry flow
+const flowAccumulator = createMetricsAccumulator();
+
+// Simulate D1 operations
+flowAccumulator.d1Reads += 1;
+flowAccumulator.d1RowsRead += 10;
+flowAccumulator.d1Writes += 2;
+flowAccumulator.d1RowsWritten += 5;
+
+// Simulate KV operations
+flowAccumulator.kvReads += 3;
+flowAccumulator.kvWrites += 1;
+
+// Simulate AI operations
+flowAccumulator.aiRequests += 1;
+flowAccumulator.aiNeurons += 500;
+
+// Convert to telemetry message
+const flowFeatureId = 'platform:usage:fetch';
+const flowMetrics: FeatureMetrics = {
+  d1Reads: flowAccumulator.d1Reads,
+  d1Writes: flowAccumulator.d1Writes,
+  d1RowsRead: flowAccumulator.d1RowsRead,
+  d1RowsWritten: flowAccumulator.d1RowsWritten,
+  kvReads: flowAccumulator.kvReads,
+  kvWrites: flowAccumulator.kvWrites,
+  aiRequests: flowAccumulator.aiRequests,
+  aiNeurons: flowAccumulator.aiNeurons,
+};
+
+const flowMessage = createTelemetryMessage(flowFeatureId, flowMetrics);
+
+// Validate the message structure for queue consumption
+const messageValidation = [
+  { name: 'Has feature_key', pass: typeof flowMessage.feature_key === 'string' },
+  { name: 'Has project', pass: typeof flowMessage.project === 'string' },
+  { name: 'Has category', pass: typeof flowMessage.category === 'string' },
+  { name: 'Has feature', pass: typeof flowMessage.feature === 'string' },
+  { name: 'Has timestamp', pass: typeof flowMessage.timestamp === 'number' },
+  { name: 'Has metrics object', pass: typeof flowMessage.metrics === 'object' },
+  { name: 'D1 reads tracked', pass: flowMessage.metrics.d1Reads === 1 },
+  { name: 'D1 writes tracked', pass: flowMessage.metrics.d1Writes === 2 },
+  { name: 'D1 rows read tracked', pass: flowMessage.metrics.d1RowsRead === 10 },
+  { name: 'D1 rows written tracked', pass: flowMessage.metrics.d1RowsWritten === 5 },
+  { name: 'KV reads tracked', pass: flowMessage.metrics.kvReads === 3 },
+  { name: 'KV writes tracked', pass: flowMessage.metrics.kvWrites === 1 },
+  { name: 'AI requests tracked', pass: flowMessage.metrics.aiRequests === 1 },
+  { name: 'AI neurons tracked', pass: flowMessage.metrics.aiNeurons === 500 },
+];
+
+for (const check of messageValidation) {
+  if (check.pass) {
+    console.log(`  ✓ ${check.name}`);
+    passed++;
+  } else {
+    console.log(`  ✗ ${check.name}`);
+    failed++;
+  }
+}
+
+console.log();
+console.log('  Simulated telemetry message:');
+console.log(
+  JSON.stringify(flowMessage, null, 2)
+    .split('\n')
+    .map((l) => '    ' + l)
+    .join('\n')
+);
+
+console.log();
+
+// ============================================================================
+// Final Summary
+// ============================================================================
+
+console.log('='.repeat(70));
+console.log('FINAL SUMMARY');
+console.log('='.repeat(70));
+console.log(`Total tests: ${passed + failed}`);
+console.log(`Passed: ${passed}`);
+console.log(`Failed: ${failed}`);
+console.log();
+
+if (failed > 0) {
+  console.log('❌ VERIFICATION FAILED');
+  process.exit(1);
+} else {
+  console.log('✅ ALL VERIFICATIONS PASSED');
+  console.log();
+  console.log('Platform SDK telemetry flow is working correctly.');
+  console.log('Ready for production deployment.');
+  process.exit(0);
+}