dspx 0.1.1-alpha.0

package/src/ts/examples/callbacks/pipeline-callbacks-example.ts

@@ -0,0 +1,137 @@
+import { createDspPipeline } from "../../bindings.js";
+import type { PipelineCallbacks } from "../../types.js";
+
+/**
+ * Example demonstrating the new pipeline callback API
+ * for monitoring, observability, and alerting
+ */
+
+// Simulated metrics collector
+const metrics = {
+  durations: [] as number[],
+  record(key: string, value: number) {
+    console.log(`Metric: ${key} = ${value.toFixed(2)}ms`);
+    this.durations.push(value);
+  },
+  getAverage() {
+    return this.durations.reduce((a, b) => a + b, 0) / this.durations.length;
+  },
+};
+
+// Simulated logger
+const logger = {
+  error(msg: string, err: Error) {
+    console.error(`❌ ${msg}:`, err.message);
+  },
+  info(msg: string, ctx?: any) {
+    console.log(`${msg}`, ctx || "");
+  },
+};
+
+// Alert threshold
+const THRESHOLD = 0.8;
+let alertCount = 0;
+
+function triggerAlert(index: number, stage: string, value: number) {
+  alertCount++;
+  console.log(
+    `ALERT #${alertCount}: Sample ${index} in ${stage} exceeded threshold: ${value.toFixed(
+      4
+    )} > ${THRESHOLD}`
+  );
+}
+
+// Configure callbacks
+const callbacks: PipelineCallbacks = {
+  onSample: (value, i, stage) => {
+    // Only trigger alerts for samples exceeding threshold
+    if (Math.abs(value) > THRESHOLD) {
+      triggerAlert(i, stage, value);
+    }
+  },
+
+  onStageComplete: (stage, durationMs) => {
+    metrics.record(`dsp.${stage}.duration`, durationMs);
+  },
+
+  onError: (stage, err) => {
+    logger.error(`Stage ${stage} failed`, err);
+  },
+
+  onLog: (level, msg, ctx) => {
+    // Filter out debug logs for cleaner output
+    if (level === "debug") return;
+
+    const emoji = {
+      info: "ℹ️",
+      warn: "⚠️",
+      error: "❌",
+      debug: "🔍",
+    }[level];
+
+    console.log(`${emoji} [${level.toUpperCase()}] ${msg}`, ctx || "");
+  },
+};
+
+async function main() {
+  console.log("=".repeat(60));
+  console.log("Pipeline Callbacks Example");
+  console.log("=".repeat(60));
+  console.log();
+
+  // Create pipeline with callbacks configured
+  const pipeline = createDspPipeline()
+    .pipeline(callbacks) // Configure callbacks first
+    .MovingAverage({ mode: "moving", windowSize: 3 })
+    .Rectify({ mode: "full" })
+    .Rms({ mode: "moving", windowSize: 5 });
+
+  // Generate test signal with some peaks
+  const sampleRate = 44100;
+  const duration = 0.01; // 10ms = 441 samples
+  const numSamples = Math.floor(sampleRate * duration);
+  const testSignal = new Float32Array(numSamples);
+
+  for (let i = 0; i < numSamples; i++) {
+    // Mix of normal samples and some peaks
+    if (i % 100 === 0) {
+      // Inject peaks that will trigger alerts
+      testSignal[i] = 0.9 + Math.random() * 0.1;
+    } else if (i % 50 === 0) {
+      testSignal[i] = -0.85;
+    } else {
+      // Normal random signal
+      testSignal[i] = Math.random() * 0.4 - 0.2;
+    }
+  }
+
+  console.log(`Input: ${numSamples} samples`);
+  console.log(`Pipeline: MovingAverage(3) → Rectify(full) → RMS(5)`);
+  console.log();
+
+  // Process with callbacks
+  const result = await pipeline.process(testSignal, {
+    sampleRate,
+    channels: 1,
+  });
+
+  console.log();
+  console.log("=".repeat(60));
+  console.log("Results Summary");
+  console.log("=".repeat(60));
+  console.log(`Output: ${result.length} samples`);
+  console.log(`Total alerts triggered: ${alertCount}`);
+  console.log(`Average processing time: ${metrics.getAverage().toFixed(3)}ms`);
+  console.log(
+    `Output range: [${Math.min(...result).toFixed(4)}, ${Math.max(
+      ...result
+    ).toFixed(4)}]`
+  );
+  console.log();
+}
+
+// Run the example
+main().catch((err) => {
+  console.error("Example failed:", err);
+  process.exit(1);
+});

package/src/ts/examples/callbacks/pooled-callbacks-example.ts

@@ -0,0 +1,274 @@
+/**
+ * Pooled Callbacks Example
+ * Demonstrates the performance benefits of using batched callbacks
+ * over individual per-sample/per-log callbacks
+ */
+
+import {
+  createDspPipeline,
+  type LogEntry,
+  type SampleBatch,
+} from "../../index.js";
+
+// Performance metrics collector using pooled callbacks
+class PooledMonitor {
+  private batchCount = 0;
+  private totalSamples = 0;
+  private sampleSum = 0;
+  private sampleMin = Infinity;
+  private sampleMax = -Infinity;
+
+  onBatch(batch: SampleBatch): void {
+    this.batchCount++;
+    this.totalSamples += batch.count;
+
+    // Process entire batch efficiently (SIMD-friendly)
+    for (let i = 0; i < batch.samples.length; i++) {
+      const value = batch.samples[i];
+      this.sampleSum += value;
+      if (value < this.sampleMin) this.sampleMin = value;
+      if (value > this.sampleMax) this.sampleMax = value;
+    }
+  }
+
+  onLogBatch(logs: LogEntry[]): void {
+    console.log(`\n📦 Received ${logs.length} pooled log entries:`);
+    logs.forEach((log) => {
+      const emoji =
+        log.level === "debug"
+          ? "🐛"
+          : log.level === "info"
+          ? "ℹ️"
+          : log.level === "warn"
+          ? "⚠️"
+          : "❌";
+      console.log(
+        `  ${emoji} [${log.level.toUpperCase()}] ${log.message}`,
+        log.context || ""
+      );
+    });
+  }
+
+  report(): void {
+    console.log("\nPooled Monitoring Report:");
+    console.log(`   Batches processed: ${this.batchCount}`);
+    console.log(`   Total samples: ${this.totalSamples}`);
+    console.log(
+      `   Sample range: [${this.sampleMin.toFixed(4)}, ${this.sampleMax.toFixed(
+        4
+      )}]`
+    );
+    console.log(
+      `   Sample average: ${(this.sampleSum / this.totalSamples).toFixed(4)}`
+    );
+  }
+}
+
+async function compareCallbackPerformance() {
+  console.log("Comparing Callback Performance\n");
+
+  // Generate test signal (10 batches of 4096 samples each)
+  const batchSize = 4096;
+  const numBatches = 10;
+
+  // ============================================================
+  // Test 1: Using POOLED callbacks (onBatch + onLogBatch)
+  // ============================================================
+  console.log("Test 1: POOLED Callbacks (onBatch + onLogBatch)");
+  const monitor1 = new PooledMonitor();
+  const pipeline1 = createDspPipeline()
+    .pipeline({
+      onBatch: (batch) => monitor1.onBatch(batch),
+      onLogBatch: (logs) => monitor1.onLogBatch(logs),
+      onStageComplete: (stage, duration) => {
+        console.log(`Stage ${stage} completed in ${duration.toFixed(3)}ms`);
+      },
+    })
+    .MovingAverage({ mode: "moving", windowSize: 10 })
+    .Rectify({ mode: "full" })
+    .Rms({ mode: "moving", windowSize: 10 });
+
+  const startPooled = performance.now();
+  for (let batch = 0; batch < numBatches; batch++) {
+    const input = Float32Array.from({ length: batchSize }, (_, i) =>
+      Math.sin(2 * Math.PI * 440 * (i / 48000))
+    );
+    await pipeline1.process(input, { sampleRate: 48000 });
+  }
+  const durationPooled = performance.now() - startPooled;
+
+  monitor1.report();
+  console.log(
+    `\nTotal time (pooled): ${durationPooled.toFixed(3)}ms for ${
+      numBatches * batchSize
+    } samples`
+  );
+  console.log(
+    `   Throughput: ${(
+      (numBatches * batchSize) /
+      (durationPooled / 1000)
+    ).toLocaleString()} samples/sec`
+  );
+
+  // ============================================================
+  // Test 2: Using INDIVIDUAL callbacks (onSample + onLog)
+  // ============================================================
+  console.log("\n\nTest 2: INDIVIDUAL Callbacks (onSample + onLog)");
+  console.log("⚠️  WARNING: This will make 40,960+ callback invocations!\n");
+
+  let sampleCallCount = 0;
+  let logCallCount = 0;
+  let sampleSum2 = 0;
+  let sampleMin2 = Infinity;
+  let sampleMax2 = -Infinity;
+
+  const pipeline2 = createDspPipeline()
+    .pipeline({
+      onSample: (value, _index, _stage) => {
+        sampleCallCount++;
+        // Realistic work: aggregate stats per sample (expensive!)
+        sampleSum2 += value;
+        if (value < sampleMin2) sampleMin2 = value;
+        if (value > sampleMax2) sampleMax2 = value;
+      },
+      onLog: (level, message, context) => {
+        logCallCount++;
+        // Realistic work: format and store log
+        const timestamp = Date.now();
+        const _logEntry = { level, message, context, timestamp };
+        // In real app: push to log buffer or write to file
+      },
+      onStageComplete: (stage, duration) => {
+        console.log(`Stage ${stage} completed in ${duration.toFixed(3)}ms`);
+      },
+    })
+    .MovingAverage({ mode: "moving", windowSize: 10 })
+    .Rectify({ mode: "full" })
+    .Rms({ mode: "moving", windowSize: 10 });
+
+  const startIndividual = performance.now();
+  for (let batch = 0; batch < numBatches; batch++) {
+    const input = Float32Array.from({ length: batchSize }, (_, i) =>
+      Math.sin(2 * Math.PI * 440 * (i / 48000))
+    );
+    await pipeline2.process(input, { sampleRate: 48000 });
+  }
+  const durationIndividual = performance.now() - startIndividual;
+
+  console.log(`\nIndividual Callbacks Report:`);
+  console.log(
+    `   onSample() called: ${sampleCallCount.toLocaleString()} times`
+  );
+  console.log(`   onLog() called: ${logCallCount} times`);
+  console.log(
+    `   Sample range: [${sampleMin2.toFixed(4)}, ${sampleMax2.toFixed(4)}]`
+  );
+  console.log(
+    `   Sample average: ${(sampleSum2 / sampleCallCount).toFixed(4)}`
+  );
+  console.log(
+    `\nTotal time (individual): ${durationIndividual.toFixed(3)}ms for ${
+      numBatches * batchSize
+    } samples`
+  );
+  console.log(
+    `   Throughput: ${(
+      (numBatches * batchSize) /
+      (durationIndividual / 1000)
+    ).toLocaleString()} samples/sec`
+  );
+
+  // ============================================================
+  // Performance Comparison
+  // ============================================================
+  console.log("\n\nPerformance Comparison:");
+  console.log(
+    `   Pooled callbacks:     ${durationPooled.toFixed(3)}ms (${(
+      (numBatches * batchSize) /
+      (durationPooled / 1000)
+    ).toFixed(1)} samples/sec)`
+  );
+  console.log(
+    `   Individual callbacks: ${durationIndividual.toFixed(3)}ms (${(
+      (numBatches * batchSize) /
+      (durationIndividual / 1000)
+    ).toFixed(1)} samples/sec)`
+  );
+
+  if (durationIndividual < durationPooled) {
+    console.log(
+      `   \n   Individual is ${(durationPooled / durationIndividual).toFixed(
+        2
+      )}x faster in raw speed`
+    );
+  } else {
+    console.log(
+      `   \n   Pooled is ${(durationIndividual / durationPooled).toFixed(
+        2
+      )}x faster`
+    );
+  }
+
+  console.log("\n\nProduction Architecture Analysis:");
+  console.log("   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+  console.log("\n   ⚠️  Individual Callbacks (onSample, onLog):");
+  console.log(
+    `      • Raw speed: ~${(
+      (numBatches * batchSize) /
+      (durationIndividual / 1000) /
+      1000000
+    ).toFixed(1)}M samples/sec`
+  );
+  console.log(
+    `      • Function calls: ${(
+      sampleCallCount + logCallCount
+    ).toLocaleString()} per ${numBatches} batches`
+  );
+  console.log("      • Event loop: BLOCKS on each callback invocation");
+  console.log("      • GC pressure: HIGH (per-call allocations)");
+  console.log("      • I/O operations: SYNCHRONOUS (stalls pipeline)");
+  console.log("      • Production safety: NOT RECOMMENDED");
+  console.log("      • Use case: Microbenchmarks, toy examples only");
+
+  console.log("\n   Pooled Callbacks (onBatch, onLogBatch):");
+  console.log(
+    `      • Sustained throughput: ~${(
+      (numBatches * batchSize) /
+      (durationPooled / 1000) /
+      1000000
+    ).toFixed(1)}M samples/sec`
+  );
+  console.log(
+    `      • Function calls: ${numBatches} batches (1 call per process)`
+  );
+  console.log("      • Event loop: NON-BLOCKING (batched execution)");
+  console.log("      • GC pressure: LOW (circular buffer reuse)");
+  console.log("      • I/O operations: BATCHED (network-friendly)");
+  console.log("      • Production safety: RECOMMENDED");
+  console.log(
+    "      • Industry alignment: Kafka producers, Loki agents, OTLP exporters"
+  );
+
+  console.log("\n   Trade-off Summary:");
+  console.log("      Individual mode: 2x faster in synthetic tests, but...");
+  console.log("         - Millions of synchronous callbacks block event loop");
+  console.log("         - Synchronous I/O in callbacks stalls entire pipeline");
+  console.log("         - Unpredictable GC pauses under load");
+  console.log("         - Cannot handle backpressure from external systems");
+  console.log("");
+  console.log("      Pooled mode: Slight raw speed reduction, but...");
+  console.log("         - Guaranteed non-blocking behavior");
+  console.log("         - Predictable memory footprint");
+  console.log("         - Natural backpressure handling");
+  console.log("         - Matches production telemetry patterns");
+
+  console.log("\n   Recommendation:");
+  console.log("      Use onBatch and onLogBatch for production servers");
+  console.log("      Individual callbacks are fast but dangerous at scale");
+  console.log(
+    "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
+  );
+}
+
+// Run the comparison
+compareCallbackPerformance().catch(console.error);

package/src/ts/examples/callbacks/priority-routing-example.ts

@@ -0,0 +1,277 @@
+/**
+ * Priority-Based Routing Example
+ *
+ * Demonstrates 10-level priority system with smart routing:
+ * - Low priority (1-3): Debug logs → Console only
+ * - Normal priority (4-6): Info logs → Loki
+ * - High priority (7-8): Warnings → Prometheus + Loki
+ * - Critical priority (9-10): Errors → PagerDuty + All backends
+ */
+
+import {
+  createDspPipeline,
+  createTopicRouter,
+  createConsoleHandler,
+  createMockHandler,
+} from "../../index.js";
+import type { LogEntry } from "../../types.js";
+
+console.log("Priority-Based Routing Example\n");
+
+// Example 1: Basic Priority Filtering
+console.log("1. Basic Priority Filtering\n");
+{
+  const lowPriority = createMockHandler((log) => {
+    console.log(`   [LOW] Priority ${log.priority}: ${log.message}`);
+  });
+
+  const normalPriority = createMockHandler((log) => {
+    console.log(`   [NORMAL] Priority ${log.priority}: ${log.message}`);
+  });
+
+  const highPriority = createMockHandler((log) => {
+    console.log(`   [HIGH] Priority ${log.priority}: ${log.message}`);
+  });
+
+  const critical = createMockHandler((log) => {
+    console.log(`   [CRITICAL] Priority ${log.priority}: ${log.message}`);
+  });
+
+  const router = createTopicRouter()
+    .custom(/^pipeline\./, lowPriority.handler, {
+      minPriority: 1,
+      maxPriority: 3,
+    })
+    .custom(/^pipeline\./, normalPriority.handler, {
+      minPriority: 4,
+      maxPriority: 6,
+    })
+    .custom(/^pipeline\./, highPriority.handler, {
+      minPriority: 7,
+      maxPriority: 8,
+    })
+    .custom(/^pipeline\./, critical.handler, {
+      minPriority: 9,
+      maxPriority: 10,
+    })
+    .build();
+
+  const pipeline = createDspPipeline();
+  pipeline
+    .pipeline({
+      onLogBatch: (logs) => router.routeBatch(logs),
+    })
+    .MovingAverage({ mode: "moving", windowSize: 3 });
+
+  const samples = new Float32Array([1, 2, 3]);
+  await pipeline.process(samples, { sampleRate: 1000 });
+
+  console.log(`\n   Summary:`);
+  console.log(`   • Low priority logs: ${lowPriority.getLogs().length}`);
+  console.log(`   • Normal priority logs: ${normalPriority.getLogs().length}`);
+  console.log(`   • High priority logs: ${highPriority.getLogs().length}`);
+  console.log(`   • Critical logs: ${critical.getLogs().length}\n`);
+}
+
+// Example 2: Production Alert Routing
+console.log("2. Production Alert Routing (Multi-Tier)\n");
+{
+  console.log("   Routing Strategy:");
+  console.log("   • Priority 1-3 (Debug): Console only");
+  console.log("   • Priority 4-6 (Info): Loki (logs storage)");
+  console.log("   • Priority 7-8 (Warn): Prometheus + Loki");
+  console.log("   • Priority 9-10 (Critical): PagerDuty + Prometheus + Loki\n");
+
+  const consoleBackend = createMockHandler((log) => {
+    console.log(`   [Console] ${log.message}`);
+  });
+
+  const lokiBackend = createMockHandler((log) => {
+    console.log(`   [Loki] Storing: ${log.message}`);
+  });
+
+  const prometheusBackend = createMockHandler((log) => {
+    console.log(`   [Prometheus] Recording metric for: ${log.message}`);
+  });
+
+  const pagerDutyBackend = createMockHandler((log) => {
+    console.log(`   [PagerDuty] ALERT triggered: ${log.message}`);
+  });
+
+  const router = createTopicRouter()
+    // Low priority: Console only
+    .custom(/^pipeline\./, consoleBackend.handler, {
+      minPriority: 1,
+      maxPriority: 3,
+    })
+    // Normal priority: Loki
+    .custom(/^pipeline\./, lokiBackend.handler, {
+      minPriority: 4,
+      maxPriority: 10, // All normal and above
+    })
+    // High priority: Prometheus
+    .custom(/^pipeline\./, prometheusBackend.handler, {
+      minPriority: 7,
+      maxPriority: 10,
+    })
+    // Critical: PagerDuty
+    .custom(/^pipeline\./, pagerDutyBackend.handler, {
+      minPriority: 9,
+      maxPriority: 10,
+      concurrency: 3, // Rate-limit alerts
+    })
+    .build();
+
+  // Simulate logs with different priorities
+  await router.routeBatch([
+    {
+      level: "debug",
+      message: "Verbose debug info",
+      topic: "pipeline.debug",
+      timestamp: Date.now(),
+      priority: 2,
+    },
+    {
+      level: "info",
+      message: "Normal operation",
+      topic: "pipeline.info",
+      timestamp: Date.now(),
+      priority: 5,
+    },
+    {
+      level: "warn",
+      message: "High CPU usage detected",
+      topic: "pipeline.warn",
+      timestamp: Date.now(),
+      priority: 7,
+    },
+    {
+      level: "error",
+      message: "Critical system failure!",
+      topic: "pipeline.error",
+      timestamp: Date.now(),
+      priority: 10,
+    },
+  ]);
+
+  console.log(`\n   Backend Summary:`);
+  console.log(`   • Console: ${consoleBackend.getLogs().length} logs`);
+  console.log(`   • Loki: ${lokiBackend.getLogs().length} logs`);
+  console.log(`   • Prometheus: ${prometheusBackend.getLogs().length} logs`);
+  console.log(`   • PagerDuty: ${pagerDutyBackend.getLogs().length} alerts\n`);
+}
+
+// Example 3: Priority with Metrics Tracking
+console.log("3. Priority Tiers with Metrics Tracking\n");
+{
+  const router = createTopicRouter()
+    .custom(/^pipeline\./, createConsoleHandler(), {
+      minPriority: 1,
+      maxPriority: 5,
+      trackMetrics: true,
+    })
+    .custom(/^pipeline\./, createConsoleHandler(), {
+      minPriority: 6,
+      maxPriority: 10,
+      trackMetrics: true,
+    })
+    .build();
+
+  const pipeline = createDspPipeline();
+  pipeline
+    .pipeline({
+      onLogBatch: (logs) => router.routeBatch(logs),
+    })
+    .MovingAverage({ mode: "moving", windowSize: 10 })
+    .Rectify()
+    .Rms({ mode: "moving", windowSize: 5 });
+
+  const samples = new Float32Array(100).map(() => Math.random() * 10 - 5);
+  await pipeline.process(samples, { sampleRate: 48000 });
+
+  console.log("   Metrics by Priority Tier:\n");
+  const metrics = router.getMetrics();
+
+  console.log("   ┌──────────────┬──────────┬─────────────┐");
+  console.log("   │ Priority     │ Exec     │ Avg (ms)    │");
+  console.log("   ├──────────────┼──────────┼─────────────┤");
+
+  metrics.forEach((m, idx) => {
+    const tier = idx === 0 ? "Low (1-5)" : "High (6-10)";
+    const exec = String(m.executionCount).padEnd(8);
+    const avg = m.averageDuration.toFixed(2).padEnd(11);
+    console.log(`   │ ${tier.padEnd(12)} │ ${exec} │ ${avg} │`);
+  });
+
+  console.log("   └──────────────┴──────────┴─────────────┘\n");
+}
+
+// Example 4: Dynamic Priority Assignment
+console.log("4. Dynamic Priority Based on Context\n");
+{
+  const mockHandler = createMockHandler((log) => {
+    const priority = log.priority ?? 1;
+    const emoji =
+      priority >= 9 ? "🚨" : priority >= 7 ? "⚠️" : priority >= 4 ? "ℹ️" : "🔍";
+    console.log(`   ${emoji} [P${priority}] ${log.message}`);
+  });
+
+  const router = createTopicRouter()
+    .custom(/^pipeline\./, mockHandler.handler)
+    .build();
+
+  // Custom logs with varying priorities
+  const customLogs: LogEntry[] = [
+    {
+      level: "debug",
+      message: "Trace-level debugging",
+      topic: "pipeline.debug",
+      timestamp: Date.now(),
+      priority: 1,
+    },
+    {
+      level: "info",
+      message: "User action completed",
+      topic: "pipeline.info",
+      timestamp: Date.now(),
+      priority: 5,
+    },
+    {
+      level: "warn",
+      message: "Memory usage at 80%",
+      topic: "pipeline.warn",
+      timestamp: Date.now(),
+      priority: 7,
+    },
+    {
+      level: "error",
+      message: "Database connection lost",
+      topic: "pipeline.error",
+      timestamp: Date.now(),
+      priority: 9,
+    },
+    {
+      level: "error",
+      message: "SYSTEM CRITICAL: Out of memory",
+      topic: "pipeline.error",
+      timestamp: Date.now(),
+      priority: 10,
+    },
+  ];
+
+  await router.routeBatch(customLogs);
+  console.log();
+}
+
+console.log("Priority System Summary:");
+console.log("   - 10-level priority (1=lowest, 10=highest)");
+console.log("   - Default priorities: debug=2, info=5, warn=7, error=9");
+console.log("   - Route filtering with minPriority/maxPriority");
+console.log("   - Multi-tier routing (low→console, high→alerts)");
+console.log("   - Priority-aware metrics tracking");
+
+console.log("\nUse Cases:");
+console.log("   • Route critical errors (9-10) to PagerDuty");
+console.log("   • Send warnings (7-8) to Prometheus + Loki");
+console.log("   • Store info logs (4-6) in Loki only");
+console.log("   • Keep debug logs (1-3) local/console");