dspx 0.1.1-alpha.0

package/src/ts/examples/timeseries/comparison-example.ts

@@ -0,0 +1,290 @@
+/**
+ * Comparison: Sample-Based vs Time-Based Processing
+ *
+ * This example demonstrates the difference between legacy sample-based
+ * and new time-based processing, especially with irregular data.
+ */
+
+import { createDspPipeline } from "../../bindings";
+
+/**
+ * Generate uniform sensor data (ideal case)
+ */
+function generateUniformData(count: number, sampleRate: number) {
+  const samples = new Float32Array(count);
+  const timestamps = new Float32Array(count);
+
+  for (let i = 0; i < count; i++) {
+    samples[i] = Math.sin(i * 0.1) * 10 + (Math.random() - 0.5) * 2;
+    timestamps[i] = i * (1000 / sampleRate); // Milliseconds
+  }
+
+  return { samples, timestamps };
+}
+
+/**
+ * Generate irregular sensor data (realistic case)
+ */
+function generateIrregularData(count: number, avgInterval: number) {
+  const samples = new Float32Array(count);
+  const timestamps = new Float32Array(count);
+  let currentTime = 0;
+
+  for (let i = 0; i < count; i++) {
+    samples[i] = Math.sin(i * 0.1) * 10 + (Math.random() - 0.5) * 2;
+
+    // Random interval: 50% to 150% of average
+    const interval = avgInterval * (0.5 + Math.random());
+    currentTime += interval;
+    timestamps[i] = currentTime;
+  }
+
+  return { samples, timestamps };
+}
+
+/**
+ * Compare sample-based vs time-based on UNIFORM data
+ */
+async function compareUniformData() {
+  console.log("=== Comparison on UNIFORM Data (Ideal Case) ===\n");
+
+  const sampleRate = 100; // 100 Hz
+  const { samples, timestamps } = generateUniformData(100, sampleRate);
+
+  // Sample-based pipeline (legacy)
+  const pipelineSample = createDspPipeline();
+  pipelineSample.MovingAverage({
+    mode: "moving",
+    windowSize: 50, // 50 samples
+  });
+
+  // Time-based pipeline (new)
+  const pipelineTime = createDspPipeline();
+  pipelineTime.MovingAverage({
+    mode: "moving",
+    windowDuration: 500, // 500ms (equivalent to 50 samples at 100 Hz)
+  });
+
+  // Process with sample-based
+  const resultSample = await pipelineSample.process(samples, {
+    sampleRate,
+    channels: 1,
+  });
+
+  // Process with time-based
+  const resultTime = await pipelineTime.process(samples, timestamps, {
+    channels: 1,
+  });
+
+  // Compare results
+  console.log("Input: Uniform 100 Hz sampling, 100 samples");
+  console.log("Window: 50 samples (sample-based) vs 500ms (time-based)");
+  console.log("\nComparison (first 5 samples):");
+  console.log("  Index | Input    | Sample-Based | Time-Based | Difference");
+  console.log("  ------|----------|--------------|------------|------------");
+
+  for (let i = 0; i < 5; i++) {
+    const diff = Math.abs(resultSample[i] - resultTime[i]);
+    console.log(
+      `  ${i.toString().padStart(5)} | ${samples[i]
+        .toFixed(3)
+        .padStart(8)} | ` +
+        `${resultSample[i].toFixed(3).padStart(12)} | ${resultTime[i]
+          .toFixed(3)
+          .padStart(10)} | ` +
+        `${diff.toFixed(6).padStart(10)}`
+    );
+  }
+
+  // Calculate max difference
+  let maxDiff = 0;
+  for (let i = 0; i < samples.length; i++) {
+    maxDiff = Math.max(maxDiff, Math.abs(resultSample[i] - resultTime[i]));
+  }
+
+  console.log(
+    `\n✅ Max difference: ${maxDiff.toFixed(6)} (essentially identical)`
+  );
+  console.log(
+    "   → On uniform data, both methods produce equivalent results\n"
+  );
+}
+
+/**
+ * Compare sample-based vs time-based on IRREGULAR data
+ */
+async function compareIrregularData() {
+  console.log("=== Comparison on IRREGULAR Data (Realistic Case) ===\n");
+
+  const avgInterval = 10; // ~100 Hz average, but irregular
+  const { samples, timestamps } = generateIrregularData(100, avgInterval);
+
+  // Calculate actual intervals
+  const intervals: number[] = [];
+  for (let i = 1; i < timestamps.length; i++) {
+    intervals.push(timestamps[i] - timestamps[i - 1]);
+  }
+  const avgActualInterval =
+    intervals.reduce((a, b) => a + b, 0) / intervals.length;
+  const minInterval = Math.min(...intervals);
+  const maxInterval = Math.max(...intervals);
+
+  console.log("Input: Irregular sampling");
+  console.log(
+    `  Average interval: ${avgActualInterval.toFixed(2)}ms (~${(
+      1000 / avgActualInterval
+    ).toFixed(0)} Hz)`
+  );
+  console.log(
+    `  Interval range: ${minInterval.toFixed(2)}ms - ${maxInterval.toFixed(
+      2
+    )}ms`
+  );
+  console.log(
+    `  Jitter: ${(
+      ((maxInterval - minInterval) / avgActualInterval) *
+      100
+    ).toFixed(1)}%\n`
+  );
+
+  // Sample-based pipeline (treats irregular data as uniform)
+  const pipelineSample = createDspPipeline();
+  pipelineSample.MovingAverage({
+    mode: "moving",
+    windowSize: 50, // 50 samples (ignores timestamps)
+  });
+
+  // Time-based pipeline (respects timestamps)
+  const pipelineTime = createDspPipeline();
+  pipelineTime.MovingAverage({
+    mode: "moving",
+    windowDuration: 500, // 500ms (uses actual timestamps)
+  });
+
+  // Process with sample-based (ignores irregularity)
+  const resultSample = await pipelineSample.process(samples, {
+    channels: 1, // No sampleRate = sequential timestamps [0,1,2,...]
+  });
+
+  // Process with time-based (respects timestamps)
+  const resultTime = await pipelineTime.process(samples, timestamps, {
+    channels: 1,
+  });
+
+  // Compare results
+  console.log("Window: 50 samples (sample-based) vs 500ms (time-based)");
+  console.log("\nComparison (samples 20-24, where differences emerge):");
+  console.log(
+    "  Index | Input    | Sample-Based | Time-Based | Difference | Δt (ms)"
+  );
+  console.log(
+    "  ------|----------|--------------|------------|------------|--------"
+  );
+
+  for (let i = 20; i < 25; i++) {
+    const diff = Math.abs(resultSample[i] - resultTime[i]);
+    const deltaT = i > 0 ? timestamps[i] - timestamps[i - 1] : 0;
+    console.log(
+      `  ${i.toString().padStart(5)} | ${samples[i]
+        .toFixed(3)
+        .padStart(8)} | ` +
+        `${resultSample[i].toFixed(3).padStart(12)} | ${resultTime[i]
+          .toFixed(3)
+          .padStart(10)} | ` +
+        `${diff.toFixed(3).padStart(10)} | ${deltaT.toFixed(2).padStart(6)}`
+    );
+  }
+
+  // Calculate statistics on differences
+  let sumDiff = 0;
+  let maxDiff = 0;
+  for (let i = 0; i < samples.length; i++) {
+    const diff = Math.abs(resultSample[i] - resultTime[i]);
+    sumDiff += diff;
+    maxDiff = Math.max(maxDiff, diff);
+  }
+  const avgDiff = sumDiff / samples.length;
+
+  console.log(`\n⚠️  Average difference: ${avgDiff.toFixed(3)}`);
+  console.log(`⚠️  Max difference: ${maxDiff.toFixed(3)}`);
+  console.log("   → On irregular data, results diverge significantly!");
+  console.log("   → Time-based processing accounts for actual timing\n");
+}
+
+/**
+ * Demonstrate when to use each approach
+ */
+async function showUseCases() {
+  console.log("=== When to Use Each Approach ===\n");
+
+  console.log("✅ USE SAMPLE-BASED (windowSize) when:");
+  console.log("   • Data is uniformly sampled (ADC, audio, high-rate sensors)");
+  console.log("   • Sample count is more important than time");
+  console.log("   • Maximum performance is critical");
+  console.log("   • Working with legacy code\n");
+
+  console.log("✅ USE TIME-BASED (windowDuration) when:");
+  console.log("   • Data arrives at irregular intervals (network, IoT)");
+  console.log('   • Time-based analysis is required ("last 5 seconds")');
+  console.log("   • Dealing with real-world sensor delays/jitter");
+  console.log("   • Need intuitive window specification\n");
+
+  console.log("✅ USE BOTH (windowSize + windowDuration) when:");
+  console.log("   • Need to enforce maximum sample count AND time window");
+  console.log("   • Want memory limits with time-based semantics");
+  console.log("   • Processing variable-rate data with hard limits\n");
+}
+
+/**
+ * Migration example: converting existing code
+ */
+async function showMigration() {
+  console.log("=== Migration Example ===\n");
+
+  console.log("BEFORE (Sample-Based):");
+  console.log("```typescript");
+  console.log("const pipeline = createDspPipeline();");
+  console.log("pipeline.MovingAverage({ mode: 'moving', windowSize: 100 });");
+  console.log("");
+  console.log("await pipeline.process(samples, {");
+  console.log("  sampleRate: 1000, // 1000 Hz");
+  console.log("  channels: 1");
+  console.log("});");
+  console.log("```\n");
+
+  console.log("AFTER (Time-Based):");
+  console.log("```typescript");
+  console.log("const pipeline = createDspPipeline();");
+  console.log("// 100 samples at 1000 Hz = 100ms");
+  console.log(
+    "pipeline.MovingAverage({ mode: 'moving', windowDuration: 100 });"
+  );
+  console.log("");
+  console.log("// Generate timestamps from your data source");
+  console.log("const timestamps = new Float32Array(samples.length);");
+  console.log("for (let i = 0; i < samples.length; i++) {");
+  console.log("  timestamps[i] = i; // 1ms per sample for 1000 Hz");
+  console.log("}");
+  console.log("");
+  console.log("await pipeline.process(samples, timestamps, { channels: 1 });");
+  console.log("```\n");
+
+  console.log("Conversion Formula:");
+  console.log("  windowDuration (ms) = (windowSize / sampleRate) * 1000\n");
+}
+
+// Run all comparisons
+async function main() {
+  await compareUniformData();
+  await compareIrregularData();
+  await showUseCases();
+  await showMigration();
+
+  console.log("=== Comparison Complete ===");
+  console.log("\n💡 Key Takeaway:");
+  console.log("   • Uniform data: Both methods equivalent");
+  console.log("   • Irregular data: Time-based processing essential");
+  console.log("   • Choose based on your data characteristics!\n");
+}
+
+main().catch(console.error);

package/src/ts/examples/timeseries/iot-sensor-example.ts

@@ -0,0 +1,143 @@
+/**
+ * IoT Sensor Processing with Irregular Timestamps
+ *
+ * This example demonstrates processing sensor data with network jitter,
+ * where samples arrive at irregular intervals.
+ */
+
+import { createDspPipeline } from "../../bindings";
+
+interface SensorReading {
+  value: number;
+  timestamp: number; // Unix timestamp in milliseconds
+  sensorId: string;
+}
+
+/**
+ * Simulates IoT sensor data with realistic network jitter
+ */
+function generateIrregularSensorData(
+  count: number,
+  baseInterval: number = 100
+): SensorReading[] {
+  const readings: SensorReading[] = [];
+  let currentTime = Date.now();
+
+  for (let i = 0; i < count; i++) {
+    // Add random jitter: ±50% of base interval
+    const jitter = (Math.random() - 0.5) * baseInterval;
+    currentTime += baseInterval + jitter;
+
+    // Simulate temperature sensor with noise
+    const baseTemp = 20 + Math.sin(i * 0.05) * 5; // Varying baseline
+    const noise = (Math.random() - 0.5) * 2; // ±1 degree noise
+
+    readings.push({
+      value: baseTemp + noise,
+      timestamp: currentTime,
+      sensorId: "TEMP-001",
+    });
+  }
+
+  return readings;
+}
+
+/**
+ * Process sensor data with time-based moving average
+ */
+async function processSensorData() {
+  console.log("=== IoT Sensor Processing Example ===\n");
+
+  // Generate 50 readings with ~100ms intervals (but irregular)
+  const readings = generateIrregularSensorData(50, 100);
+
+  console.log("Raw sensor readings (first 5):");
+  readings.slice(0, 5).forEach((r, i) => {
+    const interval = i > 0 ? r.timestamp - readings[i - 1].timestamp : 0;
+    console.log(
+      `  [${i}] ${r.value.toFixed(2)}°C at t=${
+        r.timestamp
+      } (Δ=${interval.toFixed(0)}ms)`
+    );
+  });
+
+  // Create pipeline with time-based window (5 second moving average)
+  const pipeline = createDspPipeline();
+  pipeline.MovingAverage({
+    mode: "moving",
+    windowDuration: 5000, // 5 seconds
+  });
+
+  // Convert to Float32Arrays
+  const samples = new Float32Array(readings.map((r) => r.value));
+  const timestamps = new Float32Array(readings.map((r) => r.timestamp));
+
+  // Process with timestamps
+  const smoothed = await pipeline.process(samples, timestamps, {
+    channels: 1,
+  });
+
+  console.log("\nSmoothed data (5-second moving average, first 5):");
+  for (let i = 0; i < 5; i++) {
+    console.log(
+      `  [${i}] Raw: ${samples[i].toFixed(2)}°C → Smoothed: ${smoothed[
+        i
+      ].toFixed(2)}°C`
+    );
+  }
+
+  // Calculate statistics
+  const rawStdDev = calculateStdDev(Array.from(samples));
+  const smoothedStdDev = calculateStdDev(Array.from(smoothed));
+  const noiseReduction = ((rawStdDev - smoothedStdDev) / rawStdDev) * 100;
+
+  console.log("\nNoise reduction statistics:");
+  console.log(`  Raw data std dev: ${rawStdDev.toFixed(3)}°C`);
+  console.log(`  Smoothed std dev: ${smoothedStdDev.toFixed(3)}°C`);
+  console.log(`  Noise reduction: ${noiseReduction.toFixed(1)}%`);
+
+  // Demonstrate state persistence
+  const state = await pipeline.saveState();
+  console.log(`\nSaved pipeline state (${state.length} bytes)`);
+
+  // Create new pipeline and restore state
+  const pipeline2 = createDspPipeline();
+  pipeline2.MovingAverage({
+    mode: "moving",
+    windowDuration: 5000,
+  });
+  await pipeline2.loadState(state);
+  console.log("State restored to new pipeline");
+
+  // Process new data with restored state
+  const newReadings = generateIrregularSensorData(5, 100);
+  const newSamples = new Float32Array(newReadings.map((r) => r.value));
+  const newTimestamps = new Float32Array(
+    newReadings.map(
+      (r) => r.timestamp + readings[readings.length - 1].timestamp
+    )
+  );
+  const continued = await pipeline2.process(newSamples, newTimestamps, {
+    channels: 1,
+  });
+
+  console.log("\nContinued processing with restored state:");
+  for (let i = 0; i < continued.length; i++) {
+    console.log(
+      `  [${i}] ${newSamples[i].toFixed(2)}°C → ${continued[i].toFixed(2)}°C`
+    );
+  }
+}
+
+/**
+ * Calculate standard deviation
+ */
+function calculateStdDev(values: number[]): number {
+  const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+  const variance =
+    values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
+  return Math.sqrt(variance);
+}
+
+// Run example
+processSensorData().catch(console.error);

package/src/ts/examples/timeseries/redis-streaming-example.ts

@@ -0,0 +1,233 @@
+/**
+ * Redis-Backed Streaming Data Processing
+ *
+ * This example shows how to process streaming sensor data with Redis state
+ * persistence, handling data chunks with timestamps.
+ */
+
+import { createDspPipeline } from "../../bindings";
+import { createClient } from "redis";
+
+interface DataChunk {
+  deviceId: string;
+  samples: Float32Array;
+  timestamps: Float32Array;
+}
+
+/**
+ * Streaming processor with Redis state persistence
+ */
+class StreamingProcessor {
+  private pipeline;
+  private redis;
+  private stateKey: string;
+
+  constructor(deviceId: string) {
+    this.pipeline = createDspPipeline();
+
+    // Multi-stage pipeline for sensor data
+    this.pipeline
+      .MovingAverage({
+        mode: "moving",
+        windowDuration: 10000, // 10 second smoothing
+      })
+      .Rms({
+        mode: "moving",
+        windowDuration: 5000, // 5 second RMS
+      })
+      .ZScoreNormalize({
+        mode: "moving",
+        windowDuration: 60000, // 1 minute normalization window
+        epsilon: 1e-6,
+      });
+
+    this.redis = createClient();
+    this.stateKey = `dsp:streaming:${deviceId}`;
+  }
+
+  async initialize() {
+    await this.redis.connect();
+    console.log(`Connected to Redis for device state persistence`);
+
+    // Try to restore previous state
+    const savedState = await this.redis.get(this.stateKey);
+    if (savedState) {
+      await this.pipeline.loadState(savedState);
+      console.log(`Restored previous state for ${this.stateKey}`);
+    } else {
+      console.log(`No previous state found, starting fresh`);
+    }
+  }
+
+  /**
+   * Process a chunk of streaming data
+   */
+  async processChunk(chunk: DataChunk): Promise<Float32Array> {
+    console.log(
+      `\nProcessing chunk: ${chunk.samples.length} samples from ${chunk.deviceId}`
+    );
+    console.log(
+      `  Time range: ${chunk.timestamps[0]} - ${
+        chunk.timestamps[chunk.timestamps.length - 1]
+      }`
+    );
+
+    // Process data
+    const result = await this.pipeline.process(
+      chunk.samples,
+      chunk.timestamps,
+      { channels: 1 }
+    );
+
+    // Save state to Redis
+    const newState = await this.pipeline.saveState();
+    await this.redis.set(this.stateKey, newState, {
+      EX: 3600, // 1 hour TTL
+    });
+    console.log(`  Saved state to Redis (${newState.length} bytes)`);
+
+    return result;
+  }
+
+  async close() {
+    await this.redis.quit();
+  }
+}
+
+/**
+ * Simulate streaming sensor data chunks
+ */
+function generateDataChunk(
+  chunkIndex: number,
+  samplesPerChunk: number,
+  startTime: number
+): DataChunk {
+  const samples = new Float32Array(samplesPerChunk);
+  const timestamps = new Float32Array(samplesPerChunk);
+
+  for (let i = 0; i < samplesPerChunk; i++) {
+    const globalIndex = chunkIndex * samplesPerChunk + i;
+
+    // Simulate sensor data: sine wave + noise + drift
+    const baseSignal = Math.sin(globalIndex * 0.1) * 10;
+    const noise = (Math.random() - 0.5) * 2;
+    const drift = globalIndex * 0.01;
+
+    samples[i] = baseSignal + noise + drift;
+
+    // Irregular timestamps (realistic network jitter)
+    const nominalInterval = 100; // ~100ms
+    const jitter = (Math.random() - 0.5) * 20; // ±10ms jitter
+    timestamps[i] = startTime + i * nominalInterval + jitter;
+  }
+
+  return {
+    deviceId: "SENSOR-001",
+    samples,
+    timestamps,
+  };
+}
+
+/**
+ * Main streaming example
+ */
+async function runStreamingExample() {
+  console.log("=== Redis-Backed Streaming Processing Example ===\n");
+
+  const processor = new StreamingProcessor("SENSOR-001");
+  await processor.initialize();
+
+  // Simulate 5 chunks of streaming data
+  const chunksCount = 5;
+  const samplesPerChunk = 20;
+  let startTime = Date.now();
+
+  const allResults: number[] = [];
+
+  for (let i = 0; i < chunksCount; i++) {
+    console.log(`\n--- Chunk ${i + 1}/${chunksCount} ---`);
+
+    const chunk = generateDataChunk(i, samplesPerChunk, startTime);
+    const result = await processor.processChunk(chunk);
+
+    // Show sample results
+    console.log(`  Results (first 3):`);
+    for (let j = 0; j < Math.min(3, result.length); j++) {
+      console.log(
+        `    [${j}] Input: ${chunk.samples[j].toFixed(2)} → Output: ${result[
+          j
+        ].toFixed(3)}`
+      );
+    }
+
+    allResults.push(...Array.from(result));
+
+    // Update start time for next chunk
+    startTime = chunk.timestamps[chunk.timestamps.length - 1] + 100;
+
+    // Simulate real-time delay between chunks
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+
+  console.log(
+    `\n\nProcessed ${allResults.length} total samples across ${chunksCount} chunks`
+  );
+  console.log(
+    `Final output range: [${Math.min(...allResults).toFixed(3)}, ${Math.max(
+      ...allResults
+    ).toFixed(3)}]`
+  );
+
+  await processor.close();
+  console.log("\nClosed Redis connection");
+}
+
+/**
+ * Example: Recovery from interruption
+ */
+async function demonstrateRecovery() {
+  console.log("\n\n=== Demonstrating Recovery After Interruption ===\n");
+
+  // Process first chunk
+  const processor1 = new StreamingProcessor("SENSOR-002");
+  await processor1.initialize();
+
+  const chunk1 = generateDataChunk(0, 20, Date.now());
+  await processor1.processChunk(chunk1);
+  console.log("Processed chunk 1, state saved");
+
+  await processor1.close();
+  console.log("Simulating process crash/restart...\n");
+
+  // Simulate restart - new processor instance
+  const processor2 = new StreamingProcessor("SENSOR-002");
+  await processor2.initialize(); // Automatically restores state
+
+  // Continue with next chunk
+  const chunk2 = generateDataChunk(
+    1,
+    20,
+    chunk1.timestamps[chunk1.timestamps.length - 1] + 100
+  );
+  const result2 = await processor2.processChunk(chunk2);
+
+  console.log("Successfully continued processing after restart!");
+  console.log(`  Processed ${result2.length} samples with restored context`);
+
+  await processor2.close();
+}
+
+// Run examples
+async function main() {
+  try {
+    await runStreamingExample();
+    await demonstrateRecovery();
+  } catch (error) {
+    console.error("Error:", error);
+    if (error instanceof Error && error.message.includes("ECONNREFUSED")) {
+      console.error("\n⚠️  Make sure Redis is running: redis-server");
+    }
+  }
+}
+
+main().catch(console.error);