dotdo 0.0.1 → 0.0.2

package/dist/db/primitives/stateful-operator/index.js

@@ -0,0 +1,770 @@
+/**
+ * StatefulOperator - Flink-style stream state management primitive
+ *
+ * Provides state management capabilities for stream processing:
+ * - StateBackend: Pluggable state storage interface
+ * - KeyedState: State partitioned by key with hash/range partitioning
+ * - WindowState: Time-based aggregation windows (tumbling, sliding, session)
+ * - CheckpointBarrier: Distributed snapshot coordination
+ * - StateTTL: Time-to-live state cleanup strategies
+ * - StateRecovery: Checkpoint restoration
+ * - StateMetrics: Observability for state operations
+ *
+ * @module db/primitives/stateful-operator
+ */
+import { murmurHash3, murmurHash3_32 } from '../utils/murmur3';
+// =============================================================================
+// InMemoryStateBackend
+// =============================================================================
+/**
+ * In-memory implementation of StateBackend
+ */
+export class InMemoryStateBackend {
+    name;
+    entries = new Map();
+    snapshotCounter = 0;
+    constructor(name) {
+        this.name = name;
+    }
+    async get(key) {
+        const entry = this.entries.get(key);
+        if (!entry)
+            return undefined;
+        // Check TTL expiration
+        if (entry.expiresAt !== undefined && Date.now() >= entry.expiresAt) {
+            this.entries.delete(key);
+            return undefined;
+        }
+        return entry.value;
+    }
+    async put(key, value, ttl) {
+        const entry = {
+            value,
+            createdAt: Date.now(),
+            expiresAt: ttl !== undefined ? Date.now() + ttl : undefined,
+        };
+        this.entries.set(key, entry);
+    }
+    async delete(key) {
+        this.entries.delete(key);
+    }
+    async *list(prefix) {
+        const now = Date.now();
+        for (const [key, entry] of this.entries) {
+            // Filter by prefix
+            if (prefix !== undefined && !key.startsWith(prefix)) {
+                continue;
+            }
+            // Check TTL
+            if (entry.expiresAt !== undefined && now >= entry.expiresAt) {
+                this.entries.delete(key);
+                continue;
+            }
+            yield [key, entry.value];
+        }
+    }
+    async size() {
+        // Clean up expired entries first
+        const now = Date.now();
+        for (const [key, entry] of this.entries) {
+            if (entry.expiresAt !== undefined && now >= entry.expiresAt) {
+                this.entries.delete(key);
+            }
+        }
+        return this.entries.size;
+    }
+    async snapshot() {
+        const id = `${this.name}-snap-${++this.snapshotCounter}-${Date.now()}`;
+        const timestamp = Date.now();
+        // Serialize entries
+        const serializedEntries = [];
+        for (const [key, entry] of this.entries) {
+            // Skip expired entries
+            if (entry.expiresAt !== undefined && timestamp >= entry.expiresAt) {
+                continue;
+            }
+            serializedEntries.push([key, entry]);
+        }
+        const jsonData = JSON.stringify(serializedEntries);
+        const data = new TextEncoder().encode(jsonData);
+        const checksum = this.computeChecksum(data);
+        return {
+            id,
+            timestamp,
+            data,
+            metadata: {
+                stateCount: serializedEntries.length,
+                totalBytes: data.length,
+                checksum,
+            },
+        };
+    }
+    async restore(snapshot) {
+        // Validate checksum
+        const actualChecksum = this.computeChecksum(snapshot.data);
+        if (actualChecksum !== snapshot.metadata.checksum) {
+            throw new Error(`Checksum mismatch: expected ${snapshot.metadata.checksum}, got ${actualChecksum}`);
+        }
+        // Deserialize and restore
+        const jsonData = new TextDecoder().decode(snapshot.data);
+        const entries = JSON.parse(jsonData);
+        this.entries.clear();
+        for (const [key, entry] of entries) {
+            this.entries.set(key, entry);
+        }
+    }
+    async clear() {
+        this.entries.clear();
+    }
+    computeChecksum(data) {
+        // Simple checksum using murmur hash
+        const hash = murmurHash3_32(data, 0);
+        return hash.toString(16);
+    }
+}
+/**
+ * Hash-based partitioner for even key distribution
+ */
+export class HashPartitioner {
+    numPartitions;
+    constructor(numPartitions = 256) {
+        this.numPartitions = numPartitions;
+    }
+    partition(key) {
+        const hash = murmurHash3(String(key), 0);
+        return Math.abs(hash) % this.numPartitions;
+    }
+}
+/**
+ * Range-based partitioner for ordered key access
+ */
+export class RangePartitioner {
+    boundaries;
+    numPartitions;
+    constructor(boundaries) {
+        this.boundaries = boundaries;
+        this.numPartitions = boundaries.length + 1;
+    }
+    partition(key) {
+        for (let i = 0; i < this.boundaries.length; i++) {
+            const boundary = this.boundaries[i];
+            if (boundary !== undefined && key < boundary) {
+                return i;
+            }
+        }
+        return this.boundaries.length;
+    }
+}
+/**
+ * Implementation of KeyedState
+ */
+class KeyedStateImpl {
+    backend;
+    keySerializer;
+    constructor(backend, keySerializer = String) {
+        this.backend = backend;
+        this.keySerializer = keySerializer;
+    }
+    async get(key) {
+        return this.backend.get(this.keySerializer(key));
+    }
+    async put(key, value) {
+        await this.backend.put(this.keySerializer(key), value);
+    }
+    async update(key, fn) {
+        const current = await this.get(key);
+        const updated = fn(current);
+        await this.put(key, updated);
+        return updated;
+    }
+    async delete(key) {
+        await this.backend.delete(this.keySerializer(key));
+    }
+    async *keys() {
+        for await (const [keyStr] of this.backend.list()) {
+            // Note: This assumes K can be reconstructed from string
+            // For complex keys, a deserializer would be needed
+            yield keyStr;
+        }
+    }
+    async clear() {
+        await this.backend.clear();
+    }
+}
+/**
+ * Factory function to create a KeyedState
+ */
+export function createKeyedState(backend, keySerializer) {
+    return new KeyedStateImpl(backend, keySerializer);
+}
+/**
+ * Tumbling window assigner (non-overlapping fixed-size windows)
+ */
+export class TumblingWindowAssigner {
+    windowSizeMs;
+    constructor(windowSizeMs) {
+        this.windowSizeMs = windowSizeMs;
+    }
+    assignWindows(timestamp) {
+        const windowStart = Math.floor(timestamp / this.windowSizeMs) * this.windowSizeMs;
+        return [{ start: windowStart, end: windowStart + this.windowSizeMs }];
+    }
+}
+/**
+ * Sliding window assigner (overlapping fixed-size windows)
+ */
+export class SlidingWindowAssigner {
+    windowSizeMs;
+    slideMs;
+    constructor(windowSizeMs, slideMs) {
+        this.windowSizeMs = windowSizeMs;
+        this.slideMs = slideMs;
+        if (slideMs <= 0) {
+            throw new Error('Slide must be positive');
+        }
+        if (slideMs > windowSizeMs) {
+            throw new Error('Slide cannot be larger than window size');
+        }
+    }
+    assignWindows(timestamp) {
+        const windows = [];
+        // Find the last window start that contains this timestamp
+        const lastWindowStart = Math.floor(timestamp / this.slideMs) * this.slideMs;
+        // Count backwards to find all windows
+        const numWindows = Math.ceil(this.windowSizeMs / this.slideMs);
+        for (let i = 0; i < numWindows; i++) {
+            const windowStart = lastWindowStart - i * this.slideMs;
+            const windowEnd = windowStart + this.windowSizeMs;
+            // Check if this window actually contains the timestamp
+            if (windowStart <= timestamp && timestamp < windowEnd) {
+                windows.push({ start: windowStart, end: windowEnd });
+            }
+        }
+        return windows.sort((a, b) => a.start - b.start);
+    }
+}
+/**
+ * Session window assigner (gap-based dynamic windows)
+ */
+export class SessionWindowAssigner {
+    gapMs;
+    constructor(gapMs) {
+        this.gapMs = gapMs;
+    }
+    get windowSizeMs() {
+        return this.gapMs; // Minimum size
+    }
+    assignWindows(timestamp, existingWindows) {
+        const newWindow = { start: timestamp, end: timestamp + this.gapMs };
+        if (!existingWindows || existingWindows.size === 0) {
+            return [newWindow];
+        }
+        // Find overlapping or adjacent sessions to merge
+        const windowsToMerge = [newWindow];
+        for (const w of existingWindows.values()) {
+            // Check if windows should merge (overlap or within gap)
+            const overlapOrAdjacent = (newWindow.start <= w.end && newWindow.end >= w.start) ||
+                Math.abs(newWindow.start - w.end) <= this.gapMs ||
+                Math.abs(w.start - newWindow.end) <= this.gapMs;
+            if (overlapOrAdjacent) {
+                windowsToMerge.push(w);
+            }
+        }
+        if (windowsToMerge.length === 1) {
+            return [newWindow];
+        }
+        // Merge all overlapping windows
+        const mergedStart = Math.min(...windowsToMerge.map((w) => w.start));
+        const maxTimestamp = Math.max(timestamp, ...windowsToMerge.map((w) => w.end - this.gapMs));
+        const mergedEnd = maxTimestamp + this.gapMs;
+        return [{ start: mergedStart, end: mergedEnd }];
+    }
+}
+/**
+ * Implementation of WindowState
+ */
+class WindowStateImpl {
+    backend;
+    assigner;
+    constructor(backend, assigner) {
+        this.backend = backend;
+        this.assigner = assigner;
+    }
+    encodeWindowKey(key, start, end) {
+        return `${key}:${start}:${end}`;
+    }
+    decodeWindowKey(encoded) {
+        const parts = encoded.split(':');
+        const end = parseInt(parts.pop(), 10);
+        const start = parseInt(parts.pop(), 10);
+        const key = parts.join(':');
+        return { key, windowStart: start, windowEnd: end };
+    }
+    async getWindow(key, start, end) {
+        const windowKey = this.encodeWindowKey(key, start, end);
+        return this.backend.get(windowKey);
+    }
+    async updateWindow(key, start, end, fn) {
+        const windowKey = this.encodeWindowKey(key, start, end);
+        const current = await this.backend.get(windowKey);
+        const updated = fn(current);
+        await this.backend.put(windowKey, updated);
+        return updated;
+    }
+    async expireWindows(before) {
+        let expired = 0;
+        const keysToDelete = [];
+        for await (const [encodedKey] of this.backend.list()) {
+            const windowKey = this.decodeWindowKey(encodedKey);
+            if (windowKey.windowEnd < before) {
+                keysToDelete.push(encodedKey);
+                expired++;
+            }
+        }
+        for (const key of keysToDelete) {
+            await this.backend.delete(key);
+        }
+        return expired;
+    }
+    async *activeWindows(key) {
+        const prefix = `${key}:`;
+        for await (const [encodedKey] of this.backend.list(prefix)) {
+            yield this.decodeWindowKey(encodedKey);
+        }
+    }
+}
+/**
+ * Factory function to create a WindowState
+ */
+export function createWindowState(backend, assigner) {
+    return new WindowStateImpl(backend, assigner);
+}
+/**
+ * Aligns checkpoint barriers from multiple input channels
+ */
+export class BarrierAligner {
+    inputChannels;
+    onAligned;
+    pendingBarriers = new Map();
+    blockedChannels = new Set();
+    constructor(inputChannels, onAligned) {
+        this.inputChannels = inputChannels;
+        this.onAligned = onAligned;
+    }
+    async processBarrier(channel, barrier) {
+        this.pendingBarriers.set(channel, barrier);
+        this.blockedChannels.add(channel);
+        if (this.allBarriersReceived(barrier.checkpointId)) {
+            await this.onAligned(barrier);
+            this.releaseBlocked();
+        }
+    }
+    isBlocked(channel) {
+        return this.blockedChannels.has(channel);
+    }
+    allBarriersReceived(checkpointId) {
+        return this.inputChannels.every((ch) => this.pendingBarriers.get(ch)?.checkpointId === checkpointId);
+    }
+    releaseBlocked() {
+        this.pendingBarriers.clear();
+        this.blockedChannels.clear();
+    }
+}
+/**
+ * Coordinates checkpoint barriers across operators
+ */
+export class CheckpointCoordinator {
+    config;
+    nextCheckpointId = 1n;
+    pendingCheckpoints = new Map();
+    constructor(config) {
+        this.config = config;
+    }
+    async triggerCheckpoint(options) {
+        const checkpointId = this.nextCheckpointId++;
+        const startTime = Date.now();
+        const timeout = options?.timeout ?? this.config.defaultTimeout ?? 60000;
+        const barrier = {
+            checkpointId,
+            timestamp: startTime,
+            options: {
+                mode: options?.mode ?? 'aligned',
+                timeout,
+                savepoint: options?.savepoint,
+            },
+        };
+        // Track pending checkpoint
+        this.pendingCheckpoints.set(checkpointId, {
+            startTime,
+            acknowledgedSinks: new Set(),
+            options: barrier.options,
+        });
+        try {
+            // Inject barrier at all sources
+            await Promise.all(this.config.sources.map((s) => s.injectBarrier(barrier)));
+            // Wait for all sinks to acknowledge with timeout
+            const result = await this.waitForCompletion(checkpointId, timeout);
+            return {
+                checkpointId,
+                status: result ? 'completed' : 'failed',
+                failureReason: result ? undefined : 'Timeout waiting for sink acknowledgements',
+                isSavepoint: options?.savepoint,
+                duration: Date.now() - startTime,
+            };
+        }
+        catch (error) {
+            return {
+                checkpointId,
+                status: 'failed',
+                failureReason: error instanceof Error ? error.message : 'Unknown error',
+                isSavepoint: options?.savepoint,
+                duration: Date.now() - startTime,
+            };
+        }
+        finally {
+            this.pendingCheckpoints.delete(checkpointId);
+        }
+    }
+    acknowledgeSink(checkpointId, sinkIndex) {
+        const pending = this.pendingCheckpoints.get(checkpointId);
+        if (pending) {
+            pending.acknowledgedSinks.add(sinkIndex);
+        }
+    }
+    async waitForCompletion(checkpointId, timeout) {
+        const pending = this.pendingCheckpoints.get(checkpointId);
+        if (!pending)
+            return false;
+        const numSinks = this.config.sinks.length;
+        // Create a timeout promise
+        const timeoutPromise = new Promise((resolve) => {
+            setTimeout(() => resolve('timeout'), timeout);
+        });
+        // Create acknowledgement promises with individual timeouts
+        const ackPromises = this.config.sinks.map((sink, index) => Promise.race([
+            sink.acknowledge(checkpointId).then(() => {
+                this.acknowledgeSink(checkpointId, index);
+                return 'ack';
+            }),
+            timeoutPromise,
+        ]).catch(() => 'error'));
+        // Wait for all with overall timeout
+        const results = await Promise.race([
+            Promise.all(ackPromises),
+            timeoutPromise.then(() => 'timeout'),
+        ]);
+        if (results === 'timeout') {
+            return false;
+        }
+        // Check if all sinks acknowledged
+        return pending.acknowledgedSinks.size >= numSinks;
+    }
+}
+/**
+ * TTL wrapper for StateBackend
+ */
+export class TTLStateWrapper {
+    inner;
+    policy;
+    scheduler;
+    ttlEntries = new Map();
+    constructor(inner, policy, scheduler) {
+        this.inner = inner;
+        this.policy = policy;
+        this.scheduler = scheduler;
+        if (policy.cleanupStrategy.type === 'eager' && scheduler) {
+            scheduler.schedule(() => this.runCleanup(), policy.cleanupStrategy.intervalMs);
+        }
+    }
+    get name() {
+        return this.inner.name;
+    }
+    async get(key) {
+        const entry = this.ttlEntries.get(key);
+        const now = Date.now();
+        if (!entry) {
+            return undefined;
+        }
+        // Check expiration
+        if (this.isExpired(entry, now)) {
+            this.ttlEntries.delete(key);
+            await this.inner.delete(key);
+            return undefined;
+        }
+        // Update TTL on read if configured
+        if (this.policy.updateOnRead) {
+            entry.lastAccessedAt = now;
+            entry.expiresAt = now + this.policy.ttlMs;
+        }
+        return entry.value;
+    }
+    async put(key, value, ttl) {
+        const now = Date.now();
+        const effectiveTtl = ttl ?? this.policy.ttlMs;
+        const entry = {
+            value,
+            createdAt: now,
+            lastAccessedAt: now,
+            expiresAt: now + effectiveTtl,
+        };
+        this.ttlEntries.set(key, entry);
+        await this.inner.put(key, value, effectiveTtl);
+    }
+    async delete(key) {
+        this.ttlEntries.delete(key);
+        await this.inner.delete(key);
+    }
+    async *list(prefix) {
+        const now = Date.now();
+        for (const [key, entry] of this.ttlEntries) {
+            if (prefix !== undefined && !key.startsWith(prefix)) {
+                continue;
+            }
+            if (!this.isExpired(entry, now)) {
+                yield [key, entry.value];
+            }
+        }
+    }
+    async size() {
+        const now = Date.now();
+        let count = 0;
+        for (const entry of this.ttlEntries.values()) {
+            if (!this.isExpired(entry, now)) {
+                count++;
+            }
+        }
+        return count;
+    }
+    async snapshot() {
+        return this.inner.snapshot();
+    }
+    async restore(snapshot) {
+        await this.inner.restore(snapshot);
+        // Clear TTL entries and rebuild from inner state
+        this.ttlEntries.clear();
+        const now = Date.now();
+        for await (const [key, value] of this.inner.list()) {
+            this.ttlEntries.set(key, {
+                value,
+                createdAt: now,
+                lastAccessedAt: now,
+                expiresAt: now + this.policy.ttlMs,
+            });
+        }
+    }
+    async clear() {
+        this.ttlEntries.clear();
+        await this.inner.clear();
+    }
+    async runCleanup() {
+        const now = Date.now();
+        let cleaned = 0;
+        const keysToDelete = [];
+        for (const [key, entry] of this.ttlEntries) {
+            if (this.isExpired(entry, now)) {
+                keysToDelete.push(key);
+                cleaned++;
+            }
+        }
+        for (const key of keysToDelete) {
+            this.ttlEntries.delete(key);
+            await this.inner.delete(key);
+        }
+        return { cleanedCount: cleaned, timestamp: now };
+    }
+    async onWatermark(watermark) {
+        if (this.policy.cleanupStrategy.type === 'watermark') {
+            const allowedLateness = this.policy.cleanupStrategy.allowedLateness;
+            const cutoff = watermark - allowedLateness;
+            const keysToDelete = [];
+            for (const [key, entry] of this.ttlEntries) {
+                if (entry.createdAt < cutoff) {
+                    keysToDelete.push(key);
+                }
+            }
+            for (const key of keysToDelete) {
+                this.ttlEntries.delete(key);
+                await this.inner.delete(key);
+            }
+        }
+    }
+    isExpired(entry, now) {
+        return now >= entry.expiresAt;
+    }
+}
+/**
+ * Manages state recovery from checkpoints
+ */
+export class StateRecoveryManager {
+    checkpointStore;
+    operators;
+    constructor(checkpointStore, operators) {
+        this.checkpointStore = checkpointStore;
+        this.operators = operators;
+    }
+    async findLatestCheckpoint() {
+        const checkpoints = await this.checkpointStore.listCompleted();
+        if (checkpoints.length === 0) {
+            return undefined;
+        }
+        return checkpoints.sort((a, b) => Number(b.checkpointId - a.checkpointId))[0];
+    }
+    async recover(checkpointId) {
+        const checkpoint = checkpointId
+            ? await this.checkpointStore.get(checkpointId)
+            : await this.findLatestCheckpoint();
+        if (!checkpoint) {
+            throw new Error('No checkpoint available for recovery');
+        }
+        // Restore each operator's state
+        for (const [opId, snapshotId] of checkpoint.operatorStates) {
+            const operator = this.operators.get(opId);
+            if (operator) {
+                const snapshot = await this.checkpointStore.getSnapshot(snapshotId);
+                await operator.restoreState(snapshot);
+            }
+        }
+    }
+}
+/**
+ * Simple counter implementation
+ */
+class SimpleCounter {
+    count = 0;
+    inc(amount = 1) {
+        this.count += amount;
+    }
+    value() {
+        return this.count;
+    }
+}
+/**
+ * Simple gauge implementation
+ */
+class SimpleGauge {
+    val = 0;
+    set(value) {
+        this.val = value;
+    }
+    value() {
+        return this.val;
+    }
+}
+/**
+ * Simple histogram implementation
+ */
+class SimpleHistogram {
+    observations = [];
+    observe(value) {
+        this.observations.push(value);
+    }
+    values() {
+        return [...this.observations];
+    }
+}
+/**
+ * Create a metrics registry
+ */
+export function createStateMetrics() {
+    return {
+        createCounter: () => new SimpleCounter(),
+        createGauge: () => new SimpleGauge(),
+        createHistogram: () => new SimpleHistogram(),
+    };
+}
+/**
+ * Metrics wrapper for StateBackend
+ */
+export class MetricsStateWrapper {
+    inner;
+    registry;
+    operatorName;
+    metrics;
+    constructor(inner, registry, operatorName) {
+        this.inner = inner;
+        this.registry = registry;
+        this.operatorName = operatorName;
+        this.metrics = {
+            keyCount: 0,
+            totalBytes: 0,
+            largestKey: { key: '', bytes: 0 },
+            reads: registry.createCounter(`${operatorName}_reads`),
+            writes: registry.createCounter(`${operatorName}_writes`),
+            deletes: registry.createCounter(`${operatorName}_deletes`),
+            cacheHits: registry.createCounter(`${operatorName}_cache_hits`),
+            cacheMisses: registry.createCounter(`${operatorName}_cache_misses`),
+            lastCheckpointDuration: registry.createGauge(`${operatorName}_checkpoint_duration`),
+            lastCheckpointSize: registry.createGauge(`${operatorName}_checkpoint_size`),
+            checkpointFailures: registry.createCounter(`${operatorName}_checkpoint_failures`),
+            expiredKeys: registry.createCounter(`${operatorName}_expired_keys`),
+            cleanupDuration: registry.createHistogram(`${operatorName}_cleanup_duration`),
+        };
+    }
+    get name() {
+        return this.inner.name;
+    }
+    async get(key) {
+        this.metrics.reads.inc();
+        return this.inner.get(key);
+    }
+    async put(key, value, ttl) {
+        this.metrics.writes.inc();
+        await this.inner.put(key, value, ttl);
+        await this.updateSizeMetrics();
+    }
+    async delete(key) {
+        this.metrics.deletes.inc();
+        await this.inner.delete(key);
+        await this.updateSizeMetrics();
+    }
+    async *list(prefix) {
+        for await (const entry of this.inner.list(prefix)) {
+            yield entry;
+        }
+    }
+    async size() {
+        return this.inner.size();
+    }
+    async snapshot() {
+        const start = performance.now();
+        try {
+            const snapshot = await this.inner.snapshot();
+            this.metrics.lastCheckpointDuration.set(performance.now() - start);
+            this.metrics.lastCheckpointSize.set(snapshot.metadata.totalBytes);
+            return snapshot;
+        }
+        catch (error) {
+            this.metrics.checkpointFailures.inc();
+            throw error;
+        }
+    }
+    async restore(snapshot) {
+        await this.inner.restore(snapshot);
+        await this.updateSizeMetrics();
+    }
+    async clear() {
+        await this.inner.clear();
+        this.metrics.keyCount = 0;
+        this.metrics.totalBytes = 0;
+    }
+    getMetrics() {
+        return this.metrics;
+    }
+    async updateSizeMetrics() {
+        const size = await this.inner.size();
+        this.metrics.keyCount = size;
+        // Estimate total bytes (simplified)
+        let totalBytes = 0;
+        let largestKey = { key: '', bytes: 0 };
+        for await (const [key, value] of this.inner.list()) {
+            const bytes = JSON.stringify(value).length + key.length;
+            totalBytes += bytes;
+            if (bytes > largestKey.bytes) {
+                largestKey = { key, bytes };
+            }
+        }
+        this.metrics.totalBytes = totalBytes;
+        this.metrics.largestKey = largestKey;
+    }
+}
+//# sourceMappingURL=index.js.map