@playola-radio/slipstream 0.1.0

package/dist/clip-blob-reader.js

@@ -0,0 +1,104 @@
+/**
+ * The I/O-scoped clip module: resolve a change's before/after snapshots to raw
+ * blob bytes from the content-addressed store and run the pure
+ * {@link projectClips} core. This is the reusable entry the TUI / any
+ * independent client calls directly against on-disk artifacts, and the same
+ * function the projection worker runs off the main thread. It reads RAW bytes
+ * (not decoded text): the core owns UTF-8 validation and needs bytes for
+ * byte-offset spans.
+ */
+import { constants } from 'node:fs';
+import { open } from 'node:fs/promises';
+import { blobPath, isValidHex } from "./store-reader.js";
+import { projectClips, MAX_UTF8_BYTES, } from "./clip-projection.js";
+/** Parse an event snapshot (`data.before` / `data.after`) into a
+ *  {@link ClipSnapshot}, or null if malformed. Mirrors change-view's parser. */
+export function parseClipSnapshot(v) {
+    if (typeof v !== 'object' || v === null)
+        return null;
+    const s = v;
+    if (s.kind === 'content') {
+        if (typeof s.sha256 === 'string' && isValidHex(s.sha256)
+            && typeof s.size === 'number' && Number.isInteger(s.size) && s.size >= 0) {
+            return { kind: 'content', sha256: s.sha256, size: s.size };
+        }
+        return null;
+    }
+    if (s.kind === 'absent')
+        return { kind: 'absent' };
+    if (s.kind === 'unavailable' && typeof s.reason === 'string') {
+        return { kind: 'unavailable', reason: s.reason };
+    }
+    return null;
+}
+/** Resolve one snapshot to a {@link SideInput}. A content blob larger than
+ *  `maxBytes`, or one that under-reports its size, becomes `oversize`; a blob
+ *  that cannot be opened/read (GC'd, missing, unreadable) becomes `missing` with
+ *  an explicit reason — never faked content. */
+export async function resolveClipSide(storeDir, snap, maxBytes, onPhase) {
+    const startedAtNs = onPhase ? process.hrtime.bigint() : undefined;
+    let readOccurred = false;
+    try {
+        if (snap.kind === 'absent')
+            return { kind: 'absent' };
+        if (snap.kind === 'unavailable')
+            return { kind: 'unavailable', reason: snap.reason };
+        if (snap.size > maxBytes)
+            return { kind: 'oversize' };
+        if (!isValidHex(snap.sha256))
+            return { kind: 'missing', reason: 'invalid-hex' };
+        // O_NOFOLLOW: a symlink planted at a valid CAS path must serve nothing but the
+        // blob it names, never the link target's bytes.
+        let handle;
+        try {
+            handle = await open(blobPath(storeDir, snap.sha256), constants.O_RDONLY | constants.O_NOFOLLOW);
+        }
+        catch (err) {
+            return { kind: 'missing', reason: err.code ?? 'open-failed' };
+        }
+        try {
+            const { size } = await handle.stat();
+            if (size > maxBytes)
+                return { kind: 'oversize' };
+            const bytes = await handle.readFile();
+            readOccurred = true;
+            return { kind: 'bytes', bytes };
+        }
+        catch (err) {
+            return { kind: 'missing', reason: err.code ?? 'read-failed' };
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    finally {
+        if (readOccurred && startedAtNs !== undefined)
+            onPhase?.('cas-read', process.hrtime.bigint() - startedAtNs);
+    }
+}
+/** Resolve both sides and run the pure projection. Successful spans are determined by
+ *  bytes, language and version; timeouts/availability remain explicit transient
+ *  outcomes. The caller decides where it runs (worker or directly). */
+export async function computeClipProjection(job, onPhase) {
+    const maxBytes = job.opts.maxBytes ?? MAX_UTF8_BYTES;
+    const [before, after] = await Promise.all([
+        resolveClipSide(job.storeDir, job.before, maxBytes, onPhase),
+        resolveClipSide(job.storeDir, job.after, maxBytes, onPhase),
+    ]);
+    const language = job.opts.language ?? 'unsupported';
+    if (language === 'unsupported' || (before.kind !== 'bytes' && after.kind !== 'bytes')) {
+        return projectClips(before, after, job.opts, () => ({ functions: [], errors: [], reason: 'unsupported-language' }));
+    }
+    // Only supported-content computations load the parser. HTTP snapshot
+    // validation and honest unsupported/unavailable results need no WASM startup.
+    const loadStartedAtNs = onPhase ? process.hrtime.bigint() : undefined;
+    const { createFunctionIndexer } = await import("./clip-function-parser.js");
+    const indexer = await createFunctionIndexer(language);
+    if (loadStartedAtNs !== undefined)
+        onPhase?.('grammar-load', process.hrtime.bigint() - loadStartedAtNs);
+    const parseStartedAtNs = onPhase ? process.hrtime.bigint() : undefined;
+    const result = projectClips(before, after, job.opts, indexer);
+    if (parseStartedAtNs !== undefined)
+        onPhase?.('parse-compare', process.hrtime.bigint() - parseStartedAtNs);
+    return result;
+}

package/dist/clip-function-parser.js

@@ -0,0 +1,83 @@
+import { createRequire } from 'node:module';
+import { Parser, Language } from 'web-tree-sitter';
+// Computing callers prepare only the requested grammar. Initialization remains
+// inside the existing job/deadline; unused grammars do not tax cold requests.
+const require = createRequire(import.meta.url);
+let initialization;
+const grammars = new Map();
+/** Return a synchronous extractor bound to one language. Promise caching joins
+ * concurrent direct callers; JS/JSX share one grammar. Failed loads can retry. */
+export async function createFunctionIndexer(language) {
+    if (language === 'unsupported')
+        return () => ({ functions: [], errors: [], reason: 'unsupported-language' });
+    const name = language === 'jsx' ? 'javascript' : language;
+    let pending = grammars.get(name);
+    if (!pending) {
+        pending = (async () => {
+            initialization ??= Parser.init().catch(error => { initialization = undefined; throw error; });
+            await initialization;
+            return Language.load(require.resolve(`tree-sitter-wasms/out/tree-sitter-${name}.wasm`));
+        })().catch(error => { grammars.delete(name); throw error; });
+        grammars.set(name, pending);
+    }
+    const grammar = await pending;
+    return text => indexFunctions(text, grammar);
+}
+const FUNCTION_TYPES = new Set([
+    'function_declaration', 'function_expression', 'generator_function_declaration',
+    'generator_function', 'arrow_function', 'method_definition',
+]);
+const MAX_NODES = 20_000;
+// Cooperative per-side parse limit; the existing worker deadline bounds the
+// whole job, including startup, syntax indexing and diffing.
+const PARSE_TIMEOUT_MS = 20;
+/** Parse each immutable side once. Positions are converted by ROW, never by
+ * parser startIndex (UTF-16, not a raw-blob byte offset). Trees/nodes are local
+ * to this call; no retained parse state or incremental edits.
+ */
+function indexFunctions(text, grammar) {
+    const empty = (reason) => ({ functions: [], errors: [], reason });
+    const parser = new Parser();
+    let tree = null;
+    try {
+        parser.setLanguage(grammar);
+        const deadline = performance.now() + PARSE_TIMEOUT_MS;
+        tree = parser.parse(text, null, { progressCallback: () => performance.now() > deadline });
+        if (!tree)
+            return empty('timeout');
+        const result = { functions: [], errors: [] };
+        const stack = [
+            { node: tree.rootNode, unreliable: false },
+        ];
+        let visited = 0;
+        while (stack.length) {
+            if (++visited > MAX_NODES)
+                return empty('extraction-budget-exhausted');
+            const { node, unreliable } = stack.pop();
+            const damaged = unreliable || node.isError || node.isMissing;
+            const range = {
+                s0: node.startPosition.row,
+                e0: node.endPosition.row + (node.endPosition.column > 0 ? 1 : 0),
+                c0: node.startPosition.column,
+                c1: node.endPosition.column,
+            };
+            if (node.isError || node.isMissing) {
+                result.errors.push({ ...range, e0: Math.max(range.e0, range.s0 + 1) });
+            }
+            if (FUNCTION_TYPES.has(node.type) && !damaged && !node.hasError) {
+                result.functions.push(range);
+            }
+            // Include anonymous missing tokens as well: namedChildren would miss them.
+            for (let i = node.childCount - 1; i >= 0; i--) {
+                const child = node.child(i);
+                if (child)
+                    stack.push({ node: child, unreliable: damaged });
+            }
+        }
+        return result;
+    }
+    finally {
+        tree?.delete();
+        parser.delete();
+    }
+}

package/dist/clip-language.js

@@ -0,0 +1,15 @@
+import { extname } from 'node:path';
+/** Filename selection is public input, never an inference from captured bytes. */
+export function languageForPath(path) {
+    switch (extname(path).toLowerCase()) {
+        case '.js':
+        case '.mjs':
+        case '.cjs': return 'javascript';
+        case '.jsx': return 'jsx';
+        case '.ts':
+        case '.mts':
+        case '.cts': return 'typescript';
+        case '.tsx': return 'tsx';
+        default: return 'unsupported';
+    }
+}

package/dist/clip-projection-service.js

@@ -0,0 +1,240 @@
+/**
+ * Main-thread orchestrator for the clip projection. It turns a change's
+ * before/after snapshots into a {@link ClipProjection}, computed on demand from
+ * the immutable blobs and cached disposably (no log, no persistence). It layers
+ * four concerns on top of the pure core:
+ *
+ *  - Content-addressed LRU cache: keyed by (before tag, after tag,
+ *    projection_version, language) so identical inputs across changes reuse one result,
+ *    bounded by entry count AND estimated bytes. change_seq is response-only and
+ *    is stamped on the way out, never part of the key.
+ *  - Bounded admission via the shared projection budget: at most `concurrency`
+ *    computes run at once behind a bounded queue; excess is returned immediately
+ *    as `skipped`/`overloaded` (never a silent stall), so a burst of cold-cache
+ *    requests cannot starve capture. The deadline is measured from admission
+ *    (queue wait included). Coalescing and the deadline live in the budget so
+ *    the same bound covers a future interface-projection workload too. The CPU
+ *    itself runs off the shared event loop in a worker.
+ *  - Revalidate-on-hit: a cache hit is dropped if a referenced blob is gone
+ *    (GC'd), so a stale result never masquerades as available.
+ *
+ * Availability, overload, and timeout dispositions are NOT pure functions of
+ * content, so they are never cached.
+ */
+import { constants } from 'node:fs';
+import { access } from 'node:fs/promises';
+import { blobPath } from "./store-reader.js";
+import { CLIP_PROJECTION_VERSION, } from "./clip-projection.js";
+import {} from "./clip-blob-reader.js";
+import { createClipWorkerPool } from "./clip-worker-pool.js";
+import { createProjectionAdmission } from "./projection-admission.js";
+import { emitProjectionPhase, emitProjectionTrace } from "./projection-trace.js";
+// B1 runs exactly one clip worker (CPU stays off the capture path); concurrency
+// is a fixed design constant, not a knob.
+const CONCURRENCY = 1;
+const DEFAULTS = {
+    queueLimit: 8,
+    cacheEntries: 512,
+    cacheBytes: 4 * 1024 * 1024,
+    deadlineMs: 100,
+};
+/** Reasons that are transient or availability-dependent — never cached. */
+const UNCACHEABLE_REASONS = new Set(['overloaded', 'timeout', 'worker-error']);
+function sideTag(s) {
+    if (s.kind === 'content')
+        return `content:${s.sha256}`;
+    if (s.kind === 'absent')
+        return 'absent';
+    return `unavailable:${s.reason}`;
+}
+function cacheKey(req) {
+    return `${sideTag(req.before)}|${sideTag(req.after)}|${CLIP_PROJECTION_VERSION}|${req.language ?? 'unsupported'}`;
+}
+function contentShas(req) {
+    const out = [];
+    if (req.before.kind === 'content')
+        out.push(req.before.sha256);
+    if (req.after.kind === 'content')
+        out.push(req.after.sha256);
+    return out;
+}
+function transient(changeSeq, reason) {
+    return {
+        change_seq: changeSeq,
+        projection_version: CLIP_PROJECTION_VERSION,
+        status: 'skipped',
+        fallback_reason: reason,
+        clips: [],
+    };
+}
+function cacheable(value) {
+    if (value.status === 'unavailable')
+        return false;
+    if (value.fallback_reason !== undefined && UNCACHEABLE_REASONS.has(value.fallback_reason))
+        return false;
+    // A per-side `unavailable` method reflects blob presence at compute time, not
+    // content, so a later restore must recompute rather than serve the stale
+    // "before-missing" disposition. Revalidate-on-hit can't catch this: the
+    // snapshot still names a content sha whose blob is now back.
+    for (const clip of value.clips) {
+        if (clip.before.method === 'unavailable' || clip.after.method === 'unavailable')
+            return false;
+    }
+    return true;
+}
+/** Insertion-ordered LRU bounded by entry count and estimated bytes. */
+class LruCache {
+    map = new Map();
+    bytes = 0;
+    maxEntries;
+    maxBytes;
+    constructor(maxEntries, maxBytes) {
+        this.maxEntries = maxEntries;
+        this.maxBytes = maxBytes;
+    }
+    get(key) {
+        const entry = this.map.get(key);
+        if (!entry)
+            return undefined;
+        this.map.delete(key);
+        this.map.set(key, entry); // move to most-recently-used
+        return entry.value;
+    }
+    set(key, value) {
+        const existing = this.map.get(key);
+        if (existing) {
+            this.bytes -= existing.bytes;
+            this.map.delete(key);
+        }
+        const bytes = JSON.stringify(value).length;
+        this.map.set(key, { value, bytes });
+        this.bytes += bytes;
+        while (this.map.size > this.maxEntries || this.bytes > this.maxBytes) {
+            const oldest = this.map.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.bytes -= this.map.get(oldest).bytes;
+            this.map.delete(oldest);
+        }
+    }
+    delete(key) {
+        const entry = this.map.get(key);
+        if (entry) {
+            this.bytes -= entry.bytes;
+            this.map.delete(key);
+        }
+    }
+}
+export function createClipProjectionService(opts) {
+    if (opts.admission && (opts.queueLimit !== undefined || opts.deadlineMs !== undefined)) {
+        throw new Error('shared clip admission cannot also set private queue or deadline');
+    }
+    const queueLimit = opts.queueLimit ?? DEFAULTS.queueLimit;
+    const deadlineMs = opts.deadlineMs ?? DEFAULTS.deadlineMs;
+    const pool = opts.compute ? null : createClipWorkerPool(opts.projectionTrace);
+    const compute = opts.compute ?? pool.run;
+    const hasBlob = opts.hasBlob ?? ((sha256) => access(blobPath(opts.storeDir, sha256), constants.F_OK).then(() => true, () => false));
+    const cache = new LruCache(opts.cacheEntries ?? DEFAULTS.cacheEntries, opts.cacheBytes ?? DEFAULTS.cacheBytes);
+    // Concurrency, queue, coalesced-waiter bound, and the admission-time deadline
+    // live in the shared budget. Clip's own budget uses C = CONCURRENCY (1) and
+    // W = Q = queueLimit, so the outstanding ceiling stays C + queueLimit exactly
+    // (coalesced waiters counted), reproducing the prior `maxPending` bound. The
+    // budget is workload-agnostic so a future interface service can share one
+    // instance; wiring that shared instance is a later step (see ADMISSION.md).
+    const budget = opts.admission ?? createProjectionAdmission({
+        C: CONCURRENCY,
+        Q: queueLimit,
+        W: queueLimit,
+        D: deadlineMs,
+    }, opts.projectionTrace);
+    let closed = false;
+    const stamp = (value, changeSeq) => ({ ...value, change_seq: changeSeq });
+    const blobsPresent = async (req) => {
+        for (const sha of contentShas(req))
+            if (!(await hasBlob(sha)))
+                return false;
+        return true;
+    };
+    const get = async (req) => {
+        if (closed)
+            return transient(req.changeSeq, 'worker-error');
+        const key = cacheKey(req);
+        const hit = cache.get(key);
+        if (hit) {
+            if (await blobsPresent(req)) {
+                if (req.traceRouteKey !== undefined)
+                    emitProjectionTrace(opts.projectionTrace, { kind: 'clip-cache-bypass', routeKey: req.traceRouteKey, atNs: process.hrtime.bigint() });
+                return stamp(hit, req.changeSeq);
+            }
+            cache.delete(key); // referenced blob GC'd — never serve a stale hit
+        }
+        // Only the coalescing leader's `run` is ever invoked (the budget calls it
+        // once per flight); a coalesced waiter never sees this flip, so only the
+        // leader writes the cache entry — waiters would otherwise redundantly
+        // re-store the identical value the leader already cached.
+        let isLeader = false;
+        const outcome = await budget.admit({
+            workload: 'clip',
+            localConcurrency: CONCURRENCY,
+            key,
+            traceRouteKey: req.traceRouteKey,
+            run: traceUnitId => {
+                isLeader = true;
+                const start = () => {
+                    if (traceUnitId !== undefined)
+                        emitProjectionTrace(opts.projectionTrace, { kind: 'parser-request', unitId: traceUnitId, atNs: process.hrtime.bigint() });
+                    return compute({
+                        storeDir: opts.storeDir,
+                        before: req.before,
+                        after: req.after,
+                        opts: { changeSeq: req.changeSeq, language: req.language },
+                    }, traceUnitId);
+                };
+                if (!opts.dispatchBarrier)
+                    return start();
+                const beganAtNs = opts.projectionTrace ? process.hrtime.bigint() : undefined;
+                let cancelBarrier;
+                const cancelled = new Promise((_resolve, reject) => { cancelBarrier = reject; });
+                let inner;
+                let wasCancelled = false;
+                const promise = Promise.race([Promise.resolve().then(opts.dispatchBarrier), cancelled]).then(() => {
+                    if (wasCancelled)
+                        throw new Error('clip dispatch cancelled');
+                    emitProjectionPhase(opts.projectionTrace, 'dispatch-barrier', beganAtNs, { scope: 'clip', unitId: traceUnitId });
+                    inner = start();
+                    return inner.promise;
+                });
+                return { promise, cancel: () => {
+                        wasCancelled = true;
+                        cancelBarrier(new Error('clip dispatch cancelled'));
+                        inner?.cancel();
+                    } };
+            },
+        });
+        switch (outcome.kind) {
+            case 'ok':
+                // Availability, overload, and timeout dispositions are never cached.
+                if (isLeader && cacheable(outcome.value))
+                    cache.set(key, outcome.value);
+                return stamp(outcome.value, req.changeSeq);
+            case 'timeout':
+                return transient(req.changeSeq, 'timeout');
+            case 'overloaded':
+                return transient(req.changeSeq, 'overloaded');
+            // A closed budget or a failed compute both surface as worker-error, the
+            // reason the service has always used for "no result from the worker".
+            default:
+                return transient(req.changeSeq, 'worker-error');
+        }
+    };
+    const close = async () => {
+        closed = true;
+        // Settle everything admitted to this service's budget, then terminate the
+        // worker. (The budget here is private to this service; closing it never
+        // affects another workload.)
+        if (!opts.admission)
+            await budget.close();
+        await pool?.close();
+    };
+    return { get, close };
+}

package/dist/clip-projection-worker.js

@@ -0,0 +1,18 @@
+/**
+ * Worker-thread entry for clip projection. It runs the CPU-bound diff off the
+ * shared daemon event loop so a burst of cold-cache projection requests cannot
+ * starve capture. It is a thin wrapper: resolve blob bytes and run the pure
+ * core via {@link computeClipProjection}. One request/response per message,
+ * keyed by id; the service owns pooling, admission, and the wall-clock deadline
+ * (it terminates and replaces this worker on overrun — a stuck diff cannot be
+ * interrupted from inside).
+ */
+import { parentPort } from 'node:worker_threads';
+import { computeClipProjection } from "./clip-blob-reader.js";
+if (!parentPort)
+    throw new Error('clip-projection-worker must run as a worker thread');
+const port = parentPort;
+port.on('message', (msg) => {
+    const traceTimings = msg.traceTimings ? [] : undefined;
+    computeClipProjection(msg.job, traceTimings ? (phase, durationNs) => traceTimings.push({ phase, durationNs }) : undefined).then((result) => port.postMessage({ id: msg.id, ok: true, result, traceTimings }), () => port.postMessage({ id: msg.id, ok: false }));
+});