@they-juanreina/compost-cli 0.1.0-rc.1

package/dist/loops/embed_worker.js

@@ -0,0 +1,144 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { buildVectorRecords, chunkTranscript, openLanceDBForWrite } from 'compost-retrieval';
+import { CompostError } from '../errors.js';
+import { loadConfig, parseRoute } from '../lib/config.js';
+import { LLMAdapter } from '../llm/adapter.js';
+export async function runEmbedWorkerOnce(seedPath, deps = {}) {
+    const seedName = seedPath.split('/').pop() ?? 'seed';
+    const transcripts = findTranscripts(seedPath);
+    if (transcripts.length === 0) {
+        return { embedded: 0, inserted: 0, transcripts_scanned: 0 };
+    }
+    // Resolve embeddings provider + dimension.
+    let embed = deps.embed;
+    let vectorDim = deps.vectorDim ?? 1024;
+    if (embed === undefined) {
+        const config = loadConfig(seedPath);
+        const adapter = new LLMAdapter(config);
+        embed = async (texts) => {
+            const resp = await adapter.embed('embeddings', texts);
+            return resp.vectors;
+        };
+        // Try to discover dim from the provider config — most embeddings models
+        // declare it in their tag. bge-m3 is 1024; mxbai-embed-large is 1024 too.
+        // Fall back to 1024 by default; mismatches surface as a clear LanceDB error.
+        const route = config.defaults.embeddings;
+        if (route !== undefined) {
+            const { model } = parseRoute(route);
+            vectorDim = DEFAULT_DIM_FOR_MODEL[model] ?? 1024;
+        }
+    }
+    // Chunk every transcript.
+    const allChunks = [];
+    for (const t of transcripts) {
+        const transcript = JSON.parse(readFileSync(t.path, 'utf8'));
+        const chunks = chunkTranscript(transcript, { seed: seedName });
+        for (const c of chunks)
+            allChunks.push({ ...c, sessionId: t.sessionId });
+    }
+    if (allChunks.length === 0) {
+        return { embedded: 0, inserted: 0, transcripts_scanned: transcripts.length };
+    }
+    // Open / create the LanceDB table, then filter for new SHAs before embedding
+    // (avoid wasting embed calls on content we'd dedup away anyway).
+    const writer = deps.writer ??
+        (await openLanceDBForWrite(join(seedPath, '.compost', 'vectors.lancedb'), vectorDim));
+    // Embed with a worker-level batch cap. The LLMAdapter already batches
+    // internally (provider-appropriate, ~50/req for Ollama), but a worker-level
+    // cap is defense-in-depth: a very large corpus (10k+ chunks) shouldn't be
+    // a single multi-megabyte JSON request even if the adapter would split it.
+    // 500 chunks/pass is a safe ceiling for HTTP body size on default Ollama
+    // configs while keeping the round-trip count low (~36 batches for a
+    // typical 18k-chunk corpus).
+    const vectors = await embedInBatches(embed, allChunks.map((c) => c.text), EMBED_BATCH_CAP);
+    if (vectors.length !== allChunks.length) {
+        throw new CompostError('PROVIDER_ERROR', `embeddings provider returned ${vectors.length} vectors for ${allChunks.length} chunks`);
+    }
+    // Build LanceDB rows.
+    const records = buildVectorRecords(allChunks.map((c, i) => ({
+        id: c.id,
+        kind: chunkTypeToArtifactKind(c.metadata.chunk_type),
+        seed: seedName,
+        session: c.sessionId,
+        speaker_id: c.metadata.speaker_id,
+        start_ms: c.metadata.start_ms,
+        end_ms: c.metadata.end_ms,
+        text: c.text,
+        text_sha: c.text_sha,
+        vector: vectors[i],
+        metadata: {
+            source_page: c.metadata.source_page,
+            highlight_ids: c.metadata.highlight_ids,
+            code_ids: c.metadata.code_ids,
+            actor_type: c.metadata.actor_type,
+            chunk_type: c.metadata.chunk_type,
+        },
+    })));
+    const inserted = await writer.upsertByTextSha(records);
+    return {
+        embedded: allChunks.length,
+        inserted,
+        transcripts_scanned: transcripts.length,
+    };
+}
+/**
+ * Worker-level batch cap. Defense in depth against very large corpora that
+ * would otherwise produce a single multi-megabyte JSON request. The
+ * LLMAdapter splits within this cap to provider-appropriate sizes.
+ *
+ * Tuning notes: 500 chunks × ~1KB text each = ~500KB request body, well
+ * under Ollama's default HTTP body limits. A typical interview corpus
+ * (~600 utterances per session × 3 chunks/utt × 30 sessions = 54k chunks)
+ * batches to ~108 round-trips.
+ */
+export const EMBED_BATCH_CAP = 500;
+async function embedInBatches(embed, texts, cap) {
+    if (texts.length <= cap)
+        return embed(texts);
+    const out = [];
+    for (let i = 0; i < texts.length; i += cap) {
+        const slice = texts.slice(i, i + cap);
+        const partial = await embed(slice);
+        if (partial.length !== slice.length) {
+            throw new CompostError('PROVIDER_ERROR', `embeddings provider returned ${partial.length} vectors for ${slice.length} chunks ` +
+                `(batch ${i / cap + 1}/${Math.ceil(texts.length / cap)})`);
+        }
+        out.push(...partial);
+    }
+    return out;
+}
+const DEFAULT_DIM_FOR_MODEL = {
+    'bge-m3': 1024,
+    'bge-m3:q4_k_m': 1024,
+    'mxbai-embed-large': 1024,
+    'nomic-embed-text': 768,
+    'all-minilm': 384,
+};
+function chunkTypeToArtifactKind(type) {
+    switch (type) {
+        case 'highlight':
+            return 'highlight';
+        case 'term':
+            return 'term';
+        case 'page':
+            return 'legacy_chunk';
+        default:
+            return 'utterance';
+    }
+}
+function findTranscripts(seedPath) {
+    const sessionsDir = join(seedPath, 'sessions');
+    if (!existsSync(sessionsDir))
+        return [];
+    const out = [];
+    for (const entry of readdirSync(sessionsDir)) {
+        if (entry === '_inbox' || entry.startsWith('.'))
+            continue;
+        const tp = join(sessionsDir, entry, 'transcript.json');
+        if (existsSync(tp))
+            out.push({ sessionId: entry, path: tp });
+    }
+    return out;
+}
+//# sourceMappingURL=embed_worker.js.map

package/dist/loops/embed_worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embed_worker.js","sourceRoot":"","sources":["../../src/loops/embed_worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAGhC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AAE5F,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAmC9C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,OAAwB,EAAE;IAE1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAA;IACpD,MAAM,WAAW,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;IAC7C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,CAAA;IAC7D,CAAC;IAED,2CAA2C;IAC3C,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;IACtB,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAA;IACtC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;QACnC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;QACtC,KAAK,GAAG,KAAK,EAAE,KAAe,EAAE,EAAE;YAChC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;YACrD,OAAO,IAAI,CAAC,OAAO,CAAA;QACrB,CAAC,CAAA;QACD,wEAAwE;QACxE,0EAA0E;QAC1E,6EAA6E;QAC7E,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAA;QACxC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YACnC,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,IAAI,CAAA;QAClD,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,MAAM,SAAS,GAAyC,EAAE,CAAA;IAC1D,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAsB,CAAA;QAChF,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;QAC9D,KAAK,MAAM,CAAC,IAAI,MAAM;YAAE,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1E,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,mBAAmB,EAAE,WAAW,CAAC,MAAM,EAAE,CAAA;IAC9E,CAAC;IAED,6EAA6E;IAC7E,iEAAiE;IACjE,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,SAAS,CAAC,CAAC,CAAA;IAEvF,sEAAsE;IACtE,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,yEAAyE;IACzE,oEAAoE;IACpE,6BAA6B;IAC7B,MAAM,OAAO,GAAG,MAAM,cAAc,CAClC,KAAK,EACL,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAC5B,eAAe,CAChB,CAAA;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxC,MAAM,IAAI,YAAY,CACpB,gBAAgB,EAChB,gCAAgC,OAAO,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,SAAS,CACxF,CAAA;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,OAAO,GAAG,kBAAkB,CAChC,SAAS,CAAC,GAAG,CACX,CAAC,CAAC,EAAE,CAAC,EAAqB,EAAE,CAAC,CAAC;QAC5B,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QACpD,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,CAAC,CAAC,SAAS;QACpB,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;QACjC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ;QAC7B,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAa;QAC9B,QAAQ,EAAE;YACR,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW;YACnC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa;YACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ;YAC7B,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;YACjC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;SAClC;KACF,CAAC,CACH,CACF,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;IACtD,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,MAAM;QAC1B,QAAQ;QACR,mBAAmB,EAAE,WAAW,CAAC,MAAM;KACxC,CAAA;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAA;AAElC,KAAK,UAAU,cAAc,CAC3B,KAA+C,EAC/C,KAAe,EACf,GAAW;IAEX,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,CAAA;IAC5C,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QACrC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAA;QAClC,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,IAAI,YAAY,CACpB,gBAAgB,EAChB,gCAAgC,OAAO,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,UAAU;gBAClF,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,CAC5D,CAAA;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAA;IACtB,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,qBAAqB,GAA2B;IACpD,QAAQ,EAAE,IAAI;IACd,eAAe,EAAE,IAAI;IACrB,mBAAmB,EAAE,IAAI;IACzB,kBAAkB,EAAE,GAAG;IACvB,YAAY,EAAE,GAAG;CAClB,CAAA;AAED,SAAS,uBAAuB,CAC9B,IAA4D;IAE5D,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,WAAW;YACd,OAAO,WAAW,CAAA;QACpB,KAAK,MAAM;YACT,OAAO,MAAM,CAAA;QACf,KAAK,MAAM;YACT,OAAO,cAAc,CAAA;QACvB;YACE,OAAO,WAAW,CAAA;IACtB,CAAC;AACH,CAAC;AAOD,SAAS,eAAe,CAAC,QAAgB;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC9C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,EAAE,CAAA;IACvC,MAAM,GAAG,GAAuB,EAAE,CAAA;IAClC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7C,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAQ;QACzD,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,iBAAiB,CAAC,CAAA;QACtD,IAAI,UAAU,CAAC,EAAE,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;IAC9D,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC"}

package/dist/loops/eval_grader.d.ts

@@ -0,0 +1,19 @@
+import { type Judge, type Suggestion } from 'compost-evals';
+export interface EvalGraderDeps {
+    judge: Judge;
+    judgeModel: string;
+    maxPerRun?: number;
+    now?: () => Date;
+}
+export interface EvalGraderResult {
+    graded: number;
+    skipped: number;
+}
+/**
+ * Grade a batch of AI-authored suggestions for a seed and persist verdicts to
+ * .compost/evals.sqlite. Idempotent on suggestion_id (already-graded skipped)
+ * and throttled to maxPerRun calls. The caller supplies the suggestions
+ * (read from the provenance event log) and an LLM judge.
+ */
+export declare function runEvalGraderOnce(seedPath: string, suggestions: Suggestion[], deps: EvalGraderDeps): Promise<EvalGraderResult>;
+//# sourceMappingURL=eval_grader.d.ts.map

package/dist/loops/eval_grader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval_grader.d.ts","sourceRoot":"","sources":["../../src/loops/eval_grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,KAAK,EACV,KAAK,UAAU,EAChB,MAAM,eAAe,CAAA;AAEtB,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,KAAK,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,UAAU,EAAE,EACzB,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,gBAAgB,CAAC,CAY3B"}

package/dist/loops/eval_grader.js

@@ -0,0 +1,22 @@
+import { EvalStore, evalsDbPath, gradeSuggestions, } from 'compost-evals';
+/**
+ * Grade a batch of AI-authored suggestions for a seed and persist verdicts to
+ * .compost/evals.sqlite. Idempotent on suggestion_id (already-graded skipped)
+ * and throttled to maxPerRun calls. The caller supplies the suggestions
+ * (read from the provenance event log) and an LLM judge.
+ */
+export async function runEvalGraderOnce(seedPath, suggestions, deps) {
+    const store = new EvalStore(evalsDbPath(seedPath));
+    try {
+        const { graded, skipped } = await gradeSuggestions(store, suggestions, deps.judge, {
+            judgeModel: deps.judgeModel,
+            ...(deps.maxPerRun !== undefined ? { maxPerRun: deps.maxPerRun } : {}),
+            ...(deps.now !== undefined ? { now: deps.now } : {}),
+        });
+        return { graded: graded.length, skipped };
+    }
+    finally {
+        store.close();
+    }
+}
+//# sourceMappingURL=eval_grader.js.map

package/dist/loops/eval_grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval_grader.js","sourceRoot":"","sources":["../../src/loops/eval_grader.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,WAAW,EACX,gBAAgB,GAGjB,MAAM,eAAe,CAAA;AActB;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,WAAyB,EACzB,IAAoB;IAEpB,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAA;IAClD,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE;YACjF,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC,CAAA;QACF,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAA;IAC3C,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAA;IACf,CAAC;AACH,CAAC"}

package/dist/loops/ingest_watcher.d.ts

@@ -0,0 +1,17 @@
+export interface WatcherProcessResult {
+    moved: Array<{
+        from: string;
+        to: string;
+        session_id: string;
+        job_id: number;
+    }>;
+    unsupported: string[];
+}
+/**
+ * Process everything currently in a seed's sessions/_inbox/: assign a session
+ * id, move the file to sessions/<sid>/source.<ext> atomically, enqueue a job,
+ * and emit an agent `create` event. Pure of timers — call it on boot (replay)
+ * and from the debounced live watcher. Idempotent: an empty inbox is a no-op.
+ */
+export declare function processInbox(seedPath: string): WatcherProcessResult;
+//# sourceMappingURL=ingest_watcher.d.ts.map

package/dist/loops/ingest_watcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest_watcher.d.ts","sourceRoot":"","sources":["../../src/loops/ingest_watcher.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC9E,WAAW,EAAE,MAAM,EAAE,CAAA;CACtB;AAUD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,oBAAoB,CAyCnE"}

package/dist/loops/ingest_watcher.js

@@ -0,0 +1,66 @@
+import { existsSync, mkdirSync, readdirSync, renameSync, statSync } from 'node:fs';
+import { basename, extname, join } from 'node:path';
+import { classify } from '../lib/dispatch.js';
+import { emitAgentCreate, openSeedEvents } from '../lib/events.js';
+import { JobQueue, stateDbPath } from '../lib/queue.js';
+const AGENT_NAME = 'ingest-watcher';
+const AGENT_VERSION = '0.1.0';
+function nextSessionId(sessionsDir) {
+    const existing = existsSync(sessionsDir)
+        ? readdirSync(sessionsDir).filter((e) => /^S\d+$/.test(e))
+        : [];
+    const max = existing.reduce((m, e) => Math.max(m, Number(e.slice(1))), 0);
+    return `S${String(max + 1).padStart(3, '0')}`;
+}
+/**
+ * Process everything currently in a seed's sessions/_inbox/: assign a session
+ * id, move the file to sessions/<sid>/source.<ext> atomically, enqueue a job,
+ * and emit an agent `create` event. Pure of timers — call it on boot (replay)
+ * and from the debounced live watcher. Idempotent: an empty inbox is a no-op.
+ */
+export function processInbox(seedPath) {
+    const sessionsDir = join(seedPath, 'sessions');
+    const inbox = join(sessionsDir, '_inbox');
+    const result = { moved: [], unsupported: [] };
+    if (!existsSync(inbox))
+        return result;
+    const queue = new JobQueue(stateDbPath(seedPath));
+    const events = openSeedEvents(seedPath);
+    try {
+        for (const entry of readdirSync(inbox)) {
+            if (entry.startsWith('.'))
+                continue;
+            const from = join(inbox, entry);
+            if (!statSync(from).isFile())
+                continue;
+            const d = classify(from);
+            if (d === null) {
+                result.unsupported.push(from);
+                continue;
+            }
+            const sid = nextSessionId(sessionsDir);
+            const sessionDir = join(sessionsDir, sid);
+            mkdirSync(sessionDir, { recursive: true });
+            const to = join(sessionDir, `source${extname(entry).toLowerCase()}`);
+            renameSync(from, to); // atomic within the same filesystem
+            const { id } = queue.enqueue(d.kind, to, {
+                category: d.category,
+                session_id: sid,
+                original_name: basename(entry),
+            });
+            emitAgentCreate(events, {
+                artifactKind: 'session',
+                initialState: { session_id: sid, source: to, kind: d.kind },
+                agentName: AGENT_NAME,
+                agentVersion: AGENT_VERSION,
+            });
+            result.moved.push({ from, to, session_id: sid, job_id: id });
+        }
+        return result;
+    }
+    finally {
+        queue.close();
+        events.close();
+    }
+}
+//# sourceMappingURL=ingest_watcher.js.map

package/dist/loops/ingest_watcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest_watcher.js","sourceRoot":"","sources":["../../src/loops/ingest_watcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAEvD,MAAM,UAAU,GAAG,gBAAgB,CAAA;AACnC,MAAM,aAAa,GAAG,OAAO,CAAA;AAO7B,SAAS,aAAa,CAAC,WAAmB;IACxC,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC;QACtC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IACzE,OAAO,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAA;AAC/C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACzC,MAAM,MAAM,GAAyB,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IACnE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IAErC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAA;IACjD,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAA;IACvC,IAAI,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACvC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAQ;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;YAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;gBAAE,SAAQ;YACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;YACxB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC7B,SAAQ;YACV,CAAC;YACD,MAAM,GAAG,GAAG,aAAa,CAAC,WAAW,CAAC,CAAA;YACtC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;YACzC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;YACpE,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA,CAAC,oCAAoC;YACzD,MAAM,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE;gBACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,UAAU,EAAE,GAAG;gBACf,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC;aAC/B,CAAC,CAAA;YACF,eAAe,CAAC,MAAM,EAAE;gBACtB,YAAY,EAAE,SAAS;gBACvB,YAAY,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC3D,SAAS,EAAE,UAAU;gBACrB,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAA;YACF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAA;QAC9D,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,MAAM,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;AACH,CAAC"}

package/dist/loops/legacy_worker.d.ts

@@ -0,0 +1,26 @@
+import { LegacyIngestClient } from '../legacy_client.js';
+export interface LegacyWorkerResult {
+    processed: number;
+    results: Array<{
+        job_id: number;
+        source_path: string;
+        status: string;
+        normalized_path?: string;
+        utterance_count?: number;
+        warnings?: string[];
+    }>;
+}
+export interface LegacyWorkerDeps {
+    client?: LegacyIngestClient;
+}
+/**
+ * Drain all queued `legacy-ingest` jobs (PDF/DOCX/PPTX/CSV/MD/TXT/XLSX).
+ * Each job is POSTed to the transcriber's /legacy-ingest route, which writes
+ * a normalized transcript-shaped JSON under `<seed>/legacy/<basename>.json`.
+ *
+ * Transient failures (service down) requeue with backoff; permanent failures
+ * (invalid input, missing dep) burn the attempt counter so the job moves to
+ * failed status after MAX_ATTEMPTS.
+ */
+export declare function runLegacyWorkerOnce(seedPath: string, deps?: LegacyWorkerDeps): Promise<LegacyWorkerResult>;
+//# sourceMappingURL=legacy_worker.d.ts.map

package/dist/loops/legacy_worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"legacy_worker.d.ts","sourceRoot":"","sources":["../../src/loops/legacy_worker.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,kBAAkB,EAGnB,MAAM,qBAAqB,CAAA;AA2C5B,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,KAAK,CAAC;QACb,MAAM,EAAE,MAAM,CAAA;QACd,WAAW,EAAE,MAAM,CAAA;QACnB,MAAM,EAAE,MAAM,CAAA;QACd,eAAe,CAAC,EAAE,MAAM,CAAA;QACxB,eAAe,CAAC,EAAE,MAAM,CAAA;QACxB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;KACpB,CAAC,CAAA;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,kBAAkB,CAAA;CAC5B;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,gBAAqB,GAC1B,OAAO,CAAC,kBAAkB,CAAC,CAyD7B"}

package/dist/loops/legacy_worker.js

@@ -0,0 +1,102 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { LegacyIngestClient, LegacyServiceError, } from '../legacy_client.js';
+import { emitAgentCreate, openSeedEvents } from '../lib/events.js';
+import { JobQueue, stateDbPath } from '../lib/queue.js';
+const AGENT_NAME = 'legacy-ingest-worker';
+const AGENT_VERSION = '0.1.0';
+const MAX_ATTEMPTS = 3;
+function readSidecar(sourcePath) {
+    const sidecarPath = `${sourcePath}.compost.json`;
+    if (!existsSync(sidecarPath))
+        return null;
+    try {
+        const parsed = JSON.parse(readFileSync(sidecarPath, 'utf8'));
+        if (typeof parsed !== 'object' || parsed === null)
+            return null;
+        const out = {};
+        const r = parsed;
+        if (typeof r.text_col === 'string')
+            out.text_col = r.text_col;
+        if (typeof r.speaker_col === 'string')
+            out.speaker_col = r.speaker_col;
+        if (typeof r.sheet === 'string')
+            out.sheet = r.sheet;
+        return out;
+    }
+    catch {
+        // Malformed sidecar — silently fall through to server-side auto-detect.
+        // The researcher's intent is unclear; we don't want to block the ingest.
+        return null;
+    }
+}
+/**
+ * Drain all queued `legacy-ingest` jobs (PDF/DOCX/PPTX/CSV/MD/TXT/XLSX).
+ * Each job is POSTed to the transcriber's /legacy-ingest route, which writes
+ * a normalized transcript-shaped JSON under `<seed>/legacy/<basename>.json`.
+ *
+ * Transient failures (service down) requeue with backoff; permanent failures
+ * (invalid input, missing dep) burn the attempt counter so the job moves to
+ * failed status after MAX_ATTEMPTS.
+ */
+export async function runLegacyWorkerOnce(seedPath, deps = {}) {
+    const client = deps.client ?? new LegacyIngestClient();
+    const queue = new JobQueue(stateDbPath(seedPath));
+    const events = openSeedEvents(seedPath);
+    const out = { processed: 0, results: [] };
+    try {
+        while (true) {
+            const job = queue.claim('legacy-ingest');
+            if (job === null)
+                break;
+            out.processed += 1;
+            const sourcePath = job.source_path;
+            try {
+                const sidecar = readSidecar(sourcePath);
+                const ingestReq = {
+                    seed_path: seedPath,
+                    source_path: sourcePath,
+                    ...(sidecar?.text_col !== undefined ? { text_col: sidecar.text_col } : {}),
+                    ...(sidecar?.speaker_col !== undefined ? { speaker_col: sidecar.speaker_col } : {}),
+                    ...(sidecar?.sheet !== undefined ? { sheet: sidecar.sheet } : {}),
+                };
+                const resp = await client.ingest(ingestReq);
+                queue.complete(job.id);
+                emitAgentCreate(events, {
+                    artifactKind: 'legacy_chunk',
+                    initialState: {
+                        source_path: resp.source_path,
+                        normalized_path: resp.normalized_path,
+                        utterance_count: resp.utterance_count,
+                        status: resp.status,
+                        text_col_resolved: resp.text_col_resolved ?? null,
+                        sidecar_applied: sidecar !== null,
+                    },
+                    agentName: AGENT_NAME,
+                    agentVersion: AGENT_VERSION,
+                });
+                out.results.push({
+                    job_id: job.id,
+                    source_path: sourcePath,
+                    status: resp.status,
+                    normalized_path: resp.normalized_path,
+                    utterance_count: resp.utterance_count,
+                    ...(resp.warnings && resp.warnings.length > 0 ? { warnings: resp.warnings } : {}),
+                });
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                queue.fail(job.id, msg, MAX_ATTEMPTS);
+                out.results.push({ job_id: job.id, source_path: sourcePath, status: 'error' });
+                // On service-down, stop the drain — nothing else will succeed now.
+                if (err instanceof LegacyServiceError && err.kind === 'down')
+                    break;
+            }
+        }
+        return out;
+    }
+    finally {
+        queue.close();
+        events.close();
+    }
+}
+//# sourceMappingURL=legacy_worker.js.map

package/dist/loops/legacy_worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"legacy_worker.js","sourceRoot":"","sources":["../../src/loops/legacy_worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAElD,OAAO,EACL,kBAAkB,EAElB,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAEvD,MAAM,UAAU,GAAG,sBAAsB,CAAA;AACzC,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B,MAAM,YAAY,GAAG,CAAC,CAAA;AAkBtB,SAAS,WAAW,CAAC,UAAkB;IACrC,MAAM,WAAW,GAAG,GAAG,UAAU,eAAe,CAAA;IAChD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAA;IACzC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAY,CAAA;QACvE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QAC9D,MAAM,GAAG,GAAmB,EAAE,CAAA;QAC9B,MAAM,CAAC,GAAG,MAAiC,CAAA;QAC3C,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAA;QAC7D,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;YAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAA;QACtE,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;YAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAA;QACpD,OAAO,GAAG,CAAA;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;QACxE,yEAAyE;QACzE,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAkBD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAgB,EAChB,OAAyB,EAAE;IAE3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAA;IACtD,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAA;IACjD,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,GAAG,GAAuB,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAA;IAE7D,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;YACxC,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAK;YACvB,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YAClB,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAA;YAClC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAA;gBACvC,MAAM,SAAS,GAAwB;oBACrC,SAAS,EAAE,QAAQ;oBACnB,WAAW,EAAE,UAAU;oBACvB,GAAG,CAAC,OAAO,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1E,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnF,GAAG,CAAC,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAClE,CAAA;gBACD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBAC3C,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBACtB,eAAe,CAAC,MAAM,EAAE;oBACtB,YAAY,EAAE,cAAc;oBAC5B,YAAY,EAAE;wBACZ,WAAW,EAAE,IAAI,CAAC,WAAW;wBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;wBACrC,eAAe,EAAE,IAAI,CAAC,eAAe;wBACrC,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;wBACjD,eAAe,EAAE,OAAO,KAAK,IAAI;qBAClC;oBACD,SAAS,EAAE,UAAU;oBACrB,YAAY,EAAE,aAAa;iBAC5B,CAAC,CAAA;gBACF,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;oBACf,MAAM,EAAE,GAAG,CAAC,EAAE;oBACd,WAAW,EAAE,UAAU;oBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAClF,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAC5D,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,YAAY,CAAC,CAAA;gBACrC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;gBAC9E,mEAAmE;gBACnE,IAAI,GAAG,YAAY,kBAAkB,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM;oBAAE,MAAK;YACrE,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,MAAM,CAAC,KAAK,EAAE,CAAA;IAChB,CAAC;AACH,CAAC"}

package/dist/loops/supervisor.d.ts

@@ -0,0 +1,71 @@
+import { type EmbedWorkerDeps } from './embed_worker.js';
+import { type LegacyWorkerDeps } from './legacy_worker.js';
+import { type WorkerDeps } from './transcribe_worker.js';
+export interface SupervisorResult {
+    inbox: {
+        moved: number;
+        unsupported: number;
+    };
+    transcribe: {
+        processed: number;
+        failed: number;
+    };
+    legacy: {
+        processed: number;
+        failed: number;
+    };
+    embed: {
+        embedded: number;
+        inserted: number;
+        transcripts_scanned: number;
+    };
+    /** Human-readable failure summaries; empty when the pass was clean. The watch
+     * command turns a non-empty list into a non-ok status + non-zero exit (#164). */
+    failures: string[];
+}
+/** A drained job whose status marks it as failed (vs ok / needs_speaker_labels). */
+export declare function isFailedResult(r: {
+    status: string;
+}): boolean;
+/** Count DISTINCT failed jobs. A job retried up to MAX_ATTEMPTS within one pass
+ * produces several failed result rows, but it's one failed job — report jobs,
+ * not attempts. */
+export declare function countFailedJobs(results: Array<{
+    job_id: number;
+    status: string;
+}>): number;
+export interface SupervisorDeps extends WorkerDeps {
+    legacy?: LegacyWorkerDeps;
+    embed?: EmbedWorkerDeps;
+    /** Skip the legacy pass (handy for tests that don't want to touch the legacy service). */
+    skipLegacy?: boolean;
+    /** Disable the embed pass (handy for tests that don't want to touch LanceDB). */
+    skipEmbed?: boolean;
+}
+/**
+ * One cooperative pass:
+ *  1. Drain the inbox into sessions/SXXX/ shells
+ *  2. Drain transcribe jobs (audio/video → transcript.json)
+ *  3. Drain legacy-ingest jobs (PDF/DOCX/PPTX/CSV/MD/TXT/XLSX → legacy/<basename>.json)
+ *  4. Embed any newly-produced transcripts into LanceDB
+ *
+ * Order matters: embed runs LAST so it sees transcripts the transcribe and
+ * legacy passes just wrote. Transcribe and legacy are independent (different
+ * job kinds); we serialize them to keep the single Python service from
+ * saturating.
+ */
+export declare function runSupervisorOnce(seedPath: string, deps?: SupervisorDeps): Promise<SupervisorResult>;
+/** Exponential backoff schedule (ms) for crash recovery, capped at maxAttempts. */
+export declare function backoffSchedule(maxAttempts?: number, baseMs?: number): number[];
+export interface RunLiveOptions extends WorkerDeps {
+    intervalMs?: number;
+    signal?: AbortSignal;
+    onError?: (loop: string, err: unknown) => void;
+}
+/**
+ * Live supervisor loop. Runs a pass every intervalMs until aborted. A pass that
+ * throws is retried with backoff up to 3 times before the loop re-arms on the
+ * next tick. (Headless-friendly; `compost watch` wires SIGINT to the signal.)
+ */
+export declare function runLive(seedPath: string, opts?: RunLiveOptions): Promise<void>;
+//# sourceMappingURL=supervisor.d.ts.map

package/dist/loops/supervisor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"supervisor.d.ts","sourceRoot":"","sources":["../../src/loops/supervisor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,mBAAmB,CAAA;AAE5E,OAAO,EAAE,KAAK,gBAAgB,EAAuB,MAAM,oBAAoB,CAAA;AAC/E,OAAO,EAA2B,KAAK,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAEjF,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAA;IAC7C,UAAU,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IACjD,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IAC7C,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1E;qFACiF;IACjF,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,oFAAoF;AACpF,wBAAgB,cAAc,CAAC,CAAC,EAAE;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAE7D;AAED;;mBAEmB;AACnB,wBAAgB,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAE1F;AAED,MAAM,WAAW,cAAe,SAAQ,UAAU;IAChD,MAAM,CAAC,EAAE,gBAAgB,CAAA;IACzB,KAAK,CAAC,EAAE,eAAe,CAAA;IACvB,0FAA0F;IAC1F,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,iFAAiF;IACjF,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAgD3B;AAQD,mFAAmF;AACnF,wBAAgB,eAAe,CAAC,WAAW,SAAI,EAAE,MAAM,SAAM,GAAG,MAAM,EAAE,CAEvE;AAED,MAAM,WAAW,cAAe,SAAQ,UAAU;IAChD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI,CAAA;CAC/C;AAED;;;;GAIG;AACH,wBAAsB,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAE,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkBxF"}

package/dist/loops/supervisor.js

@@ -0,0 +1,122 @@
+import { getLogPath, Logger } from '../logging.js';
+import { runEmbedWorkerOnce } from './embed_worker.js';
+import { processInbox } from './ingest_watcher.js';
+import { runLegacyWorkerOnce } from './legacy_worker.js';
+import { runTranscribeWorkerOnce } from './transcribe_worker.js';
+/** A drained job whose status marks it as failed (vs ok / needs_speaker_labels). */
+export function isFailedResult(r) {
+    return r.status === 'error' || r.status === 'failed_transcription';
+}
+/** Count DISTINCT failed jobs. A job retried up to MAX_ATTEMPTS within one pass
+ * produces several failed result rows, but it's one failed job — report jobs,
+ * not attempts. */
+export function countFailedJobs(results) {
+    return new Set(results.filter(isFailedResult).map((r) => r.job_id)).size;
+}
+/**
+ * One cooperative pass:
+ *  1. Drain the inbox into sessions/SXXX/ shells
+ *  2. Drain transcribe jobs (audio/video → transcript.json)
+ *  3. Drain legacy-ingest jobs (PDF/DOCX/PPTX/CSV/MD/TXT/XLSX → legacy/<basename>.json)
+ *  4. Embed any newly-produced transcripts into LanceDB
+ *
+ * Order matters: embed runs LAST so it sees transcripts the transcribe and
+ * legacy passes just wrote. Transcribe and legacy are independent (different
+ * job kinds); we serialize them to keep the single Python service from
+ * saturating.
+ */
+export async function runSupervisorOnce(seedPath, deps = {}) {
+    const logger = loopLogger(seedPath, 'supervisor');
+    const failures = [];
+    const inbox = processInbox(seedPath);
+    await logger.info('inbox drained', {
+        moved: inbox.moved.length,
+        unsupported: inbox.unsupported.length,
+    });
+    const worker = await runTranscribeWorkerOnce(seedPath, deps);
+    const transcribeFailed = countFailedJobs(worker.results);
+    if (transcribeFailed > 0)
+        failures.push(`transcribe: ${transcribeFailed} job(s) failed`);
+    await logger.info('transcribe drained', { processed: worker.processed, failed: transcribeFailed });
+    let legacy = { processed: 0, failed: 0 };
+    if (deps.skipLegacy !== true) {
+        try {
+            const result = await runLegacyWorkerOnce(seedPath, deps.legacy ?? {});
+            const failed = countFailedJobs(result.results);
+            legacy = { processed: result.processed, failed };
+            if (failed > 0)
+                failures.push(`legacy: ${failed} job(s) failed`);
+            await logger.info('legacy drained', legacy);
+        }
+        catch (err) {
+            // The whole legacy pass threw (e.g. service down) — surface it, don't block.
+            failures.push(`legacy: ${String(err)}`);
+            await logger.error('legacy failed', { error: String(err) });
+        }
+    }
+    let embed = { embedded: 0, inserted: 0, transcripts_scanned: 0 };
+    if (deps.skipEmbed !== true) {
+        try {
+            embed = await runEmbedWorkerOnce(seedPath, deps.embed ?? {});
+            await logger.info('embed drained', embed);
+        }
+        catch (err) {
+            // Embed failures must not block ingest/transcribe/legacy progress — surface + continue.
+            // Common cause: Ollama not running. Surfaced clearly by `compost-setup` (v0.1-07).
+            failures.push(`embed: ${String(err)}`);
+            await logger.error('embed failed', { error: String(err) });
+        }
+    }
+    return {
+        inbox: { moved: inbox.moved.length, unsupported: inbox.unsupported.length },
+        transcribe: { processed: worker.processed, failed: transcribeFailed },
+        legacy,
+        embed,
+        failures,
+    };
+}
+function loopLogger(seedPath, loop) {
+    const base = getLogPath(seedPath); // <seed>/.compost/logs/<date>.jsonl
+    const perLoop = base.replace(/[^/]+$/, `${loop}.jsonl`);
+    return new Logger(perLoop);
+}
+/** Exponential backoff schedule (ms) for crash recovery, capped at maxAttempts. */
+export function backoffSchedule(maxAttempts = 3, baseMs = 500) {
+    return Array.from({ length: maxAttempts }, (_, i) => baseMs * 2 ** i);
+}
+/**
+ * Live supervisor loop. Runs a pass every intervalMs until aborted. A pass that
+ * throws is retried with backoff up to 3 times before the loop re-arms on the
+ * next tick. (Headless-friendly; `compost watch` wires SIGINT to the signal.)
+ */
+export async function runLive(seedPath, opts = {}) {
+    const interval = opts.intervalMs ?? 2000;
+    const backoff = backoffSchedule();
+    while (opts.signal?.aborted !== true) {
+        let attempt = 0;
+        while (true) {
+            try {
+                await runSupervisorOnce(seedPath, opts);
+                break;
+            }
+            catch (err) {
+                opts.onError?.('supervisor', err);
+                if (attempt >= backoff.length)
+                    break;
+                await sleep(backoff[attempt], opts.signal);
+                attempt += 1;
+            }
+        }
+        await sleep(interval, opts.signal);
+    }
+}
+function sleep(ms, signal) {
+    return new Promise((resolve) => {
+        const t = setTimeout(resolve, ms);
+        signal?.addEventListener('abort', () => {
+            clearTimeout(t);
+            resolve();
+        });
+    });
+}
+//# sourceMappingURL=supervisor.js.map