agentwrangler 0.1.0

package/dist/ingest/detector-hook.js

@@ -0,0 +1,52 @@
+/**
+ * src/ingest/detector-hook.ts — post-ingest hook seam.
+ *
+ * Mirrors the setRuntimeResetHook pattern (query/settings-store.ts): a
+ * module-level callback the daemon wires at boot so the Ingestor can trigger a
+ * detector pass WITHOUT `ingest/**` statically importing `detector/**` or the
+ * query layer (no ingest→query import cycle). Zero-op when unset (tests that do
+ * not wire it, e.g. the WP1 ingestion suite, are unaffected).
+ *
+ * Determinism (NFR-107 / Review F4): the hook is invoked with the ingestor's
+ * injected clock, never new Date() — see Ingestor.runPostIngest().
+ */
+let _hook = null;
+/** Wire the post-ingest hook (daemon boot / tests). */
+export function setPostIngestHook(fn) {
+    _hook = fn;
+}
+/** Clear the post-ingest hook (tests, between cases). */
+export function clearPostIngestHook() {
+    _hook = null;
+}
+/** Invoke the hook if wired; a thrown hook never breaks the ingest loop. */
+export function runPostIngestHook(db, now) {
+    if (_hook === null)
+        return;
+    try {
+        _hook(db, now);
+    }
+    catch (e) {
+        console.warn(`post-ingest hook failed: ${e instanceof Error ? e.message : String(e)}`);
+    }
+}
+let _probeHook = null;
+/** Wire the post-probe hook (daemon boot / tests). */
+export function setPostProbeHook(fn) {
+    _probeHook = fn;
+}
+/** Clear the post-probe hook (tests, between cases). */
+export function clearPostProbeHook() {
+    _probeHook = null;
+}
+/** Invoke the post-probe hook if wired; a thrown hook never breaks the probe pass. */
+export function runPostProbeHook(db, now) {
+    if (_probeHook === null)
+        return;
+    try {
+        _probeHook(db, now);
+    }
+    catch (e) {
+        console.warn(`post-probe hook failed: ${e instanceof Error ? e.message : String(e)}`);
+    }
+}

package/dist/ingest/discovery.js

@@ -0,0 +1,207 @@
+/**
+ * src/ingest/discovery.ts — transcript-file discovery + workspace registration.
+ *
+ * Each subdirectory of a scan root is a candidate workspace (project_slug);
+ * every *.jsonl beneath it (recursively, including nested subagent transcripts)
+ * is a session file. New slugs auto-register as unmapped
+ * workspaces so spend is visible immediately (repo mapping is a later WP4 step).
+ *
+ * Discovery is poll-based (default 30s cadence), distinct from the faster tail
+ * cadence. Filesystem watch is an optional optimisation not required for
+ * correctness — offsets + dedupe make a full re-scan cheap and idempotent.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+/** Recursively collect every *.jsonl under `dir`, tagging each with `slug`. */
+function collectJsonl(dir, slug, out) {
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    // Emit files at this level BEFORE descending, so a session's own transcript
+    // (<slug>/<uuid>.jsonl) is discovered before its nested subagent files
+    // (<slug>/<uuid>/subagents/agent-*.jsonl). Both carry the same parent sessionId
+    // and ensureSession records file_path first-seen — files-before-dirs keeps the
+    // session's file_path pointing at the top-level transcript, not a subagent file.
+    for (const e of entries) {
+        if (e.isFile() && e.name.endsWith(".jsonl")) {
+            out.push({ filePath: path.join(dir, e.name), projectSlug: slug });
+        }
+    }
+    for (const e of entries) {
+        if (e.isDirectory()) {
+            collectJsonl(path.join(dir, e.name), slug, out);
+        }
+    }
+}
+/** Create an empty cache; the first refresh performs one full walk. */
+export function createDiscoveryCache() {
+    return { files: [], roots: new Map(), initialized: false };
+}
+function directoryMtimeMs(dir) {
+    try {
+        const stat = fs.statSync(dir);
+        return stat.isDirectory() ? stat.mtimeMs : null;
+    }
+    catch {
+        return null;
+    }
+}
+function readSlugDirectories(root) {
+    let entries;
+    try {
+        entries = fs.readdirSync(root, { withFileTypes: true });
+    }
+    catch {
+        return null;
+    }
+    const slugDirs = new Map();
+    for (const entry of entries) {
+        if (!entry.isDirectory())
+            continue;
+        const dir = path.join(root, entry.name);
+        const mtimeMs = directoryMtimeMs(dir);
+        if (mtimeMs === null)
+            continue;
+        slugDirs.set(dir, { projectSlug: entry.name, mtimeMs });
+    }
+    return slugDirs;
+}
+function isInside(dir, filePath) {
+    const relative = path.relative(dir, filePath);
+    return (relative !== "" &&
+        relative !== ".." &&
+        !relative.startsWith(`..${path.sep}`) &&
+        !path.isAbsolute(relative));
+}
+function filesForSlug(discovered, dir, projectSlug) {
+    return discovered.filter((file) => file.projectSlug === projectSlug && isInside(dir, file.filePath));
+}
+function walkSlugFiles(dir, projectSlug) {
+    const files = [];
+    collectJsonl(dir, projectSlug, files);
+    return files;
+}
+function replaceFullCache(roots, cache, discovered) {
+    const nextRoots = new Map();
+    for (const root of roots) {
+        const rootMtimeMs = directoryMtimeMs(root);
+        if (rootMtimeMs === null)
+            continue;
+        const slugDirs = readSlugDirectories(root);
+        if (slugDirs === null)
+            continue;
+        const cachedSlugDirs = new Map();
+        for (const [dir, slug] of slugDirs) {
+            cachedSlugDirs.set(dir, {
+                projectSlug: slug.projectSlug,
+                mtimeMs: slug.mtimeMs,
+                files: filesForSlug(discovered, dir, slug.projectSlug),
+            });
+        }
+        nextRoots.set(root, { mtimeMs: rootMtimeMs, slugDirs: cachedSlugDirs });
+    }
+    cache.files = discovered;
+    cache.roots = nextRoots;
+    cache.initialized = true;
+}
+/** Refresh only changed slug directories, or force a periodic full walk. */
+export function refreshDiscoveryCache(roots, cache, forceRefresh = false) {
+    if (forceRefresh || !cache.initialized) {
+        replaceFullCache(roots, cache, discoverFiles(roots));
+        return cache.files;
+    }
+    const activeRoots = new Set(roots);
+    for (const root of cache.roots.keys()) {
+        if (!activeRoots.has(root))
+            cache.roots.delete(root);
+    }
+    for (const root of roots) {
+        const rootMtimeMs = directoryMtimeMs(root);
+        if (rootMtimeMs === null) {
+            cache.roots.delete(root);
+            continue;
+        }
+        let cachedRoot = cache.roots.get(root);
+        const isNewRoot = cachedRoot === undefined;
+        if (cachedRoot === undefined) {
+            cachedRoot = { mtimeMs: rootMtimeMs, slugDirs: new Map() };
+            cache.roots.set(root, cachedRoot);
+        }
+        const rootChanged = isNewRoot || cachedRoot.mtimeMs !== rootMtimeMs;
+        if (rootChanged) {
+            const currentSlugDirs = readSlugDirectories(root);
+            if (currentSlugDirs !== null) {
+                const nextSlugDirs = new Map();
+                for (const [dir, slug] of currentSlugDirs) {
+                    const previous = cachedRoot.slugDirs.get(dir);
+                    nextSlugDirs.set(dir, previous !== undefined && previous.mtimeMs === slug.mtimeMs
+                        ? previous
+                        : {
+                            projectSlug: slug.projectSlug,
+                            mtimeMs: slug.mtimeMs,
+                            files: walkSlugFiles(dir, slug.projectSlug),
+                        });
+                }
+                cachedRoot.slugDirs = nextSlugDirs;
+            }
+            cachedRoot.mtimeMs = rootMtimeMs;
+            continue;
+        }
+        for (const [dir, cachedSlug] of cachedRoot.slugDirs) {
+            const mtimeMs = directoryMtimeMs(dir);
+            if (mtimeMs === null) {
+                cachedRoot.slugDirs.delete(dir);
+                continue;
+            }
+            if (mtimeMs === cachedSlug.mtimeMs)
+                continue;
+            const files = walkSlugFiles(dir, cachedSlug.projectSlug);
+            cachedSlug.mtimeMs = mtimeMs;
+            cachedSlug.files = files;
+        }
+    }
+    cache.files = [...cache.roots.values()].flatMap((root) => [...root.slugDirs.values()].flatMap((slug) => slug.files));
+    return cache.files;
+}
+/** Walk each root; return every *.jsonl file with its owning project slug. */
+export function discoverFiles(roots) {
+    const out = [];
+    for (const root of roots) {
+        if (!fs.existsSync(root))
+            continue;
+        let dirents;
+        try {
+            dirents = fs.readdirSync(root, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const d of dirents) {
+            if (!d.isDirectory())
+                continue;
+            // projectSlug is always the top-level slug dir; recurse so nested subagent
+            // transcripts (<slug>/<session>/subagents/agent-*.jsonl) are discovered and
+            // attributed to the same workspace as their parent session.
+            collectJsonl(path.join(root, d.name), d.name, out);
+        }
+    }
+    return out;
+}
+/**
+ * Register a workspace for `slug` if not already present, returning its
+ * workspace_id. The slug is used as the workspace_id (deterministic, keeps
+ * rebuild-equality stable). Idempotent via INSERT OR IGNORE.
+ */
+export function registerWorkspace(db, slug) {
+    db.prepare(`INSERT OR IGNORE INTO workspaces (workspace_id, project_slug, registered_at)
+     VALUES (?,?,?)`).run(slug, slug, new Date().toISOString());
+    return slug;
+}
+/** The session id used when a record carries none: the file's basename stem. */
+export function sessionStemFor(filePath) {
+    return path.basename(filePath, ".jsonl");
+}

package/dist/ingest/health.js

@@ -0,0 +1,43 @@
+/**
+ * src/ingest/health.ts — parser-health counters (NFR-106, first-class metric).
+ *
+ * Drift in any of these is the early-warning that a Claude Code update changed
+ * the transcript schema. Counters are in-memory on the running ingestor and
+ * surfaced in Settings (WP4). No content; counts only.
+ */
+import { newHealthCounters } from "./types.js";
+export class Health {
+    c = newHealthCounters();
+    fileSeen() {
+        this.c.filesSeen++;
+    }
+    fileParsed() {
+        this.c.filesParsed++;
+    }
+    quarantined() {
+        this.c.linesQuarantined++;
+    }
+    synthetic() {
+        this.c.syntheticExcluded++;
+    }
+    duplicateDrop() {
+        this.c.duplicateDrops++;
+    }
+    turnIngested(parserVersion) {
+        this.c.turnsIngested++;
+        this.c.parserVersionMix[parserVersion] = (this.c.parserVersionMix[parserVersion] ?? 0) + 1;
+    }
+    unknownFields(fields) {
+        for (const f of fields) {
+            this.c.unknownFieldKinds[f] = (this.c.unknownFieldKinds[f] ?? 0) + 1;
+        }
+    }
+    /** Return an immutable snapshot of the current counters. */
+    snapshot() {
+        return {
+            ...this.c,
+            unknownFieldKinds: { ...this.c.unknownFieldKinds },
+            parserVersionMix: { ...this.c.parserVersionMix },
+        };
+    }
+}

package/dist/ingest/index.js

@@ -0,0 +1,28 @@
+/**
+ * src/ingest/index.ts — WP1 ingestion pipeline public surface.
+ *
+ * The daemon orchestrator (src/daemon/index.ts) wires these at boot; this module
+ * exposes clean importable entrypoints and does NOT touch daemon internals.
+ *
+ *   runBackscan(db, roots)  — one full idempotent scan of all transcript files,
+ *                             then reconcile; returns parser-health counters.
+ *   startTail(db, roots)    — initial pass, then discovery (30s) + tail (2s)
+ *                             loops with periodic reconciliation; returns a
+ *                             handle whose stop() clears the timers.
+ */
+import { DEFAULT_INGESTOR_OPTIONS, Ingestor, } from "./ingestor.js";
+export { Ingestor, DEFAULT_INGESTOR_OPTIONS };
+export { PARSER_VERSION } from "./types.js";
+export { PricingSnapshotStore, seedListPrices, LIST_PRICES } from "./pricing.js";
+export { reconcileSessions, DEFAULT_RECONCILE_OPTIONS } from "./reconcile.js";
+export { discoverFiles, registerWorkspace } from "./discovery.js";
+/** Run a single full back-scan of `roots` into `db`. Returns health counters. */
+export function runBackscan(db, roots, options = {}) {
+    return new Ingestor(db, roots, options).runBackscan();
+}
+/** Start incremental tailing of `roots` into `db`. Returns a stop handle. */
+export function startTail(db, roots, options = {}) {
+    const ingestor = new Ingestor(db, roots, options);
+    const handle = ingestor.startTail();
+    return { handle, ingestor };
+}