@faithfulalabi/agent-lens 0.1.0

package/dist/src/cli/commands/warm.js

@@ -0,0 +1,96 @@
+import { resolveArchiveRoot, resolveDataDir, resolveTranscriptRoot } from '../../archive/index.js';
+import { createArchiveReader } from '../../archive/read.js';
+import { createProjectionEnv } from '../../corpus/env.js';
+import { createCorpusSweep, createSpillIndexEnv } from '../../corpus/watch.js';
+import { DbLockedError, openDb } from '../../db/open.js';
+import { readWarmableIds } from '../../db/read.js';
+import { indexSpills } from '../../db/spill-index.js';
+import { createWarmQueue } from '../../server/warm.js';
+import { EXIT_INCOMPLETE, EXIT_OK, parseStringFlag } from './archive.js';
+const MAX_PASSES = 12;
+const POLL_MS = 10;
+export function printingHub(write = console.log) {
+    let published = 0;
+    return {
+        get published() {
+            return published;
+        },
+        attach: () => Promise.resolve(),
+        publish: (event, data) => {
+            if (event === 'warm_progress') {
+                const frame = data;
+                published += 1;
+                write(`  warmed ${frame.done}/${frame.total}`);
+            }
+            return Promise.resolve();
+        },
+        beat: () => Promise.resolve(),
+        drain: () => Promise.resolve(),
+        size: () => 0,
+    };
+}
+async function waitForFrames(hub, target) {
+    while (hub.published < target) {
+        await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+    }
+}
+function summarize(perPass, residual, spills) {
+    const total = perPass.reduce((sum, n) => sum + n, 0);
+    const spillTail = spills.indexed === 0 && spills.skipped.length === 0
+        ? ''
+        : `; ${spills.indexed} spilled output(s) indexed` +
+            (spills.skipped.length === 0 ? '' : `, ${spills.skipped.length} skipped`);
+    if (total === 0 && residual === 0) {
+        return `agent-lens warm: nothing to warm — every indexed session is already projected${spillTail}`;
+    }
+    const tail = residual === 0 ? '' : `; ${residual} still unprojected — see \`agent-lens doctor\``;
+    return `agent-lens warm: ${total} warmed over ${perPass.length} pass(es) [${perPass.join(', ')}]${tail}${spillTail}`;
+}
+async function drainCorpus(dataDir, transcriptRoot) {
+    const opened = openDb({ dataDir });
+    const hub = printingHub();
+    const reader = createArchiveReader();
+    const roots = {
+        archiveRoot: resolveArchiveRoot(dataDir),
+        transcriptRoot: resolveTranscriptRoot(transcriptRoot),
+    };
+    const env = createProjectionEnv(reader, roots);
+    const queue = createWarmQueue({ db: opened.db, env, hub });
+    try {
+        createCorpusSweep({ db: opened.db, dataDir, transcriptRoot }).wave1();
+        const perPass = [];
+        let previousKey = '';
+        for (let pass = 0; pass < MAX_PASSES; pass += 1) {
+            const remaining = readWarmableIds(opened.db);
+            const key = remaining.join(',');
+            if (remaining.length === 0 || key === previousKey)
+                break;
+            previousKey = key;
+            const before = hub.published;
+            const started = queue.start();
+            perPass.push(started);
+            await waitForFrames(hub, before + started);
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+        const spills = indexSpills(opened.db, createSpillIndexEnv(opened.db, reader, [roots.archiveRoot, roots.transcriptRoot]));
+        console.log(summarize(perPass, readWarmableIds(opened.db).length, spills));
+        return EXIT_OK;
+    }
+    finally {
+        queue.close();
+        opened.close();
+    }
+}
+export async function warm(args = []) {
+    try {
+        return await drainCorpus(resolveDataDir(parseStringFlag(args, 'dataDir')), parseStringFlag(args, 'transcriptRoot'));
+    }
+    catch (error) {
+        if (error instanceof DbLockedError) {
+            console.error(`${error.message} — warm it through the running server instead: the UI's warm button, or POST /api/warm`);
+            return EXIT_INCOMPLETE;
+        }
+        console.error(`agent-lens warm: ${String(error.message ?? error)}`);
+        return EXIT_INCOMPLETE;
+    }
+}

package/dist/src/cli/index.js

@@ -0,0 +1,102 @@
+import { argv, exit } from 'node:process';
+import { validateArgs } from './args.js';
+export const COMMANDS = [
+    {
+        name: 'start',
+        summary: 'Start the local tracing server + UI (default)',
+        flags: { '--port': 'value', '--host': 'value' },
+        positional: 'none',
+        run: (args) => import('./commands/start.js').then((m) => m.start(args)),
+    },
+    {
+        name: 'doctor',
+        summary: 'Report archive coverage, integrity and retention',
+        flags: {
+            '--dataDir': 'value',
+            '--transcriptRoot': 'value',
+            '--settingsPath': 'value',
+            '--verify': 'boolean',
+            '--json': 'boolean',
+        },
+        positional: 'none',
+        run: (args) => import('./commands/doctor.js').then((m) => m.doctor(args)),
+    },
+    {
+        name: 'archive',
+        summary: 'Mirror Claude Code transcripts into the durable archive',
+        flags: {
+            '--dataDir': 'value',
+            '--transcriptRoot': 'value',
+            '--verify': 'boolean',
+            '--json': 'boolean',
+        },
+        positional: 'none',
+        run: (args) => import('./commands/archive.js').then((m) => m.archive(args)),
+    },
+    {
+        name: 'rebuild',
+        summary: 'Drop the disposable cache, or one session’s projection',
+        flags: { '--dataDir': 'value', '--transcriptRoot': 'value' },
+        positional: 'anywhere',
+        run: (args) => import('./commands/rebuild.js').then((m) => m.rebuild(args)),
+    },
+    {
+        name: 'warm',
+        summary: 'Project every indexed session, printing progress to completion',
+        flags: { '--dataDir': 'value', '--transcriptRoot': 'value' },
+        positional: 'none',
+        run: (args) => import('./commands/warm.js').then((m) => m.warm(args)),
+    },
+    {
+        name: 'schedule',
+        summary: 'Manage the recurring archive job (turn on, report, turn off)',
+        flags: { '--dataDir': 'value' },
+        positional: 'anywhere',
+        run: (args) => import('./commands/schedule.js').then((m) => m.schedule(args)),
+    },
+    {
+        name: 'prune',
+        summary: 'Permanently delete archived transcripts — asks first, no undo',
+        flags: { '--dataDir': 'value', '--transcriptRoot': 'value', '--settingsPath': 'value' },
+        positional: 'anywhere',
+        run: (args) => import('./commands/prune.js').then((m) => m.prune(args)),
+    },
+];
+export function printHelp() {
+    const width = Math.max(...COMMANDS.map((c) => c.name.length));
+    console.log('agent-lens — local-first agentic tracing platform\n');
+    console.log('Usage: agent-lens <command> [options]\n');
+    console.log('Commands:');
+    for (const cmd of COMMANDS) {
+        console.log(`  ${cmd.name.padEnd(width)}  ${cmd.summary}`);
+    }
+}
+export async function main(argv) {
+    const [command, ...rest] = argv;
+    if (!command || command === '--help' || command === '-h' || command === 'help') {
+        printHelp();
+        return 0;
+    }
+    const match = COMMANDS.find((c) => c.name === command);
+    if (!match) {
+        console.error(`Unknown command: ${command}\n`);
+        printHelp();
+        return 1;
+    }
+    const checked = validateArgs(match, rest);
+    if (!checked.ok) {
+        console.error(`agent-lens ${command}: ${checked.message}`);
+        return 1;
+    }
+    try {
+        const code = await match.run(rest);
+        return code ?? 0;
+    }
+    catch (error) {
+        console.error(`agent-lens ${command}: ${String(error?.message ?? error)}`);
+        return 1;
+    }
+}
+if (import.meta.url === `file://${argv[1]}`) {
+    main(argv.slice(2)).then(exit);
+}

package/dist/src/content/resolve.js

@@ -0,0 +1,163 @@
+import { existsSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { isUnderAnyRoot } from '../archive/paths.js';
+import { toolResultsDirOf } from '../corpus/paths.js';
+import { contentBlocks } from '../transcript/blocks.js';
+import { DriftCounter } from '../transcript/drift.js';
+import { classifyLine } from '../transcript/line.js';
+function byteLength(text) {
+    return Buffer.byteLength(text, 'utf8');
+}
+function resultText(children) {
+    return children.flatMap((child) => (child.kind === 'text' ? [child.text] : [])).join('\n');
+}
+function blockAt(env, archivePath, byteOffset, len, blockIndex) {
+    let bytes;
+    try {
+        bytes = env.reader.read(archivePath, byteOffset, len);
+    }
+    catch {
+        return undefined;
+    }
+    if (bytes.length === 0)
+        return undefined;
+    try {
+        const line = classifyLine(JSON.parse(bytes.toString('utf8')), {
+            byteOffset,
+            byteLength: len,
+            drift: new DriftCounter(),
+        });
+        return contentBlocks(line)[blockIndex];
+    }
+    catch {
+        return undefined;
+    }
+}
+function stored(row, field) {
+    const text = (field === 'text' ? row.text : row.input) ?? '';
+    return { text, bytes: field === 'text' ? row.text_bytes : row.input_bytes };
+}
+function fromColumn(row, field, storage) {
+    const { text, bytes } = stored(row, field);
+    return { storage, content: text, byte_size: bytes ?? byteLength(text) };
+}
+function empty(storage) {
+    return { storage, content: '', byte_size: 0 };
+}
+function fromLineRef(row, field, text) {
+    if (text === undefined)
+        return fromColumn(row, field, 'line_ref');
+    const { bytes } = stored(row, field);
+    return { storage: 'line_ref', content: text, byte_size: bytes ?? byteLength(text) };
+}
+function probe(env, path) {
+    try {
+        return env.exists(path) === true;
+    }
+    catch {
+        return false;
+    }
+}
+function within(env, path) {
+    if (env.withinRoots === undefined)
+        return true;
+    try {
+        return env.withinRoots(path) === true;
+    }
+    catch {
+        return false;
+    }
+}
+function spillSource(row, archivePath, env) {
+    if (row.spill_path === null)
+        return undefined;
+    const name = basename(row.spill_path);
+    if (archivePath !== undefined && name !== '') {
+        const mirrored = join(toolResultsDirOf(archivePath), name);
+        if (probe(env, mirrored))
+            return mirrored;
+    }
+    return within(env, row.spill_path) && probe(env, row.spill_path) ? row.spill_path : undefined;
+}
+function fromSpill(row, archivePath, env) {
+    const path = spillSource(row, archivePath, env);
+    if (path === undefined)
+        return empty('missing');
+    try {
+        const size = env.reader.size(path);
+        return {
+            storage: 'spill',
+            content: env.reader.read(path, 0, size).toString('utf8'),
+            byte_size: size,
+            spill_path: path,
+        };
+    }
+    catch {
+        return empty('missing');
+    }
+}
+export function resolveContent(row, field, archivePath, env) {
+    return field === 'input'
+        ? resolveInput(row, archivePath, env)
+        : resolveText(row, archivePath, env);
+}
+function resolveInput(row, archivePath, env) {
+    switch (row.input_storage) {
+        case 'inline':
+            return fromColumn(row, 'input', 'inline');
+        case 'line_ref': {
+            const block = archivePath === undefined || row.block_index === null
+                ? undefined
+                : blockAt(env, archivePath, row.src_offset, row.src_len, row.block_index);
+            const input = block?.kind === 'tool_use' ? block.input : undefined;
+            return fromLineRef(row, 'input', input === undefined ? undefined : JSON.stringify(input));
+        }
+        case 'absent':
+        case null:
+            return empty('absent');
+        default:
+            return fromColumn(row, 'input', row.input_storage);
+    }
+}
+function resolveText(row, archivePath, env) {
+    switch (row.output_storage) {
+        case null: {
+            const text = row.text ?? '';
+            return { storage: 'inline', content: text, byte_size: byteLength(text) };
+        }
+        case 'inline':
+            return fromColumn(row, 'text', 'inline');
+        case 'line_ref': {
+            const { result_offset, result_len, result_block } = row;
+            const reachable = archivePath !== undefined &&
+                result_offset !== null &&
+                result_len !== null &&
+                result_block !== null;
+            const block = reachable
+                ? blockAt(env, archivePath, result_offset, result_len, result_block)
+                : undefined;
+            return fromLineRef(row, 'text', block?.kind === 'tool_result' ? resultText(block.children) : undefined);
+        }
+        case 'spill':
+            return fromSpill(row, archivePath, env);
+        case 'missing':
+            return empty('missing');
+        case 'absent':
+            return empty('absent');
+        default:
+            return fromColumn(row, 'text', row.output_storage);
+    }
+}
+export function createContentEnv(reader, roots) {
+    return {
+        reader,
+        exists: (path) => existsSync(path) || existsSync(`${path}.zst`),
+        withinRoots: (path) => isUnderAnyRoot(path, roots),
+    };
+}
+export function createContentResolver(archivePathOf, env) {
+    return (row, field) => resolveContent(row, field, archivePathOf(row.session_id), env);
+}
+export function createSpillLocator(archivePathOf, env) {
+    return (row) => spillSource(row, archivePathOf(row.session_id), env);
+}

package/dist/src/corpus/env.js

@@ -0,0 +1,32 @@
+import { existsSync } from 'node:fs';
+import { isUnderAnyRoot } from '../archive/paths.js';
+import { readSidecars } from '../db/sidecars.js';
+import { DriftCounter } from '../transcript/drift.js';
+import { classifyLine } from '../transcript/line.js';
+import { sessionRootOf } from './paths.js';
+function readLines(reader, archivePath) {
+    const text = reader.read(archivePath, 0, reader.size(archivePath)).toString('utf8');
+    const drift = new DriftCounter();
+    const lines = [];
+    let byteOffset = 0;
+    for (const line of text.split('\n')) {
+        const byteLength = Buffer.byteLength(line, 'utf8');
+        if (line !== '') {
+            lines.push(classifyLine(JSON.parse(line), { byteOffset, byteLength, drift }));
+        }
+        byteOffset += byteLength + 1;
+    }
+    return { lines, drift };
+}
+export function createProjectionEnv(reader, roots) {
+    return {
+        readLines: (archivePath) => readLines(reader, archivePath),
+        spillEnv: (archivePath) => ({
+            exists: (path) => existsSync(path) || existsSync(`${path}.zst`),
+            sessionRoot: sessionRootOf(archivePath),
+            archiveRoot: roots.archiveRoot,
+            withinRoots: (path) => isUnderAnyRoot(path, [roots.archiveRoot, roots.transcriptRoot, sessionRootOf(archivePath)]),
+        }),
+        sidecars: (archivePath, sourcePath, toolUseIds) => readSidecars(archivePath, sourcePath, toolUseIds, reader),
+    };
+}

package/dist/src/corpus/paths.js

@@ -0,0 +1,70 @@
+import { join } from 'node:path';
+const TRANSCRIPT_EXT = '.jsonl';
+const SEALED_SUFFIX = '.zst';
+const SUBAGENTS_DIR = 'subagents';
+const TOOL_RESULTS_DIR = 'tool-results';
+const WORKFLOWS_DIR = 'workflows';
+const AGENT_PREFIX = 'agent-';
+const JOURNAL_NAME = 'journal.jsonl';
+function segments(relPath) {
+    return relPath
+        .split('\\')
+        .join('/')
+        .split('/')
+        .filter((part) => part !== '' && part !== '.');
+}
+export function encodeProjectDir(projectPath) {
+    return projectPath.split('/').join('-');
+}
+export function decodeProjectDir(slug) {
+    return slug.split('-').join('/');
+}
+export function logicalPathOf(physicalPath) {
+    return physicalPath.endsWith(SEALED_SUFFIX)
+        ? physicalPath.slice(0, -SEALED_SUFFIX.length)
+        : physicalPath;
+}
+export function sessionDirOf(archivePath) {
+    return archivePath.endsWith(TRANSCRIPT_EXT)
+        ? archivePath.slice(0, -TRANSCRIPT_EXT.length)
+        : archivePath;
+}
+export function subagentsDirOf(archivePath) {
+    return join(sessionDirOf(archivePath), SUBAGENTS_DIR);
+}
+export function sessionRootOf(archivePath) {
+    const cut = archivePath.lastIndexOf(`/${SUBAGENTS_DIR}/`);
+    return cut === -1 ? sessionDirOf(archivePath) : archivePath.slice(0, cut);
+}
+export function toolResultsDirOf(archivePath) {
+    return join(sessionRootOf(archivePath), TOOL_RESULTS_DIR);
+}
+export function classifyCorpusPath(relPath) {
+    const parts = segments(relPath);
+    const leaf = parts[parts.length - 1] ?? '';
+    if (parts.length === 2)
+        return leaf.endsWith(TRANSCRIPT_EXT) ? 'session' : 'ignored';
+    if (parts.length >= 4 && parts[2] === SUBAGENTS_DIR) {
+        if (leaf === JOURNAL_NAME)
+            return 'excluded';
+        return leaf.startsWith(AGENT_PREFIX) && leaf.endsWith(TRANSCRIPT_EXT) ? 'sidecar' : 'ignored';
+    }
+    return 'ignored';
+}
+export function workflowParentOf(relPath) {
+    const parts = segments(relPath);
+    if (parts.length < 5)
+        return undefined;
+    if (parts[2] !== SUBAGENTS_DIR || parts[3] !== WORKFLOWS_DIR)
+        return undefined;
+    return parts[1];
+}
+export function projectSlugOf(relPath) {
+    return segments(relPath)[0] ?? '';
+}
+export function rowIdOf(relPath) {
+    const parts = segments(relPath);
+    const leaf = parts[parts.length - 1] ?? '';
+    const stem = leaf.endsWith(TRANSCRIPT_EXT) ? leaf.slice(0, -TRANSCRIPT_EXT.length) : leaf;
+    return stem.startsWith(AGENT_PREFIX) ? stem.slice(AGENT_PREFIX.length) : stem;
+}

package/dist/src/corpus/scan.js

@@ -0,0 +1,85 @@
+import { readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { relativeUnder } from '../archive/paths.js';
+import { foldArchive } from '../db/freshness.js';
+import { readIndexedFolds } from '../db/read.js';
+import { classifyCorpusPath, logicalPathOf, workflowParentOf } from './paths.js';
+export function scanCorpus(db, archiveRoot, sourceRoot) {
+    const result = {
+        changed: [],
+        unchanged: 0,
+        deferred: 0,
+        excluded: [],
+        ignored: 0,
+        unkeyable: [],
+        walked: 0,
+    };
+    walk(archiveRoot, {
+        archiveRoot,
+        sourceRoot,
+        indexed: readIndexedFolds(db),
+        seen: new Set(),
+        result,
+    });
+    return result;
+}
+function walk(dir, state) {
+    let entries;
+    try {
+        entries = readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        const path = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walk(path, state);
+            continue;
+        }
+        if (entry.isFile())
+            visitFile(path, state);
+    }
+}
+function visitFile(path, state) {
+    const { result } = state;
+    result.walked += 1;
+    const archivePath = logicalPathOf(path);
+    const relPath = relativeUnder(state.archiveRoot, archivePath);
+    if (relPath === undefined) {
+        result.ignored += 1;
+        return;
+    }
+    const kind = classifyCorpusPath(relPath);
+    if (kind === 'excluded') {
+        result.excluded.push(relPath);
+        return;
+    }
+    if (kind === 'ignored' || state.seen.has(relPath)) {
+        result.ignored += 1;
+        return;
+    }
+    state.seen.add(relPath);
+    if (kind === 'sidecar' && workflowParentOf(relPath) === undefined) {
+        result.deferred += 1;
+        return;
+    }
+    const fold = foldArchive(archivePath);
+    if (fold === undefined) {
+        result.unkeyable.push(relPath);
+        return;
+    }
+    const row = state.indexed.get(archivePath);
+    if (row !== undefined && row.file_mtime_ms === fold.mtime_ms && row.file_size === fold.size) {
+        result.unchanged += 1;
+        return;
+    }
+    result.changed.push({
+        relPath,
+        archivePath,
+        sourcePath: join(state.sourceRoot, relPath),
+        kind,
+        sealed: path !== archivePath,
+        fold,
+    });
+}