@tangle-network/harness-sessions 0.1.4 → 0.1.6

package/dist/formats/claude-code.js

@@ -12,7 +12,7 @@ import { basename, join } from 'node:path';
 import { SessionBuilder, count, isRecord, sessionError, str, textOf, usageOrNull } from '../builder.js';
 import { fileRecords } from '../source.js';
 import { runFold } from './fold.js';
-import { EVIDENCE, STORES, mtimeMs, storeGlob, walkFiles, withinPath } from '../stores.js';
+import { EVIDENCE, LOCATE_READS, STORES, mapLimit, mtimeMs, storeGlob, walkFiles, withinPath } from '../stores.js';
 import { headRecord } from './head.js';
 import { claudeRecordIsMachine, claudeTypedPromptText } from './claude-records.js';
 const FORMAT = 'claude-code.projects-jsonl';
@@ -254,27 +254,25 @@ async function locate(home, opts) {
         const key = `${project}/${id}`;
         subagentsBySession.set(key, [...(subagentsBySession.get(key) ?? []), join(root, rel)]);
     }
-    const refs = [];
-    for (const rel of all) {
-        if (!main.test(rel))
-            continue;
+    const sessions = await mapLimit(all.filter((rel) => main.test(rel)), LOCATE_READS, async (rel) => {
         const path = join(root, rel);
         const id = basename(rel, '.jsonl');
         const mtime = await mtimeMs(path);
         if (opts.sinceMs && mtime < opts.sinceMs)
-            continue;
+            return [];
         const head = await headRecord(path, (r) => typeof r.cwd === 'string');
         const cwd = head ? str(head.cwd) : null;
         if (opts.cwd && (cwd === null || !withinPath(cwd, opts.cwd)))
-            continue;
+            return [];
         const subagents = subagentsBySession.get(`${rel.split('/')[0]}/${id}`) ?? [];
-        refs.push({ harness: HARNESS, format: FORMAT, nativeSessionId: id, path, files: [path, ...subagents], home, cwd, mtimeMs: mtime, parentNativeSessionId: null });
+        const refs = [{ harness: HARNESS, format: FORMAT, nativeSessionId: id, path, files: [path, ...subagents], home, cwd, mtimeMs: mtime, parentNativeSessionId: null }];
         for (const file of subagents) {
             const agentId = basename(file, '.jsonl').replace(/^agent-/u, '');
             refs.push({ harness: HARNESS, format: FORMAT, nativeSessionId: agentId, path: file, files: [file], home, cwd, mtimeMs: await mtimeMs(file), parentNativeSessionId: id });
         }
-    }
-    return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
+        return refs;
+    });
+    return sessions.flat().sort((a, b) => b.mtimeMs - a.mtimeMs);
 }
 export const claudeCodeReader = {
     harness: HARNESS,

package/dist/formats/codex.js

@@ -13,7 +13,7 @@ import { basename } from 'node:path';
 import { SessionBuilder, argumentsOf, count, isRecord, sessionError, str, textOf, usageOrNull, } from '../builder.js';
 import { fileRecords, isMissing } from '../source.js';
 import { runFold } from './fold.js';
-import { CODEX_HOME_RULE, EVIDENCE, STORES, codexHomes, storeGlob, walkFiles, withinPath } from '../stores.js';
+import { CODEX_HOME_RULE, EVIDENCE, LOCATE_READS, STORES, codexHomes, mapLimit, storeGlob, walkFiles, withinPath } from '../stores.js';
 import { headRecord } from './head.js';
 import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
@@ -339,27 +339,26 @@ function uuidV7Ms(id) {
 }
 /** A ref for each of `files` that passes the filters, from its `session_meta`, with the file's bytes. */
 async function refsOf(files, home, opts) {
-    const found = [];
-    for (const path of files) {
+    const found = await mapLimit(files, LOCATE_READS, async (path) => {
         let info;
         try {
             info = await stat(path);
         }
         catch (error) {
             if (isMissing(error))
-                continue;
+                return null;
             throw error;
         }
         if (opts.sinceMs && info.mtimeMs < opts.sinceMs)
-            continue;
+            return null;
         const meta = await headRecord(path, (r) => r.type === 'session_meta', 4);
         const payload = meta && isRecord(meta.payload) ? meta.payload : {};
         const cwd = str(payload.cwd);
         if (opts.cwd && (cwd === null || !withinPath(cwd, opts.cwd)))
-            continue;
+            return null;
         const source = isRecord(payload.source) ? payload.source : {};
         const spawn = isRecord(source.subagent) && isRecord(source.subagent.thread_spawn) ? source.subagent.thread_spawn : {};
-        found.push({
+        return {
             bytes: info.size,
             ref: {
                 harness: HARNESS,
@@ -372,9 +371,9 @@ async function refsOf(files, home, opts) {
                 mtimeMs: info.mtimeMs,
                 parentNativeSessionId: str(payload.parent_thread_id) ?? str(spawn.parent_thread_id),
             },
-        });
-    }
-    return found;
+        };
+    });
+    return found.filter((entry) => entry !== null);
 }
 /**
  * Sessions in every Codex home under `home`. A lookup by id takes the rollout named for it; when no

package/dist/formats/head.d.ts

@@ -1,5 +1,8 @@
 /**
- * The first record among a session file's first `limit` records that satisfies `match`, read
- * without consuming the rest of the file. Used to learn a session's id and cwd while locating.
+ * The first record among a session file's first `limit` records that satisfies `match`. Locate
+ * reads one for every session in a store, so it reads the file a small step at a time and stops
+ * at the answer: never the rest of the file, and no digest of it. Lines parse as the session
+ * reader parses them: a blank, malformed or non-object line is skipped, and a last line with no
+ * newline counts when it is a whole record.
  */
 export declare function headRecord(path: string, match: (record: Record<string, unknown>) => boolean, limit?: number): Promise<Record<string, unknown> | null>;

package/dist/formats/head.js

@@ -1,23 +1,69 @@
-import { newSourceStats, readJsonlRecords } from '../source.js';
+import { open } from 'node:fs/promises';
+import { StringDecoder } from 'node:string_decoder';
+import { isMissing } from '../source.js';
+/** Bytes a head read takes at a time. A session's identifying record is almost always in the first. */
+const HEAD_CHUNK = 64 * 1024;
 /**
- * The first record among a session file's first `limit` records that satisfies `match`, read
- * without consuming the rest of the file. Used to learn a session's id and cwd while locating.
+ * The first record among a session file's first `limit` records that satisfies `match`. Locate
+ * reads one for every session in a store, so it reads the file a small step at a time and stops
+ * at the answer: never the rest of the file, and no digest of it. Lines parse as the session
+ * reader parses them: a blank, malformed or non-object line is skipped, and a last line with no
+ * newline counts when it is a whole record.
  */
 export async function headRecord(path, match, limit = 64) {
-    let seen = 0;
+    let handle;
     try {
-        for await (const record of readJsonlRecords(path, newSourceStats(path))) {
-            if (match(record))
-                return record;
-            seen += 1;
-            if (seen >= limit)
-                return null;
-        }
+        handle = await open(path, 'r');
     }
     catch (error) {
-        if (error?.code === 'ENOENT')
+        if (isMissing(error))
             return null;
         throw error;
     }
-    return null;
+    let seen = 0;
+    // undefined: read on; null: `limit` records without a match; a record: the match.
+    const take = (line) => {
+        const text = line.replace(/\r$/u, '');
+        if (text.trim().length === 0)
+            return undefined;
+        let value;
+        try {
+            value = JSON.parse(text);
+        }
+        catch {
+            return undefined;
+        }
+        if (value === null || typeof value !== 'object' || Array.isArray(value))
+            return undefined;
+        if (match(value))
+            return value;
+        seen += 1;
+        return seen >= limit ? null : undefined;
+    };
+    try {
+        const decoder = new StringDecoder('utf8');
+        const buffer = Buffer.allocUnsafe(HEAD_CHUNK);
+        // The line being read, in pieces, so a record longer than a step is joined once.
+        let pending = [];
+        let position = 0;
+        for (;;) {
+            const { bytesRead } = await handle.read(buffer, 0, HEAD_CHUNK, position);
+            if (bytesRead === 0)
+                return take(pending.join('') + decoder.end()) ?? null;
+            position += bytesRead;
+            const text = decoder.write(buffer.subarray(0, bytesRead));
+            let start = 0;
+            for (let end = text.indexOf('\n'); end !== -1; end = text.indexOf('\n', start)) {
+                const found = take(pending.join('') + text.slice(start, end));
+                pending = [];
+                if (found !== undefined)
+                    return found;
+                start = end + 1;
+            }
+            pending.push(text.slice(start));
+        }
+    }
+    finally {
+        await handle.close();
+    }
 }

package/dist/input.d.ts

@@ -11,6 +11,12 @@ export interface SessionInputInit {
     parentNativeSessionId?: string | null;
     /** Names the source in integrity records (a path, a blob ref). */
     label?: string;
+    /**
+     * The file line `input` starts on, when it continues a longer file (records appended since a
+     * previous read). A record without an id of its own is named by its line, so the same record
+     * gets the same id whichever way the file is read. Default 1.
+     */
+    firstLine?: number;
 }
 /** Read a JSONL-format session (Claude Code, Codex, Pi, Kimi) from text or parsed records, synchronously. */
 export declare function readSessionInput(harness: string, input: SessionInput, init?: SessionInputInit, options?: Pick<ReadOptions, 'corruption'>): HarnessSession;

package/dist/input.js

@@ -31,7 +31,10 @@ function foldInput(harness, input, init, mode, options) {
         parentNativeSessionId: init.parentNativeSessionId ?? null,
     };
     const fold = create(ref, mode, stats);
-    const records = 'text' in input ? textRecordsSync(input.text, stats, { strict: options.corruption === 'strict' }) : arrayRecordsSync(input.records, stats);
+    const firstLine = init.firstLine ?? 1;
+    const records = 'text' in input
+        ? textRecordsSync(input.text, stats, { strict: options.corruption === 'strict', firstLine })
+        : arrayRecordsSync(input.records, stats, firstLine);
     for (const record of records)
         fold.observe(record);
     return fold.finish();

package/dist/source.d.ts

@@ -40,12 +40,16 @@ export declare function fileRecords(path: string, options?: {
 }): RecordSource;
 /** Stats for bytes already in memory. */
 export declare function memoryStats(label: string, text?: string): SourceStats;
-/** The records of JSONL text already in memory (a retained copy, a blob), read synchronously. */
+/**
+ * The records of JSONL text already in memory (a retained copy, a blob), read synchronously.
+ * `firstLine` numbers the text's first line, for text that continues a longer file.
+ */
 export declare function textRecordsSync(text: string, stats: SourceStats, options?: {
     strict?: boolean;
+    firstLine?: number;
 }): Generator<Record<string, unknown>>;
-/** Records a caller already parsed (one per line, in order). Non-objects count as unparsed. */
-export declare function arrayRecordsSync(values: readonly unknown[], stats: SourceStats): Generator<Record<string, unknown>>;
+/** Records a caller already parsed (one per line, in order, the first on line `firstLine`). Non-objects count as unparsed. */
+export declare function arrayRecordsSync(values: readonly unknown[], stats: SourceStats, firstLine?: number): Generator<Record<string, unknown>>;
 /**
  * Run `use` against a SQLite store opened read-only in place ({@link openSqlite}). A store SQLite
  * cannot open in place is read from a private copy instead: a write-ahead log with no `-shm` in a

package/dist/source.js

@@ -116,10 +116,15 @@ export function memoryStats(label, text) {
     const bytes = Buffer.from(text, 'utf8');
     return { path: label, sha256: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, bytes: bytes.length, unparsed: 0, tornTail: false, line: 0 };
 }
-/** The records of JSONL text already in memory (a retained copy, a blob), read synchronously. */
+/**
+ * The records of JSONL text already in memory (a retained copy, a blob), read synchronously.
+ * `firstLine` numbers the text's first line, for text that continues a longer file.
+ */
 export function* textRecordsSync(text, stats, options = {}) {
     const lines = text.split('\n');
-    for (const [index, raw] of lines.entries()) {
+    const first = options.firstLine ?? 1;
+    for (const [position, raw] of lines.entries()) {
+        const index = first - 1 + position;
         const line = raw.replace(/\r$/u, '');
         if (line.trim().length === 0)
             continue;
@@ -128,7 +133,7 @@ export function* textRecordsSync(text, stats, options = {}) {
             value = JSON.parse(line);
         }
         catch {
-            if (index === lines.length - 1) {
+            if (position === lines.length - 1) {
                 stats.tornTail = true;
                 continue;
             }
@@ -147,9 +152,10 @@ export function* textRecordsSync(text, stats, options = {}) {
         yield value;
     }
 }
-/** Records a caller already parsed (one per line, in order). Non-objects count as unparsed. */
-export function* arrayRecordsSync(values, stats) {
-    for (const [index, value] of values.entries()) {
+/** Records a caller already parsed (one per line, in order, the first on line `firstLine`). Non-objects count as unparsed. */
+export function* arrayRecordsSync(values, stats, firstLine = 1) {
+    for (const [position, value] of values.entries()) {
+        const index = firstLine - 1 + position;
         if (value === null || typeof value !== 'object' || Array.isArray(value)) {
             noteUnparsed(stats, index + 1);
             continue;

package/dist/stores.d.ts

@@ -2,6 +2,10 @@ import type { NativeSessionStore } from './schema.js';
 export { assertStoreRoot, CODEX_HOME_RULE, EVIDENCE, isCodexHome, SESSION_FORMAT_IDS, STORES, storeGlob, withinPath } from './catalog.js';
 /** Every file under `dir`, as paths relative to it with `/` separators. Missing dirs yield none. */
 export declare function walkFiles(dir: string, signal?: AbortSignal): Promise<string[]>;
+/** Session files a locate reads at once: a store holds thousands, and each read waits on the disk. */
+export declare const LOCATE_READS = 32;
+/** `fn` over `items`, at most `limit` at a time, with the results in the order of `items`. */
+export declare function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
 /** Files under `home` that one store glob matches, as absolute paths. */
 export declare function storeFiles(home: string, store: NativeSessionStore, glob: string, nativeSessionId?: string): Promise<string[]>;
 export declare function mtimeMs(path: string): Promise<number>;

package/dist/stores.js

@@ -29,6 +29,22 @@ export async function walkFiles(dir, signal) {
     await visit(dir);
     return out.sort();
 }
+/** Session files a locate reads at once: a store holds thousands, and each read waits on the disk. */
+export const LOCATE_READS = 32;
+/** `fn` over `items`, at most `limit` at a time, with the results in the order of `items`. */
+export async function mapLimit(items, limit, fn) {
+    const results = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length) {
+            const index = next;
+            next += 1;
+            results[index] = await fn(items[index]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return results;
+}
 /** Files under `home` that one store glob matches, as absolute paths. */
 export async function storeFiles(home, store, glob, nativeSessionId) {
     const root = join(home, store.root);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/harness-sessions",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "One reader for every coding-agent harness's native session (Claude Code, Codex, OpenCode, Pi): store layouts, a normalized session with messages, tool calls, served models, usage and ending errors, and projections to chat messages, OTLP spans and turns",
   "type": "module",
   "sideEffects": false,