@particle-academy/prism-harness 0.5.0 → 0.7.0

package/README.md

@@ -3,9 +3,11 @@
 Durable agent sessions — threads, session state and store drivers. The
 TypeScript port of [`particle-academy/prism-harness`](https://github.com/Particle-Academy/prism-harness).
 
-Zero runtime dependencies. Node 20+.
+Zero runtime dependencies. Node 22+.
 
-This package is private while coordinated parity work is in progress.
+```
+npm install @particle-academy/prism-harness
+```
 
 ```ts
 import { FileSessionStore, MemorySessionStore, PrismHarness } from '@particle-academy/prism-harness';
@@ -107,6 +109,8 @@ as `request.providerOptions`, on every step of every run in that mode. Pass them
 to your provider call (with `prism-ts`, `withProviderOptions()`):
 
 ```ts
+import { ModeRegistry } from '@particle-academy/prism-harness';
+
 new ModeRegistry({
   modes: { overseer: { system_prompt: '...', provider_options: { thinking: { type: 'adaptive' } } } },
 });
@@ -166,6 +170,8 @@ the text, so a thread that contains a structured answer reads like any other.
 **A document that misses the schema is refused, not repaired.**
 
 ```ts
+import { HarnessError } from '@particle-academy/prism-harness';
+
 try {
   const plan = (await runtime.sendStructured(session, brief, schema)).structured;
 } catch (error) {
@@ -201,7 +207,11 @@ in the other for the same stated reason.
 A mode names the tools a person must approve:
 
 ```ts
-guarded: { system_prompt: '...', tools: ['read', 'delete'], requires_approval: ['delete'] },
+new ModeRegistry({
+  modes: {
+    guarded: { system_prompt: '...', tools: ['read', 'delete'], requires_approval: ['delete'] },
+  },
+});
 ```
 
 A step that calls one stops with `finishReason: 'awaiting_approval'`. The calls
@@ -209,6 +219,8 @@ that need nobody have already run. Record a decision for every pending approval,
 then resume with an empty prompt:
 
 ```ts
+import { recordApproval } from '@particle-academy/prism-harness';
+
 const response = await runtime.send(session, 'Clean up the failed run');
 
 if (response.finishReason === 'awaiting_approval') {
@@ -329,6 +341,18 @@ badly.
   as held, bounded by the file's own age so a process that died mid-write cannot
   wedge the key either.
 
+  **Threads are append-only logs** (`<digest>.jsonl` beside the payload file):
+  one line per message, one write and an `fsync` per `record()`, so recording
+  costs the size of the new messages however long the thread already is.
+  Before 0.7.0, every message re-read and rewrote the whole conversation. A
+  thread stored that way is read as before and moved into the log the first
+  time it grows. A line cut off by a crash is dropped, and the next append
+  starts after the last complete line. A damaged line anywhere else is refused
+  with `unmappable_content` rather than silently skipped.
+
+  The thread log is **not yet shared with the PHP and Python ports**: a Python
+  worker pointed at the same directory does not read it.
+
 Implement `SessionStore` for anything else. Declare your own `durability()`:
 only you know whether your Redis is persistent or a disposable cache, and that
 declaration is an assertion about your infrastructure, not a preference.

package/dist/index.d.ts

@@ -2,8 +2,8 @@ export { HarnessError } from './errors.js';
 export type { HarnessErrorCode, HarnessErrorOptions } from './errors.js';
 export type { JsonObject, JsonValue } from './json.js';
 export { isJsonObject } from './json.js';
-export type { Durability, SessionStore } from './stores/session-store.js';
-export { isDurable } from './stores/session-store.js';
+export type { AppendLogStore, Durability, SessionStore } from './stores/session-store.js';
+export { isDurable, supportsAppendLog } from './stores/session-store.js';
 export { MemorySessionStore } from './stores/memory-session-store.js';
 export { FileSessionStore } from './stores/file-session-store.js';
 export { SessionStoreManager, SLOT_DURABLE, SLOT_EPHEMERAL } from './stores/store-manager.js';

package/dist/index.js

@@ -1,6 +1,6 @@
 export { HarnessError } from './errors.js';
 export { isJsonObject } from './json.js';
-export { isDurable } from './stores/session-store.js';
+export { isDurable, supportsAppendLog } from './stores/session-store.js';
 export { MemorySessionStore } from './stores/memory-session-store.js';
 export { FileSessionStore } from './stores/file-session-store.js';
 export { SessionStoreManager, SLOT_DURABLE, SLOT_EPHEMERAL } from './stores/store-manager.js';

package/dist/stores/file-session-store.d.ts

@@ -1,5 +1,5 @@
 import type { JsonObject } from '../json.js';
-import type { Durability, SessionStore } from './session-store.js';
+import type { AppendLogStore, Durability, SessionStore } from './session-store.js';
 /**
  * State on disk, as one JSON file per key. DURABLE, and it says so.
  *
@@ -23,12 +23,23 @@ import type { Durability, SessionStore } from './session-store.js';
  * other. Two workers on different machines sharing a network filesystem do NOT
  * reliably, and no file lock can promise that; use a database or Redis there.
  */
-export declare class FileSessionStore implements SessionStore {
+export declare class FileSessionStore implements SessionStore, AppendLogStore {
     #private;
     constructor(directory: string);
     durability(): Durability;
     get(key: string): Promise<JsonObject | null>;
     put(key: string, payload: JsonObject, ttlSeconds?: number | null): Promise<void>;
     forget(key: string): Promise<void>;
+    /**
+     * One JSON document per line, appended through a single write and then
+     * synced, so a message that `record()` returned is on disk. Before writing,
+     * a TORN tail (a previous write cut off by a crash) is truncated back to the
+     * last complete line; appending after it would glue the new line onto the
+     * fragment and corrupt both.
+     */
+    appendLog(key: string, entries: readonly JsonObject[]): Promise<void>;
+    readLog(key: string): Promise<JsonObject[] | null>;
+    logLength(key: string): Promise<number | null>;
+    clearLog(key: string): Promise<void>;
     withLock<T>(key: string, callback: () => T | Promise<T>, ttlSeconds?: number, waitSeconds?: number): Promise<T>;
 }

package/dist/stores/file-session-store.js

@@ -1,6 +1,6 @@
 import { createHash } from 'node:crypto';
 import { mkdirSync } from 'node:fs';
-import { mkdir, open, readFile, rename, rm, stat, unlink, writeFile } from 'node:fs/promises';
+import { mkdir, open, readFile, rename, rm, stat, truncate, unlink, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { isJsonObject } from '../json.js';
 import { HarnessError } from '../errors.js';
@@ -29,6 +29,14 @@ import { HarnessError } from '../errors.js';
  */
 export class FileSessionStore {
     #directory;
+    /**
+     * What this instance last saw of each log: its size on disk and how many
+     * complete entries that size holds. Lets `logLength()` answer without reading
+     * the log, and stays honest across processes because it is only trusted while
+     * the file is still exactly that size. Another writer changes the size, and
+     * the next call recounts.
+     */
+    #logs = new Map();
     constructor(directory) {
         this.#directory = directory;
         mkdirSync(directory, { recursive: true });
@@ -87,6 +95,130 @@ export class FileSessionStore {
     }
     async forget(key) {
         await rm(this.#pathFor(key), { force: true });
+        await rm(this.#logPathFor(key), { force: true });
+        this.#logs.delete(key);
+    }
+    /**
+     * One JSON document per line, appended through a single write and then
+     * synced, so a message that `record()` returned is on disk. Before writing,
+     * a TORN tail (a previous write cut off by a crash) is truncated back to the
+     * last complete line; appending after it would glue the new line onto the
+     * fragment and corrupt both.
+     */
+    async appendLog(key, entries) {
+        const path = this.#logPathFor(key);
+        await mkdir(this.#directory, { recursive: true });
+        const before = await this.#repairTail(path);
+        const data = entries.map((entry) => `${JSON.stringify(entry)}\n`).join('');
+        const handle = await open(path, 'a');
+        try {
+            if (data !== '')
+                await handle.write(data);
+            await handle.sync();
+        }
+        finally {
+            await handle.close();
+        }
+        const known = this.#logs.get(key);
+        if (known !== undefined && known.size === before) {
+            this.#logs.set(key, { size: before + Buffer.byteLength(data), count: known.count + entries.length });
+        }
+        else {
+            this.#logs.delete(key);
+        }
+    }
+    async readLog(key) {
+        let raw;
+        try {
+            raw = await readFile(this.#logPathFor(key), 'utf8');
+        }
+        catch (error) {
+            if (isMissing(error))
+                return null;
+            throw error;
+        }
+        const lines = raw.split('\n');
+        // Everything after the last newline is a line that never finished. Empty
+        // when the file ends cleanly, a fragment when a crash cut a write short.
+        lines.pop();
+        const entries = lines.map((line, index) => {
+            let entry;
+            try {
+                entry = JSON.parse(line);
+            }
+            catch (error) {
+                throw HarnessError.unmappableContent(`line ${index + 1} of the thread log for [${key}] is not valid JSON`, {
+                    cause: error,
+                });
+            }
+            if (!isJsonObject(entry)) {
+                throw HarnessError.unmappableContent(`line ${index + 1} of the thread log for [${key}] is not an object`);
+            }
+            return entry;
+        });
+        this.#logs.set(key, { size: Buffer.byteLength(raw), count: entries.length });
+        return entries;
+    }
+    async logLength(key) {
+        let size;
+        try {
+            size = (await stat(this.#logPathFor(key))).size;
+        }
+        catch (error) {
+            if (isMissing(error))
+                return null;
+            throw error;
+        }
+        const known = this.#logs.get(key);
+        if (known !== undefined && known.size === size)
+            return known.count;
+        return (await this.readLog(key))?.length ?? null;
+    }
+    async clearLog(key) {
+        await mkdir(this.#directory, { recursive: true });
+        await writeFile(this.#logPathFor(key), '', 'utf8');
+        this.#logs.set(key, { size: 0, count: 0 });
+    }
+    /** Cut a torn tail back to the last newline. Returns the file's size afterwards. */
+    async #repairTail(path) {
+        let size;
+        try {
+            size = (await stat(path)).size;
+        }
+        catch (error) {
+            if (isMissing(error))
+                return 0;
+            throw error;
+        }
+        if (size === 0)
+            return 0;
+        const handle = await open(path, 'r');
+        try {
+            const last = Buffer.alloc(1);
+            await handle.read(last, 0, 1, size - 1);
+            if (last[0] === 0x0a)
+                return size;
+            // Walk back a chunk at a time: one message can be larger than any chunk.
+            const chunk = Buffer.alloc(64 * 1024);
+            let end = size;
+            while (end > 0) {
+                const start = Math.max(0, end - chunk.length);
+                const { bytesRead } = await handle.read(chunk, 0, end - start, start);
+                const newline = chunk.subarray(0, bytesRead).lastIndexOf(0x0a);
+                if (newline !== -1) {
+                    await handle.close();
+                    await truncate(path, start + newline + 1);
+                    return start + newline + 1;
+                }
+                end = start;
+            }
+            await handle.close();
+            await truncate(path, 0);
+            return 0;
+        }
+        finally {
+            await handle.close().catch(() => undefined);
+        }
     }
     async withLock(key, callback, ttlSeconds = 10, waitSeconds = 5) {
         const lockPath = `${this.#pathFor(key)}.lock`;
@@ -237,7 +369,11 @@ export class FileSessionStore {
      * the mapping stays inspectable.
      */
     #pathFor(key) {
-        return join(this.#directory, `${createHash('sha256').update(key).digest('hex').slice(0, 32)}.json`);
+        return join(this.#directory, `${digestOf(key)}.json`);
+    }
+    /** The key's append-only log, beside its payload: same digest, `.jsonl`. */
+    #logPathFor(key) {
+        return join(this.#directory, `${digestOf(key)}.jsonl`);
     }
 }
 /**
@@ -251,6 +387,10 @@ export class FileSessionStore {
  * else.
  */
 const STAMP = /^\d+\n$/;
+/** A key's file name. Colons are illegal on Windows and keys run long, so it is digested. */
+function digestOf(key) {
+    return createHash('sha256').update(key).digest('hex').slice(0, 32);
+}
 function stamp(expiresAt) {
     return `${String(Math.trunc(expiresAt))}\n`;
 }

package/dist/stores/session-store.d.ts

@@ -63,3 +63,31 @@ export interface SessionStore {
      */
     durability(): Durability;
 }
+/**
+ * OPTIONAL: an append-only log per key, which a store may offer for threads.
+ *
+ * A thread only ever grows at the end, and through `get`/`put` every new
+ * message costs a read, parse, serialise and rewrite of the WHOLE conversation.
+ * That is quadratic over a session: Genie 2 measured a 9 MB thread being
+ * rewritten for every chat row. A store that can append says so by
+ * implementing this, and `Thread` then writes one line per message. A store
+ * that does not keeps working through `get`/`put` exactly as before.
+ *
+ * Every method is called with the key's `withLock` held by the caller.
+ */
+export interface AppendLogStore {
+    /** Durably append entries in order. One line each; nothing earlier is rewritten. */
+    appendLog(key: string, entries: readonly JsonObject[]): Promise<void>;
+    /**
+     * Every complete entry, or null when no log exists for the key.
+     *
+     * A torn LAST line (a write cut off by a crash) is dropped. A damaged line
+     * anywhere else is history that cannot be read, and is refused loudly.
+     */
+    readLog(key: string): Promise<JsonObject[] | null>;
+    /** How many complete entries, or null when no log exists. Need not read them all. */
+    logLength(key: string): Promise<number | null>;
+    /** Drop every entry; the log then exists and is empty. */
+    clearLog(key: string): Promise<void>;
+}
+export declare function supportsAppendLog(store: SessionStore): store is SessionStore & AppendLogStore;

package/dist/stores/session-store.js

@@ -1,3 +1,8 @@
 export function isDurable(durability) {
     return durability === 'durable';
 }
+export function supportsAppendLog(store) {
+    const candidate = store;
+    return typeof candidate.appendLog === 'function' && typeof candidate.readLog === 'function'
+        && typeof candidate.logLength === 'function' && typeof candidate.clearLog === 'function';
+}

package/dist/thread.js

@@ -1,5 +1,6 @@
 import { isJsonObject } from './json.js';
 import { HarnessError } from './errors.js';
+import { supportsAppendLog } from './stores/session-store.js';
 /**
  * The stored conversation a session is bound to.
  *
@@ -23,17 +24,29 @@ export class Thread {
         this.key = key;
     }
     async messages() {
+        if (supportsAppendLog(this.store)) {
+            const log = await this.store.readLog(this.key);
+            if (log !== null)
+                return log.map((entry) => this.#toMessage(entry));
+        }
+        return (await this.#stored()).map((entry) => this.#toMessage(entry));
+    }
+    /** The whole-payload form: every store before the log, and any store without one. */
+    async #stored() {
         const stored = await this.store.get(this.key);
         if (stored === null)
             return [];
         const messages = stored.messages;
-        if (!Array.isArray(messages))
-            return [];
-        return messages.filter(isJsonObject).map((entry) => this.#toMessage(entry));
+        return Array.isArray(messages) ? messages.filter(isJsonObject) : [];
     }
     /** How many messages the thread holds. */
     async count() {
-        return (await this.messages()).length;
+        if (supportsAppendLog(this.store)) {
+            const length = await this.store.logLength(this.key);
+            if (length !== null)
+                return length;
+        }
+        return (await this.#stored()).length;
     }
     /**
      * Append messages, in order, and return them with their assigned positions.
@@ -43,17 +56,20 @@ export class Thread {
     async record(messages, runId = null) {
         if (messages.length === 0)
             return [];
-        return this.store.withLock(this.key, async () => {
-            const stored = (await this.store.get(this.key)) ?? {};
+        const store = this.store;
+        if (supportsAppendLog(store)) {
+            return store.withLock(this.key, async () => {
+                const length = (await store.logLength(this.key)) ?? (await this.#migrate(store));
+                const appended = this.#entries(messages, length, runId);
+                await store.appendLog(this.key, appended);
+                return appended.map((entry) => this.#toMessage(entry));
+            });
+        }
+        return store.withLock(this.key, async () => {
+            const stored = (await store.get(this.key)) ?? {};
             const existing = Array.isArray(stored.messages) ? stored.messages.filter(isJsonObject) : [];
-            const recordedAt = new Date().toISOString();
-            const appended = messages.map((message, index) => ({
-                position: existing.length + index + 1,
-                message,
-                run_id: runId,
-                recorded_at: recordedAt,
-            }));
-            await this.store.put(this.key, {
+            const appended = this.#entries(messages, existing.length, runId);
+            await store.put(this.key, {
                 ...stored,
                 messages: [...existing, ...appended],
             });
@@ -68,11 +84,40 @@ export class Thread {
      * side effect of clearing session state.
      */
     async clear() {
-        await this.store.withLock(this.key, async () => {
-            const stored = (await this.store.get(this.key)) ?? {};
-            await this.store.put(this.key, { ...stored, messages: [] });
+        const store = this.store;
+        await store.withLock(this.key, async () => {
+            if (supportsAppendLog(store))
+                await store.clearLog(this.key);
+            const stored = await store.get(this.key);
+            if (stored !== null)
+                await store.put(this.key, { ...stored, messages: [] });
         });
     }
+    /**
+     * Move a whole-payload thread into the log, once, the first time it grows.
+     *
+     * The log is written FIRST and the payload emptied after it. A crash between
+     * the two leaves both, and the log wins on every read, so nothing is lost
+     * and nothing is counted twice. Returns how many entries the log now holds.
+     */
+    async #migrate(store) {
+        const existing = await this.#stored();
+        await store.appendLog(this.key, existing);
+        if (existing.length > 0) {
+            const stored = (await store.get(this.key)) ?? {};
+            await store.put(this.key, { ...stored, messages: [] });
+        }
+        return existing.length;
+    }
+    #entries(messages, length, runId) {
+        const recordedAt = new Date().toISOString();
+        return messages.map((message, index) => ({
+            position: length + index + 1,
+            message,
+            run_id: runId,
+            recorded_at: recordedAt,
+        }));
+    }
     #toMessage(entry) {
         const message = entry.message;
         if (!isJsonObject(message)) {

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@particle-academy/prism-harness",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Durable agent sessions for TypeScript — threads, session state and store drivers.",
   "type": "module",
   "sideEffects": false,
   "engines": {
-    "node": ">=20"
+    "node": ">=22"
   },
   "exports": {
     ".": {