@zenera/neo 1.1.0

package/dist/payload-stores/file.js

@@ -0,0 +1,112 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { hash, previewOf } from "../payload.js";
+// ---------------------------------------------------------------------------
+// Filesystem-backed payload store
+// ---------------------------------------------------------------------------
+/**
+ * A content address is the only thing that ever becomes a path segment, so it
+ * is validated before it touches the filesystem: addresses arrive from
+ * deserialized state, which may come from an untrusted bundle.
+ */
+const ADDRESS = /^[0-9a-f]{64}$/;
+/**
+ * Same contract as the in-memory store, on disk. Content addressing does the
+ * hard part: writes are write-once and therefore idempotent under retry and
+ * replay, and two processes racing on the same blob produce identical bytes.
+ */
+export class FilePayloadStore {
+    id;
+    #dir;
+    #concurrency;
+    #verify;
+    constructor(opts) {
+        const o = typeof opts === 'string' ? { dir: opts } : opts;
+        this.#dir = resolve(o.dir);
+        this.id = o.id ?? 'file';
+        this.#concurrency = Math.max(1, o.concurrency ?? 32);
+        this.#verify = o.verify ?? false;
+    }
+    get dir() {
+        return this.#dir;
+    }
+    async put(value) {
+        const sha256 = hash(value);
+        const path = this.#path(sha256);
+        if (!(await exists(path))) {
+            await mkdir(dirname(path), { recursive: true });
+            // Write elsewhere then rename: a reader never observes a partial
+            // blob, and a crash leaves at worst an orphaned temp file.
+            const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
+            await writeFile(tmp, value, 'utf8');
+            try {
+                await rename(tmp, path);
+            }
+            catch (err) {
+                await rm(tmp, { force: true });
+                throw err;
+            }
+        }
+        return {
+            store: this.id,
+            sha256,
+            size: Buffer.byteLength(value),
+            preview: previewOf(value),
+        };
+    }
+    async get(p) {
+        return this.#read(p.sha256);
+    }
+    async getMany(ps) {
+        return mapPool(ps, this.#concurrency, (p) => this.#read(p.sha256));
+    }
+    /** Two levels of fan-out keep any single directory small. */
+    #path(sha256) {
+        if (!ADDRESS.test(sha256)) {
+            throw new Error(`invalid payload address: ${JSON.stringify(sha256)}`);
+        }
+        return join(this.#dir, sha256.slice(0, 2), sha256.slice(2, 4), sha256);
+    }
+    async #read(sha256) {
+        const path = this.#path(sha256);
+        let value;
+        try {
+            value = await readFile(path, 'utf8');
+        }
+        catch (err) {
+            if (err.code === 'ENOENT') {
+                throw new Error(`payload not found in store "${this.id}": ${sha256}`);
+            }
+            throw err;
+        }
+        if (this.#verify && hash(value) !== sha256) {
+            throw new Error(`payload corrupted in store "${this.id}": ${sha256}`);
+        }
+        return value;
+    }
+}
+async function exists(path) {
+    try {
+        await stat(path);
+        return true;
+    }
+    catch (err) {
+        if (err.code === 'ENOENT') {
+            return false;
+        }
+        throw err;
+    }
+}
+async function mapPool(items, limit, fn) {
+    const out = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        for (let i = next++; i < items.length; i = next++) {
+            out[i] = await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return out;
+}
+//# sourceMappingURL=file.js.map

package/dist/payload-stores/in-memory.d.ts

@@ -0,0 +1,17 @@
+import { type Payload, type PayloadStore } from '../payload.ts';
+/**
+ * The zero-I/O backend, and the one `Services` falls back to. Content-addressed
+ * and therefore write-once: putting existing content is a no-op returning the
+ * same reference, which makes writes idempotent under retry and replay for
+ * free.
+ */
+export declare class InMemoryPayloadStore implements PayloadStore {
+    #private;
+    readonly id: string;
+    constructor(id?: string);
+    put(value: string): Promise<Payload>;
+    get(p: Payload): Promise<string>;
+    getMany(ps: Payload[]): Promise<string[]>;
+    get size(): number;
+}
+//# sourceMappingURL=in-memory.d.ts.map

package/dist/payload-stores/in-memory.js

@@ -0,0 +1,40 @@
+import { hash, previewOf } from "../payload.js";
+/**
+ * The zero-I/O backend, and the one `Services` falls back to. Content-addressed
+ * and therefore write-once: putting existing content is a no-op returning the
+ * same reference, which makes writes idempotent under retry and replay for
+ * free.
+ */
+export class InMemoryPayloadStore {
+    id;
+    #blobs = new Map();
+    constructor(id = 'mem') {
+        this.id = id;
+    }
+    async put(value) {
+        const sha256 = hash(value);
+        if (!this.#blobs.has(sha256)) {
+            this.#blobs.set(sha256, value);
+        }
+        return {
+            store: this.id,
+            sha256,
+            size: Buffer.byteLength(value),
+            preview: previewOf(value),
+        };
+    }
+    async get(p) {
+        const v = this.#blobs.get(p.sha256);
+        if (v === undefined) {
+            throw new Error(`payload not found in store "${this.id}": ${p.sha256}`);
+        }
+        return v;
+    }
+    async getMany(ps) {
+        return Promise.all(ps.map((p) => this.get(p)));
+    }
+    get size() {
+        return this.#blobs.size;
+    }
+}
+//# sourceMappingURL=in-memory.js.map

package/dist/payload-stores/index.d.ts

@@ -0,0 +1,4 @@
+export * from './factory.ts';
+export * from './file.ts';
+export * from './in-memory.ts';
+//# sourceMappingURL=index.d.ts.map

package/dist/payload-stores/index.js

@@ -0,0 +1,5 @@
+// Backends for the `PayloadStore` contract in `../payload.ts`.
+export * from "./factory.js";
+export * from "./file.js";
+export * from "./in-memory.js";
+//# sourceMappingURL=index.js.map

package/dist/payload.d.ts

@@ -0,0 +1,89 @@
+/**
+ * A content-addressed reference. Uniform (never inline) so a state stays
+ * O(number of nodes) in size no matter how much data flowed through the run,
+ * which is what makes checkpointing every step affordable.
+ */
+export interface Payload {
+    /** store id, e.g. 's3://bucket' or 'mem' */
+    store: string;
+    /** content address — also the key */
+    sha256: string;
+    /** bytes, for budgeting without fetching */
+    size: number;
+    /** first ~200 chars, for logs, UIs and debugging */
+    preview?: string;
+}
+export declare const PREVIEW_LEN = 200;
+export declare function hash(value: string): string;
+export declare function previewOf(value: string): string | undefined;
+export declare function isPayload(v: unknown): v is Payload;
+export interface PayloadStore {
+    readonly id: string;
+    put(value: string): Promise<Payload>;
+    get(p: Payload): Promise<string>;
+    /** one round trip for a whole projection */
+    getMany(ps: Payload[]): Promise<string[]>;
+}
+/**
+ * Fans a projection out over the stores it references and caches results.
+ * Because payloads are content-addressed, a growing trajectory re-projects its
+ * unchanged prefix with exact cache hits — repeated projection is cheap.
+ */
+export declare class PayloadResolver {
+    #private;
+    constructor(def: PayloadStore, maxCacheBytes?: number);
+    register(store: PayloadStore): this;
+    get defaultStore(): PayloadStore;
+    store(id: string): PayloadStore;
+    put(value: string, store?: PayloadStore): Promise<Payload>;
+    get(p: Payload): Promise<string>;
+    /**
+     * Batched resolution keyed by content address: duplicates collapse, and one
+     * store round trip covers a whole projection.
+     */
+    getMany(ps: Payload[]): Promise<Map<string, string>>;
+}
+/** Every payload reachable from an arbitrary JSON value, deduped by address. */
+export declare function collectPayloads(value: unknown, into?: Map<string, Payload>): Payload[];
+/**
+ * A run made whole: a state plus every blob it points at.
+ *
+ * On its own an `AgentState` is a graph of references — meaningless without
+ * the stores those references name, which may be a bucket the reader cannot
+ * reach or an in-memory store that died with its process. A bundle closes
+ * that gap, so it is the unit for anything crossing a machine, a process or a
+ * trust boundary: test fixtures, bug reports, archival, handing a run to
+ * another service.
+ *
+ * Plain JSON by design — `JSON.stringify` it, and `importRun` rehydrates it
+ * against whatever store the reader has. Note the two halves are indexed
+ * differently on purpose: `state` keeps full `Payload` refs (store id, size,
+ * preview) because it is still a working state, while `blobs` is keyed by
+ * content address alone, which both drops the origin store (an imported run
+ * belongs to its new one) and dedupes for free — content repeated across
+ * steps or fork branches is carried once.
+ */
+export interface RunBundle {
+    /** typically an `AgentState`, but any payload-bearing JSON works */
+    state: unknown;
+    /** sha256 → content; deduped by construction */
+    blobs: Record<string, string>;
+}
+/**
+ * Self-containment on demand: one portable artifact for tests, bug reports and
+ * archival. A fork tree needs no special handling — branch nodes live in the
+ * one trajectory, so their payloads are already reachable.
+ */
+export declare function exportRun(state: unknown, payloads: PayloadResolver): Promise<RunBundle>;
+/**
+ * Writes a bundle's blobs into the target store and rewrites every reference
+ * to point at it, so an imported run resolves locally.
+ *
+ * It re-`put`s rather than inserting under the bundle's keys, which keeps the
+ * contract at `PayloadStore` (any backend works, not just the in-memory one)
+ * and means nothing has to trust those keys: the store re-derives the address
+ * from the bytes, so a tampered blob lands under its true address and leaves a
+ * dangling reference that fails loudly instead of poisoning the store.
+ */
+export declare function importRun<T>(bundle: RunBundle, store: PayloadStore): Promise<T>;
+//# sourceMappingURL=payload.d.ts.map

package/dist/payload.js

@@ -0,0 +1,181 @@
+import { createHash } from 'node:crypto';
+export const PREVIEW_LEN = 200;
+export function hash(value) {
+    return createHash('sha256').update(value, 'utf8').digest('hex');
+}
+export function previewOf(value) {
+    return value ? value.slice(0, PREVIEW_LEN) : undefined;
+}
+export function isPayload(v) {
+    if (v === null || typeof v !== 'object') {
+        return false;
+    }
+    const p = v;
+    return (typeof p.store === 'string' && typeof p.sha256 === 'string' && typeof p.size === 'number');
+}
+/**
+ * Fans a projection out over the stores it references and caches results.
+ * Because payloads are content-addressed, a growing trajectory re-projects its
+ * unchanged prefix with exact cache hits — repeated projection is cheap.
+ */
+export class PayloadResolver {
+    #stores = new Map();
+    #cache = new Map();
+    #maxCacheBytes;
+    #cacheBytes = 0;
+    #default;
+    constructor(def, maxCacheBytes = 8 << 20) {
+        this.#default = def;
+        this.#maxCacheBytes = maxCacheBytes;
+        this.register(def);
+    }
+    register(store) {
+        this.#stores.set(store.id, store);
+        return this;
+    }
+    get defaultStore() {
+        return this.#default;
+    }
+    store(id) {
+        const s = this.#stores.get(id);
+        if (!s) {
+            throw new Error(`unknown payload store: ${id} (known: ${[...this.#stores.keys()]})`);
+        }
+        return s;
+    }
+    async put(value, store = this.#default) {
+        const p = await store.put(value);
+        this.#remember(p.sha256, value);
+        return p;
+    }
+    async get(p) {
+        const cached = this.#cache.get(p.sha256);
+        if (cached !== undefined) {
+            return cached;
+        }
+        const value = await this.store(p.store).get(p);
+        this.#remember(p.sha256, value);
+        return value;
+    }
+    /**
+     * Batched resolution keyed by content address: duplicates collapse, and one
+     * store round trip covers a whole projection.
+     */
+    async getMany(ps) {
+        const out = new Map();
+        const missing = new Map();
+        for (const p of ps) {
+            if (out.has(p.sha256)) {
+                continue;
+            }
+            const cached = this.#cache.get(p.sha256);
+            if (cached !== undefined) {
+                out.set(p.sha256, cached);
+                continue;
+            }
+            const byStore = missing.get(p.store) ?? [];
+            if (!byStore.some((q) => q.sha256 === p.sha256)) {
+                byStore.push(p);
+            }
+            missing.set(p.store, byStore);
+        }
+        await Promise.all([...missing].map(async ([storeId, refs]) => {
+            const values = await this.store(storeId).getMany(refs);
+            refs.forEach((ref, i) => {
+                const value = values[i] ?? '';
+                out.set(ref.sha256, value);
+                this.#remember(ref.sha256, value);
+            });
+        }));
+        return out;
+    }
+    #remember(sha256, value) {
+        if (this.#cache.has(sha256)) {
+            return;
+        }
+        this.#cache.set(sha256, value);
+        this.#cacheBytes += value.length;
+        // Insertion-ordered Map == FIFO eviction, good enough for a prefix-heavy
+        // access pattern where the hot entries are re-inserted on every miss.
+        for (const [key, val] of this.#cache) {
+            if (this.#cacheBytes <= this.#maxCacheBytes) {
+                break;
+            }
+            if (key === sha256) {
+                continue;
+            }
+            this.#cache.delete(key);
+            this.#cacheBytes -= val.length;
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Deep payload traversal — export/import and GC accounting
+// ---------------------------------------------------------------------------
+/** Every payload reachable from an arbitrary JSON value, deduped by address. */
+export function collectPayloads(value, into = new Map()) {
+    if (Array.isArray(value)) {
+        for (const v of value) {
+            collectPayloads(v, into);
+        }
+    }
+    else if (value !== null && typeof value === 'object') {
+        if (isPayload(value)) {
+            into.set(value.sha256, value);
+            return [...into.values()];
+        }
+        for (const v of Object.values(value)) {
+            collectPayloads(v, into);
+        }
+    }
+    return [...into.values()];
+}
+/**
+ * Self-containment on demand: one portable artifact for tests, bug reports and
+ * archival. A fork tree needs no special handling — branch nodes live in the
+ * one trajectory, so their payloads are already reachable.
+ */
+export async function exportRun(state, payloads) {
+    const refs = collectPayloads(state);
+    const values = await payloads.getMany(refs);
+    const blobs = {};
+    for (const p of refs) {
+        blobs[p.sha256] = values.get(p.sha256) ?? '';
+    }
+    return { state, blobs };
+}
+/** Bounded fan-out: a bundle can hold thousands of blobs, a store has limits. */
+const IMPORT_CONCURRENCY = 32;
+/**
+ * Writes a bundle's blobs into the target store and rewrites every reference
+ * to point at it, so an imported run resolves locally.
+ *
+ * It re-`put`s rather than inserting under the bundle's keys, which keeps the
+ * contract at `PayloadStore` (any backend works, not just the in-memory one)
+ * and means nothing has to trust those keys: the store re-derives the address
+ * from the bytes, so a tampered blob lands under its true address and leaves a
+ * dangling reference that fails loudly instead of poisoning the store.
+ */
+export async function importRun(bundle, store) {
+    const values = Object.values(bundle.blobs);
+    for (let i = 0; i < values.length; i += IMPORT_CONCURRENCY) {
+        await Promise.all(values.slice(i, i + IMPORT_CONCURRENCY).map((v) => store.put(v)));
+    }
+    return rewriteStore(structuredClone(bundle.state), store.id);
+}
+function rewriteStore(value, storeId) {
+    if (Array.isArray(value)) {
+        value.forEach((v) => rewriteStore(v, storeId));
+    }
+    else if (value !== null && typeof value === 'object') {
+        if (isPayload(value)) {
+            value.store = storeId;
+            return value;
+        }
+        for (const v of Object.values(value)) {
+            rewriteStore(v, storeId);
+        }
+    }
+    return value;
+}
+//# sourceMappingURL=payload.js.map