@verax-ai/proxy 0.1.1 → 0.1.3

package/dist/ledger-manifest.d.ts

@@ -0,0 +1,88 @@
+export declare const LEGACY_PIECE_ID = "legacy";
+export declare const DEFAULT_PIECE_MAX_ROWS = 50000;
+export declare const DEFAULT_PIECE_MAX_BYTES: number;
+export declare const MANIFEST_NAME = "ledger-manifest.json";
+export declare const INDEX_NAME = "index.jsonl";
+export type LedgerPieceRow = {
+    id: string;
+    decisions: string;
+    effects: string;
+    inputs: string;
+    n: number;
+    effectN: number;
+    firstMs: number | null;
+    lastMs: number | null;
+    lastHash: string | null;
+    closed: boolean;
+    closedAtMs?: number;
+};
+export type LedgerManifest = {
+    version: 1;
+    pieces: LedgerPieceRow[];
+    countedAtMs: number[];
+};
+export type LedgerIndexLine = {
+    ref: string;
+    tenantRef: string;
+    piece: string;
+    ts: number;
+    kind: string;
+    requestHash: string;
+    resolves: string | null;
+    hasEffect: boolean;
+    reasonCode: string;
+    subject: string;
+    policyHash: string;
+};
+export type PieceFiles = {
+    id: string;
+    decisions: string;
+    effects: string;
+    inputs: string;
+    copyDecisions: string;
+    copyEffects: string;
+    closed: boolean;
+    firstMs: number | null;
+    lastMs: number | null;
+    n: number;
+    effectN: number;
+};
+export declare function manifestPath(dir: string): string;
+export declare function indexPath(dir: string): string;
+export declare function legacyPieceRow(): LedgerPieceRow;
+export declare function newPieceRow(id: string): LedgerPieceRow;
+export declare function copyRel(pieceId: string, name: "decisions.jsonl" | "effects.jsonl"): string;
+export declare function nextPieceId(existingIds: string[], atMs: number): string;
+/**
+ * Null only when there is no manifest (the flat legacy layout). A manifest
+ * that is there but cannot be read or parsed throws: opening as a single
+ * legacy piece would chain new rows onto a closed file and hide every piece
+ * the manifest named.
+ */
+export declare function readLedgerManifest(dir: string): LedgerManifest | null;
+export declare function writeLedgerManifest(dir: string, manifest: LedgerManifest): void;
+export declare function listPieceFiles(dir: string): PieceFiles[];
+export declare function pieceOverlaps(piece: {
+    firstMs: number | null;
+    lastMs: number | null;
+    n: number;
+    closed: boolean;
+}, fromMs: number, toMs: number): boolean;
+export declare function parseIndexText(text: string): LedgerIndexLine[];
+export declare function readLedgerIndex(dir: string): LedgerIndexLine[];
+/**
+ * Decisions the index knows, by distinct ref. An effect adds a second line
+ * for its decision's ref, so a line total would overstate what the index
+ * holds and let a short index pass as complete.
+ */
+export declare function decisionIndexCount(lines: readonly LedgerIndexLine[]): number;
+/**
+ * What the on-disk index names, or null when the file is missing. Doctor
+ * compares this to the pieces; a short index is rebuilt on the next open
+ * but a running body does not see the missing refs until then.
+ */
+export declare function indexCoverage(dir: string): {
+    refs: number;
+    pieces: Set<string>;
+} | null;
+export declare function manifestExists(dir: string): boolean;

package/dist/ledger-manifest.js

@@ -0,0 +1,225 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { writeFileAtomic } from "./atomic-write.js";
+export const LEGACY_PIECE_ID = "legacy";
+export const DEFAULT_PIECE_MAX_ROWS = 50_000;
+export const DEFAULT_PIECE_MAX_BYTES = 200 * 1024 * 1024;
+export const MANIFEST_NAME = "ledger-manifest.json";
+export const INDEX_NAME = "index.jsonl";
+export function manifestPath(dir) {
+    return join(dir, MANIFEST_NAME);
+}
+export function indexPath(dir) {
+    return join(dir, INDEX_NAME);
+}
+export function legacyPieceRow() {
+    return {
+        id: LEGACY_PIECE_ID,
+        decisions: "decisions.jsonl",
+        effects: "effects.jsonl",
+        inputs: "inputs.jsonl",
+        n: 0,
+        effectN: 0,
+        firstMs: null,
+        lastMs: null,
+        lastHash: null,
+        closed: false,
+    };
+}
+export function newPieceRow(id) {
+    return {
+        id,
+        decisions: `pieces/${id}/decisions.jsonl`,
+        effects: `pieces/${id}/effects.jsonl`,
+        inputs: `pieces/${id}/inputs.jsonl`,
+        n: 0,
+        effectN: 0,
+        firstMs: null,
+        lastMs: null,
+        lastHash: null,
+        closed: false,
+    };
+}
+export function copyRel(pieceId, name) {
+    if (pieceId === LEGACY_PIECE_ID)
+        return `evidence-copy/${name}`;
+    return `evidence-copy/pieces/${pieceId}/${name}`;
+}
+export function nextPieceId(existingIds, atMs) {
+    const d = new Date(atMs);
+    const ym = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`;
+    const letters = "abcdefghijklmnopqrstuvwxyz";
+    for (let i = 0; i < letters.length; i += 1) {
+        const id = `${ym}-${letters[i]}`;
+        if (!existingIds.includes(id))
+            return id;
+    }
+    let n = letters.length;
+    for (;;) {
+        const id = `${ym}-z${n}`;
+        if (!existingIds.includes(id))
+            return id;
+        n += 1;
+    }
+}
+/**
+ * Null only when there is no manifest (the flat legacy layout). A manifest
+ * that is there but cannot be read or parsed throws: opening as a single
+ * legacy piece would chain new rows onto a closed file and hide every piece
+ * the manifest named.
+ */
+export function readLedgerManifest(dir) {
+    let text;
+    try {
+        text = readFileSync(manifestPath(dir), "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return null;
+        throw new Error("ledger-manifest-unreadable: " + err.message);
+    }
+    let raw;
+    try {
+        raw = JSON.parse(text);
+    }
+    catch (err) {
+        throw new Error("ledger-manifest-corrupt: " + err.message);
+    }
+    if (!raw || raw.version !== 1 || !Array.isArray(raw.pieces) || raw.pieces.length === 0) {
+        throw new Error("ledger-manifest-invalid: version 1 with at least one piece expected");
+    }
+    const pieces = raw.pieces.map(asPieceRow);
+    if (pieces.some((p) => p === null))
+        throw new Error("ledger-manifest-invalid: piece row missing a field");
+    return {
+        version: 1,
+        pieces: pieces,
+        countedAtMs: Array.isArray(raw.countedAtMs)
+            ? raw.countedAtMs.filter((n) => typeof n === "number")
+            : [],
+    };
+}
+function asPieceRow(raw) {
+    if (!raw || typeof raw !== "object")
+        return null;
+    const r = raw;
+    if (typeof r.id !== "string" || typeof r.decisions !== "string")
+        return null;
+    if (typeof r.effects !== "string" || typeof r.inputs !== "string")
+        return null;
+    if (typeof r.n !== "number" || typeof r.effectN !== "number")
+        return null;
+    if (typeof r.closed !== "boolean")
+        return null;
+    return {
+        id: r.id,
+        decisions: r.decisions,
+        effects: r.effects,
+        inputs: r.inputs,
+        n: r.n,
+        effectN: r.effectN,
+        firstMs: typeof r.firstMs === "number" ? r.firstMs : null,
+        lastMs: typeof r.lastMs === "number" ? r.lastMs : null,
+        lastHash: typeof r.lastHash === "string" ? r.lastHash : null,
+        closed: r.closed,
+        ...(typeof r.closedAtMs === "number" ? { closedAtMs: r.closedAtMs } : {}),
+    };
+}
+export function writeLedgerManifest(dir, manifest) {
+    writeFileAtomic(manifestPath(dir), `${JSON.stringify(manifest)}\n`);
+}
+export function listPieceFiles(dir) {
+    const manifest = readLedgerManifest(dir);
+    const pieces = manifest?.pieces ?? [legacyPieceRow()];
+    return pieces.map((p) => ({
+        id: p.id,
+        decisions: join(dir, p.decisions),
+        effects: join(dir, p.effects),
+        inputs: join(dir, p.inputs),
+        copyDecisions: join(dir, copyRel(p.id, "decisions.jsonl")),
+        copyEffects: join(dir, copyRel(p.id, "effects.jsonl")),
+        closed: p.closed,
+        firstMs: p.firstMs,
+        lastMs: p.lastMs,
+        n: p.n,
+        effectN: p.effectN,
+    }));
+}
+export function pieceOverlaps(piece, fromMs, toMs) {
+    if (piece.firstMs === null || piece.lastMs === null)
+        return piece.n > 0 || !piece.closed;
+    return piece.lastMs >= fromMs && piece.firstMs < toMs;
+}
+export function parseIndexText(text) {
+    const rows = [];
+    for (const line of text.split("\n")) {
+        if (line === "")
+            continue;
+        // The index is appended without fsync, so a crash can leave its last
+        // line half written. Such a line is skipped; the ref count below then
+        // falls short of the ledger's and the index is rebuilt from the pieces.
+        let raw;
+        try {
+            raw = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (typeof raw.ref !== "string" || typeof raw.piece !== "string")
+            continue;
+        rows.push({
+            ref: raw.ref,
+            tenantRef: typeof raw.tenantRef === "string" ? raw.tenantRef : "",
+            piece: raw.piece,
+            ts: typeof raw.ts === "number" ? raw.ts : 0,
+            kind: typeof raw.kind === "string" ? raw.kind : "",
+            requestHash: typeof raw.requestHash === "string" ? raw.requestHash : "",
+            resolves: typeof raw.resolves === "string" ? raw.resolves : null,
+            hasEffect: raw.hasEffect === true,
+            reasonCode: typeof raw.reasonCode === "string" ? raw.reasonCode : "",
+            subject: typeof raw.subject === "string" ? raw.subject : "",
+            policyHash: typeof raw.policyHash === "string" ? raw.policyHash : "",
+        });
+    }
+    return rows;
+}
+export function readLedgerIndex(dir) {
+    try {
+        return parseIndexText(readFileSync(indexPath(dir), "utf8"));
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return [];
+        throw err;
+    }
+}
+/**
+ * Decisions the index knows, by distinct ref. An effect adds a second line
+ * for its decision's ref, so a line total would overstate what the index
+ * holds and let a short index pass as complete.
+ */
+export function decisionIndexCount(lines) {
+    const refs = new Set();
+    for (const l of lines) {
+        if (l.kind === "allow" || l.kind === "deny" || l.kind === "defer")
+            refs.add(l.ref);
+    }
+    return refs.size;
+}
+/**
+ * What the on-disk index names, or null when the file is missing. Doctor
+ * compares this to the pieces; a short index is rebuilt on the next open
+ * but a running body does not see the missing refs until then.
+ */
+export function indexCoverage(dir) {
+    if (!existsSync(indexPath(dir)))
+        return null;
+    const lines = readLedgerIndex(dir);
+    const pieces = new Set();
+    for (const line of lines)
+        pieces.add(line.piece);
+    return { refs: decisionIndexCount(lines), pieces };
+}
+export function manifestExists(dir) {
+    return existsSync(manifestPath(dir));
+}

package/dist/ledger.d.ts

@@ -7,9 +7,24 @@ export declare const ledgerFs: {
     open: typeof open;
 };
 import { type EffectRow, type SignedEffectExtract } from "@cedulon/effect-extract";
-import type { EffectSigner, ExtractWindow, Ledger, LedgerEffect, WitnessClass } from "./types.ts";
+import { type TailVisit } from "./jsonl-tail.ts";
+import type { DecisionInputs, EffectSigner, ExtractWindow, Ledger, LedgerEffect, WitnessClass } from "./types.ts";
 import type { DecisionKind, SignedDecisionRecord } from "@cedulon/core";
 export type PermissionCheck = "owner-only" | "not checked on this platform";
+/**
+ * A ledger file read from its end through the same seam the reads above use,
+ * so a test can count what a window costs.
+ */
+export declare function readLedgerTail<T>(path: string, visit: (row: T) => TailVisit, chunkBytesOrOpts?: number | {
+    chunkBytes?: number;
+    endOffset?: number;
+}): Promise<T[]>;
+/**
+ * How far past a window's lower edge the tail reader keeps looking before it
+ * stops. Rows are in file order; their stamps are the clock's, and a clock
+ * can be set back. A row stamped a minute before its neighbour is still seen.
+ */
+export declare const WINDOW_SLACK_MS = 60000;
 /** Write then fsync so a crash cannot drop a committed line. */
 export declare function appendDurable(path: string, line: string): Promise<void>;
 export type RemoteWitnessSign = (row: EffectRow, resultHash: string | undefined) => Promise<Pick<LedgerEffect, "receipt" | "attestation" | "witnessClass"> | null>;
@@ -55,14 +70,37 @@ export declare class MemoryLedger implements Ledger {
     lastDecisionHash(): Promise<string | null>;
     exportExtract(window: ExtractWindow, signer: EffectSigner): Promise<SignedEffectExtract>;
 }
+export type FileLedgerOpts = {
+    pieceMaxRows?: number;
+    pieceMaxBytes?: number;
+    onPieceClose?: (window: {
+        startMs: number;
+        endMs: number;
+        pieceId: string;
+    }) => Promise<void>;
+};
+export type LedgerCounts = {
+    decisions: number;
+    effects: number;
+    lastDecisionMs: number | null;
+    activeDecisions: number;
+    pieces: number;
+};
 export declare class FileLedger implements Ledger {
     readonly permissionCheck: PermissionCheck;
     readonly dir: string;
     effectSigner?: EffectSigner;
     /** Optional signer in another process. Null means stay `self`. */
     remoteWitness?: RemoteWitnessSign;
-    private readonly decisionsPath;
-    private readonly effectsPath;
+    /** Asked after a piece closes. A missing or failing hook does not block the close. */
+    onPieceClose?: FileLedgerOpts["onPieceClose"];
+    pieceMaxRows: number;
+    pieceMaxBytes: number;
+    private decisionsPath;
+    private effectsPath;
+    private inputsPath;
+    private copyDecisionsPath;
+    private copyEffectsPath;
     private readonly lockPath;
     private readonly q;
     private readonly token;
@@ -75,17 +113,37 @@ export declare class FileLedger implements Ledger {
     private readonly reauthByHash;
     private readonly countedAt;
     countsReadable: boolean;
-    private readonly copyDir;
     private readonly heartbeatPath;
     private decisionCount;
     private effectCount;
-    constructor(dir: string);
+    private lastDecisionMs;
+    private pieces;
+    /** Inputs rows appended through this instance, until their decision lands. */
+    private readonly recentInputs;
+    private activeId;
+    private activeDecisionN;
+    private activeEffectN;
+    constructor(dir: string, opts?: FileLedgerOpts);
+    noteInputs(ref: string, inputs: DecisionInputs): void;
+    inputsPaths(): {
+        active: string;
+        all: string[];
+    };
     /** After lock handoff a new instance reloads; the lost owner cannot append. */
     private loadCaches;
-    /** Pair inputs.jsonl with decisions so a restart keeps per-tenant `_ref` keys. */
-    private loadTenantRefs;
-    /** Single pass over inputs.jsonl. FileInputsLog itself stays cache-less. */
-    private loadResolvedBy;
+    private applyActivePaths;
+    private activePiece;
+    private applyIndexLine;
+    private pruneCounted;
+    private rebuildIndex;
+    private writeIndexForDecision;
+    private writeIndexHasEffect;
+    private noteActiveStamp;
+    private persistManifest;
+    private fsyncPath;
+    private ensurePieceFiles;
+    private closeActivePiece;
+    private maybeRotate;
     private lockBody;
     private ownsLock;
     /** Lock file on disk, not a process counter. */
@@ -97,6 +155,27 @@ export declare class FileLedger implements Ledger {
     close(): void;
     appendDecision(signed: SignedDecisionRecord): Promise<void>;
     appendDecisionChained(build: (prevRecordHash: string | null) => SignedDecisionRecord): Promise<void>;
+    /**
+     * What /healthz reports, from memory: the counts are kept on load and on
+     * every append. Rereading both files to count lines cost a second per call
+     * on a 100k-decision ledger (measured 17 Sep 2026), and the panel asks
+     * every five seconds. Null when the decision file could not be parsed on
+     * load; then the caller reads for itself and fails the way it always did.
+     */
+    counts(): LedgerCounts | null;
+    /**
+     * Decisions stamped in [fromMs, toMs), read from the end of each overlapping
+     * piece so the cost is the window's, not the ledger's. With a limit, the
+     * newest rows of the window come back and `more` says the window went on,
+     * including in a closed piece.
+     */
+    decisionsWindow(fromMs: number, toMs: number, limit?: number): Promise<{
+        rows: SignedDecisionRecord[];
+        more: boolean;
+        piecesTouched: string[];
+    }>;
+    /** Effects stamped in [fromMs, toMs), read from the end of each overlapping piece. */
+    effectsWindow(fromMs: number, toMs: number): Promise<LedgerEffect[]>;
     appendEffect(row: EffectRow, witnessClass?: WitnessClass, resultHash?: string): Promise<void>;
     private appendEffectUnlocked;
     decisions(): Promise<SignedDecisionRecord[]>;