@verax-ai/proxy 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approvals.d.ts +30 -1
- package/dist/approvals.js +64 -3
- package/dist/atomic-write.d.ts +1 -0
- package/dist/atomic-write.js +65 -0
- package/dist/in-flight-log.d.ts +12 -0
- package/dist/in-flight-log.js +65 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +3 -2
- package/dist/inputs.d.ts +7 -1
- package/dist/inputs.js +35 -18
- package/dist/jsonl-tail.d.ts +26 -0
- package/dist/jsonl-tail.js +218 -0
- package/dist/ledger-manifest.d.ts +88 -0
- package/dist/ledger-manifest.js +225 -0
- package/dist/ledger.d.ts +88 -9
- package/dist/ledger.js +541 -86
- package/dist/policy.js +9 -1
- package/dist/proxy.js +266 -166
- package/dist/serial-queue.d.ts +5 -0
- package/dist/serial-queue.js +9 -0
- package/dist/types.d.ts +3 -0
- package/package.json +1 -1
package/dist/approvals.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ export declare const approvalFs: {
|
|
|
6
6
|
export declare const drainMetrics: {
|
|
7
7
|
renameBusy: number;
|
|
8
8
|
};
|
|
9
|
-
import type { ApprovalChannel, InputsLog, Ledger, RecordSigner } from "./types.ts";
|
|
9
|
+
import type { ApprovalChannel, InputsLog, Ledger, Policy, RecordSigner } from "./types.ts";
|
|
10
10
|
export type ApprovalRow = {
|
|
11
11
|
ref: string;
|
|
12
12
|
requestHash: string;
|
|
@@ -68,7 +68,25 @@ export type ApproveResult = {
|
|
|
68
68
|
} | {
|
|
69
69
|
ok: false;
|
|
70
70
|
reason: string;
|
|
71
|
+
allowRef?: string;
|
|
71
72
|
};
|
|
73
|
+
export declare function spentTodayMinorOf(rows: Array<{
|
|
74
|
+
subject: string;
|
|
75
|
+
status: string;
|
|
76
|
+
createdAtMs?: number;
|
|
77
|
+
expiresAtMs: number;
|
|
78
|
+
args: Record<string, unknown>;
|
|
79
|
+
}>, nowMs: number, currency: string, approvalTtlMs: number): number;
|
|
80
|
+
export declare function createApprovalBudgetGuard(opts: {
|
|
81
|
+
policy: Policy;
|
|
82
|
+
approvals: ApprovalsLog;
|
|
83
|
+
now: () => number;
|
|
84
|
+
}): (snap: ApprovalRow) => Promise<{
|
|
85
|
+
ok: true;
|
|
86
|
+
} | {
|
|
87
|
+
ok: false;
|
|
88
|
+
reason: "budget-exceeded";
|
|
89
|
+
}>;
|
|
72
90
|
export declare function approvePending(opts: {
|
|
73
91
|
ledger: Ledger;
|
|
74
92
|
recordSigner: RecordSigner;
|
|
@@ -81,6 +99,17 @@ export declare function approvePending(opts: {
|
|
|
81
99
|
policyHash: string;
|
|
82
100
|
approvals: ApprovalsLog;
|
|
83
101
|
inputsLog?: InputsLog;
|
|
102
|
+
budgetGuard?: (snap: ApprovalRow) => {
|
|
103
|
+
ok: true;
|
|
104
|
+
} | {
|
|
105
|
+
ok: false;
|
|
106
|
+
reason: "budget-exceeded";
|
|
107
|
+
} | Promise<{
|
|
108
|
+
ok: true;
|
|
109
|
+
} | {
|
|
110
|
+
ok: false;
|
|
111
|
+
reason: "budget-exceeded";
|
|
112
|
+
}>;
|
|
84
113
|
}): Promise<ApproveResult>;
|
|
85
114
|
export type ApprovalCommand = {
|
|
86
115
|
ref: string;
|
package/dist/approvals.js
CHANGED
|
@@ -9,6 +9,7 @@ import { signDecisionRecord } from "@cedulon/core";
|
|
|
9
9
|
import { appendDurable, lookupDecisionByRef, lookupResolvedBy, noteResolution } from "./ledger.js";
|
|
10
10
|
import { effectDescriptor, sha256Canonical } from "./hash.js";
|
|
11
11
|
import { inputsLogFor } from "./inputs.js";
|
|
12
|
+
import { SerialQueue } from "./serial-queue.js";
|
|
12
13
|
function lastByRef(rows) {
|
|
13
14
|
const map = new Map();
|
|
14
15
|
for (const row of rows)
|
|
@@ -105,6 +106,56 @@ export function loadApprovalsFromDir(dir) {
|
|
|
105
106
|
throw err;
|
|
106
107
|
}
|
|
107
108
|
}
|
|
109
|
+
const APPROVAL_LOCK = Symbol.for("verax.approvalLock");
|
|
110
|
+
function approvalLockFor(ledger) {
|
|
111
|
+
const bag = ledger;
|
|
112
|
+
if (!bag[APPROVAL_LOCK])
|
|
113
|
+
bag[APPROVAL_LOCK] = new SerialQueue();
|
|
114
|
+
return bag[APPROVAL_LOCK];
|
|
115
|
+
}
|
|
116
|
+
export function spentTodayMinorOf(rows, nowMs, currency, approvalTtlMs) {
|
|
117
|
+
const d = new Date(nowMs);
|
|
118
|
+
const start = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate());
|
|
119
|
+
const end = start + 86_400_000;
|
|
120
|
+
let sum = 0;
|
|
121
|
+
for (const row of rows) {
|
|
122
|
+
if (row.subject !== "spend")
|
|
123
|
+
continue;
|
|
124
|
+
if (row.status !== "pending" && row.status !== "approved")
|
|
125
|
+
continue;
|
|
126
|
+
const created = row.createdAtMs ?? row.expiresAtMs - approvalTtlMs;
|
|
127
|
+
if (created < start || created >= end)
|
|
128
|
+
continue;
|
|
129
|
+
if (row.args.currency !== currency)
|
|
130
|
+
continue;
|
|
131
|
+
const amt = row.args.amountMinor;
|
|
132
|
+
if (typeof amt === "number")
|
|
133
|
+
sum += amt;
|
|
134
|
+
}
|
|
135
|
+
return sum;
|
|
136
|
+
}
|
|
137
|
+
export function createApprovalBudgetGuard(opts) {
|
|
138
|
+
return async (snap) => {
|
|
139
|
+
if (snap.subject !== "spend")
|
|
140
|
+
return { ok: true };
|
|
141
|
+
const dailyMax = opts.policy.rule(snap.ruleId)?.spend?.dailyMaxMinor;
|
|
142
|
+
if (dailyMax === undefined)
|
|
143
|
+
return { ok: true };
|
|
144
|
+
const currency = typeof snap.args.currency === "string" ? snap.args.currency : "";
|
|
145
|
+
const amount = typeof snap.args.amountMinor === "number" ? snap.args.amountMinor : 0;
|
|
146
|
+
// Sibling pending rows are not authorized yet. Counting them here would
|
|
147
|
+
// deadlock two 100-unit pendings against a later 150 cap; approved spend
|
|
148
|
+
// plus this amount is what the operator is about to commit.
|
|
149
|
+
const others = (await opts.approvals.listAll()).filter((row) => row.ref !== snap.ref && row.status === "approved");
|
|
150
|
+
const spent = spentTodayMinorOf(others, opts.now(), currency, opts.policy.approvalTtlMs);
|
|
151
|
+
if (spent + amount > dailyMax)
|
|
152
|
+
return { ok: false, reason: "budget-exceeded" };
|
|
153
|
+
return { ok: true };
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
function alreadyResolved(allowRef) {
|
|
157
|
+
return { ok: false, reason: "already-resolved", ...(allowRef ? { allowRef } : {}) };
|
|
158
|
+
}
|
|
108
159
|
async function hasResolves(inputsLog, ledger, deferRef) {
|
|
109
160
|
const indexed = ledger;
|
|
110
161
|
if (typeof indexed.lookupResolvedBy === "function") {
|
|
@@ -120,20 +171,25 @@ async function hasResolves(inputsLog, ledger, deferRef) {
|
|
|
120
171
|
return false;
|
|
121
172
|
}
|
|
122
173
|
export async function approvePending(opts) {
|
|
174
|
+
return approvalLockFor(opts.ledger).enqueue(() => approvePendingUnlocked(opts));
|
|
175
|
+
}
|
|
176
|
+
async function approvePendingUnlocked(opts) {
|
|
123
177
|
const inputsLog = opts.inputsLog ?? inputsLogFor(opts.ledger);
|
|
124
178
|
const defer = await lookupDecisionByRef(opts.ledger, opts.ref);
|
|
125
179
|
if (!defer || defer.decision !== "defer")
|
|
126
180
|
return { ok: false, reason: "unknown-ref" };
|
|
127
181
|
const snap = await opts.approvals.get(opts.ref);
|
|
128
|
-
if (snap && snap.status !== "pending")
|
|
129
|
-
|
|
182
|
+
if (snap && snap.status !== "pending") {
|
|
183
|
+
const hit = lookupResolvedBy(opts.ledger, opts.ref);
|
|
184
|
+
return alreadyResolved(snap.allowRef ?? (hit?.kind === "allow" ? hit.ref : undefined));
|
|
185
|
+
}
|
|
130
186
|
const resolved = lookupResolvedBy(opts.ledger, opts.ref);
|
|
131
187
|
if (resolved || (await hasResolves(inputsLog, opts.ledger, opts.ref))) {
|
|
132
188
|
const hit = resolved ?? lookupResolvedBy(opts.ledger, opts.ref);
|
|
133
189
|
if (snap?.status === "pending" && hit) {
|
|
134
190
|
await opts.approvals.updateStatus(opts.ref, hit.kind === "allow" ? "approved" : "expired", hit.kind === "allow" ? { allowRef: hit.ref } : undefined);
|
|
135
191
|
}
|
|
136
|
-
return
|
|
192
|
+
return alreadyResolved(hit?.kind === "allow" ? hit.ref : snap?.allowRef);
|
|
137
193
|
}
|
|
138
194
|
if (!snap)
|
|
139
195
|
return { ok: false, reason: "snapshot-missing" };
|
|
@@ -172,6 +228,11 @@ export async function approvePending(opts) {
|
|
|
172
228
|
await opts.approvals.updateStatus(opts.ref, "expired");
|
|
173
229
|
return { ok: false, reason: "expired" };
|
|
174
230
|
}
|
|
231
|
+
if (opts.budgetGuard) {
|
|
232
|
+
const guarded = await opts.budgetGuard(snap);
|
|
233
|
+
if (!guarded.ok)
|
|
234
|
+
return guarded;
|
|
235
|
+
}
|
|
175
236
|
const allowRef = opts.nonce();
|
|
176
237
|
const prior = await inputsLog.get(opts.ref);
|
|
177
238
|
const inputs = {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function writeFileAtomic(path: string, text: string): void;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { chmodSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
/**
|
|
5
|
+
* Replace a file's contents in one step, so a reader sees the old bytes or the
|
|
6
|
+
* new ones and never a half-written file.
|
|
7
|
+
*
|
|
8
|
+
* Writing in place is not that: measured on 11 Sep 2026, a reader polling this
|
|
9
|
+
* path every 20 ms while the file was rewritten 60 times got 12 unparseable
|
|
10
|
+
* reads. The listen file is read by another process — that is its whole job —
|
|
11
|
+
* and a torn read becomes `null`, which the caller cannot tell from "nothing
|
|
12
|
+
* is running".
|
|
13
|
+
*
|
|
14
|
+
* The rename needs the retry. On Windows it fails with EPERM while a reader
|
|
15
|
+
* holds the destination open, and in the same measurement 26 of 60 plain
|
|
16
|
+
* renames failed that way. Retrying briefly lost none of them and still let no
|
|
17
|
+
* torn read through.
|
|
18
|
+
*
|
|
19
|
+
* The temporary name carries this process and a random suffix. A fixed `.tmp`
|
|
20
|
+
* next to the destination is what two writers, or a crashed writer, collide
|
|
21
|
+
* on.
|
|
22
|
+
*/
|
|
23
|
+
const napper = new Int32Array(new SharedArrayBuffer(4));
|
|
24
|
+
/**
|
|
25
|
+
* Sleep without spinning. An earlier retry burned the processor between
|
|
26
|
+
* attempts, which is the worst thing to do while waiting for another process
|
|
27
|
+
* to let go of a file: under the unit suite's own load the spin starved the
|
|
28
|
+
* reader it was waiting on, the retries ran out in 200 ms, and the write
|
|
29
|
+
* threw EPERM. Atomics.wait yields the core instead.
|
|
30
|
+
*/
|
|
31
|
+
function napSync(ms) {
|
|
32
|
+
Atomics.wait(napper, 0, 0, ms);
|
|
33
|
+
}
|
|
34
|
+
export function writeFileAtomic(path, text) {
|
|
35
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
36
|
+
const tmp = `${path}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`;
|
|
37
|
+
writeFileSync(tmp, text, { encoding: "utf8", mode: 0o600 });
|
|
38
|
+
try {
|
|
39
|
+
for (let attempt = 0;; attempt += 1) {
|
|
40
|
+
try {
|
|
41
|
+
renameSync(tmp, path);
|
|
42
|
+
chmodSync(path, 0o600);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
catch (err) {
|
|
46
|
+
const code = err.code;
|
|
47
|
+
// Two seconds of patience. A reader holds this file for microseconds
|
|
48
|
+
// when the machine is idle; the budget is for the machine that is not.
|
|
49
|
+
if (attempt >= 100 || (code !== "EPERM" && code !== "EACCES" && code !== "EBUSY")) {
|
|
50
|
+
throw err;
|
|
51
|
+
}
|
|
52
|
+
napSync(20);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
try {
|
|
58
|
+
unlinkSync(tmp);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// rename may already have moved it
|
|
62
|
+
}
|
|
63
|
+
throw err;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Called before `inner` runs. A failure here must not block the call.
|
|
3
|
+
*
|
|
4
|
+
* The stamp is wall-clock time and not the proxy's injected `now`: a mark is
|
|
5
|
+
* not a ledger row, and taking a tick from a deterministic clock would shift
|
|
6
|
+
* every timestamp recorded after it — the golden ledger caught exactly that.
|
|
7
|
+
*/
|
|
8
|
+
export declare function markStarted(stateDir: string, ref: string, subject: string): void;
|
|
9
|
+
/** Called once the effect row is written, on success and on failure alike. */
|
|
10
|
+
export declare function markEnded(stateDir: string, ref: string): void;
|
|
11
|
+
/** True when a run started and no end was recorded: the outcome is unknown. */
|
|
12
|
+
export declare function startedWithoutEnd(stateDir: string, ref: string): boolean;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable in-flight marks: "this ref started running".
|
|
3
|
+
*
|
|
4
|
+
* The in-memory registry that stops a second run dies with the process. On the
|
|
5
|
+
* approved path an `allow` does not mean the tool ran — it means the operator
|
|
6
|
+
* said yes — so a restarted body cannot tell "approved, never started" from
|
|
7
|
+
* "approved, started, crashed before the effect". Without that distinction it
|
|
8
|
+
* has to either run everything twice or run nothing at all.
|
|
9
|
+
*
|
|
10
|
+
* A mark is one empty file per ref, created before `inner` and removed after
|
|
11
|
+
* the effect is written. One file per ref rather than a shared log because two
|
|
12
|
+
* refs may start and finish at the same moment: file create and unlink need no
|
|
13
|
+
* lock, a shared append-only log would need a compaction pass and would race.
|
|
14
|
+
*
|
|
15
|
+
* A mark left behind by a crash is exactly the signal wanted: it outlives the
|
|
16
|
+
* process, and the retry that finds it answers `outcome-unknown` instead of
|
|
17
|
+
* running the tool a second time.
|
|
18
|
+
*/
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
const KLASOR = "in-flight";
|
|
23
|
+
/** Ref may carry a tenant prefix and any path character; the name never does. */
|
|
24
|
+
function markPath(stateDir, ref) {
|
|
25
|
+
const ad = createHash("sha256").update(ref, "utf8").digest("hex").slice(0, 32);
|
|
26
|
+
return join(stateDir, KLASOR, `${ad}.start`);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Called before `inner` runs. A failure here must not block the call.
|
|
30
|
+
*
|
|
31
|
+
* The stamp is wall-clock time and not the proxy's injected `now`: a mark is
|
|
32
|
+
* not a ledger row, and taking a tick from a deterministic clock would shift
|
|
33
|
+
* every timestamp recorded after it — the golden ledger caught exactly that.
|
|
34
|
+
*/
|
|
35
|
+
export function markStarted(stateDir, ref, subject) {
|
|
36
|
+
try {
|
|
37
|
+
mkdirSync(join(stateDir, KLASOR), { recursive: true, mode: 0o700 });
|
|
38
|
+
// The body is for a human reading the directory after a crash; nothing parses it.
|
|
39
|
+
writeFileSync(markPath(stateDir, ref), `${JSON.stringify({ ref, subject, atMs: Date.now() })}\n`, {
|
|
40
|
+
encoding: "utf8",
|
|
41
|
+
mode: 0o600,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// A missing mark degrades to today's behaviour, it does not deny the call.
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/** Called once the effect row is written, on success and on failure alike. */
|
|
49
|
+
export function markEnded(stateDir, ref) {
|
|
50
|
+
try {
|
|
51
|
+
rmSync(markPath(stateDir, ref), { force: true });
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// Leaving a stale mark makes the next retry cautious, never wrong.
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** True when a run started and no end was recorded: the outcome is unknown. */
|
|
58
|
+
export function startedWithoutEnd(stateDir, ref) {
|
|
59
|
+
try {
|
|
60
|
+
return existsSync(markPath(stateDir, ref));
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
export { createProxy, LedgerDenyUnrecorded } from "./proxy.ts";
|
|
2
2
|
export { loadPolicy } from "./policy.ts";
|
|
3
|
-
export { FileLedger, MemoryLedger } from "./ledger.ts";
|
|
3
|
+
export { FileLedger, MemoryLedger, readLedgerTail } from "./ledger.ts";
|
|
4
|
+
export { indexCoverage, listPieceFiles } from "./ledger-manifest.ts";
|
|
4
5
|
export { explain } from "./explain.ts";
|
|
5
|
-
export { approvePending, approvalsLogFor, enqueueApprovalCommand, loadApprovalsFromDir } from "./approvals.ts";
|
|
6
|
+
export { approvePending, approvalsLogFor, createApprovalBudgetGuard, enqueueApprovalCommand, loadApprovalsFromDir, } from "./approvals.ts";
|
|
6
7
|
export { loadEffectsFromDir, parseCardCsv, parseChannelJsonl, reconcile } from "./reconcile.ts";
|
|
7
8
|
export { tenantKey } from "./tenant.ts";
|
|
8
9
|
export { diskProbe } from "./disk.ts";
|
|
@@ -11,4 +12,5 @@ export { signEffectAttestation } from "./ledger.ts";
|
|
|
11
12
|
export { spokenReason } from "./spoken-reason.ts";
|
|
12
13
|
export type { ApprovalRow, ApproveResult } from "./approvals.ts";
|
|
13
14
|
export type { CardCsvOpts, ChannelRow, ReconcileReport } from "./reconcile.ts";
|
|
15
|
+
export type { FileLedgerOpts, LedgerCounts } from "./ledger.ts";
|
|
14
16
|
export type { EffectSigner, ExplainChain, ExplainPair, ExplainFinding, ExplainOpts, ExplainResult, ExplainTrustRoot, ExplainWarning, ExtractWindow, Ledger, LedgerEffect, Policy, PolicyDecision, Principal, ProxyDeps, RecordSigner, ToolCall, ToolResult, WitnessClass, } from "./types.ts";
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
export { createProxy, LedgerDenyUnrecorded } from "./proxy.js";
|
|
2
2
|
export { loadPolicy } from "./policy.js";
|
|
3
|
-
export { FileLedger, MemoryLedger } from "./ledger.js";
|
|
3
|
+
export { FileLedger, MemoryLedger, readLedgerTail } from "./ledger.js";
|
|
4
|
+
export { indexCoverage, listPieceFiles } from "./ledger-manifest.js";
|
|
4
5
|
export { explain } from "./explain.js";
|
|
5
|
-
export { approvePending, approvalsLogFor, enqueueApprovalCommand, loadApprovalsFromDir } from "./approvals.js";
|
|
6
|
+
export { approvePending, approvalsLogFor, createApprovalBudgetGuard, enqueueApprovalCommand, loadApprovalsFromDir, } from "./approvals.js";
|
|
6
7
|
export { loadEffectsFromDir, parseCardCsv, parseChannelJsonl, reconcile } from "./reconcile.js";
|
|
7
8
|
export { tenantKey } from "./tenant.js";
|
|
8
9
|
// The body writes checkpoints and signs its own effect attestations. Both were
|
package/dist/inputs.d.ts
CHANGED
|
@@ -6,7 +6,13 @@ export declare class MemoryInputsLog implements InputsLog {
|
|
|
6
6
|
}
|
|
7
7
|
export declare class FileInputsLog implements InputsLog {
|
|
8
8
|
private readonly dir;
|
|
9
|
-
|
|
9
|
+
private readonly resolve;
|
|
10
|
+
private readonly note;
|
|
11
|
+
constructor(dir: string, resolve?: () => {
|
|
12
|
+
active: string;
|
|
13
|
+
all: string[];
|
|
14
|
+
}, note?: (ref: string, inputs: DecisionInputs) => void);
|
|
15
|
+
private paths;
|
|
10
16
|
append(ref: string, inputs: DecisionInputs): Promise<void>;
|
|
11
17
|
get(ref: string): Promise<DecisionInputs | null>;
|
|
12
18
|
}
|
package/dist/inputs.js
CHANGED
|
@@ -11,29 +11,44 @@ export class MemoryInputsLog {
|
|
|
11
11
|
}
|
|
12
12
|
export class FileInputsLog {
|
|
13
13
|
dir;
|
|
14
|
-
|
|
14
|
+
resolve;
|
|
15
|
+
note;
|
|
16
|
+
constructor(dir, resolve, note) {
|
|
15
17
|
this.dir = dir;
|
|
18
|
+
this.resolve = resolve;
|
|
19
|
+
this.note = note;
|
|
20
|
+
}
|
|
21
|
+
paths() {
|
|
22
|
+
if (this.resolve)
|
|
23
|
+
return this.resolve();
|
|
24
|
+
const root = join(this.dir, "inputs.jsonl");
|
|
25
|
+
return { active: root, all: [root] };
|
|
16
26
|
}
|
|
17
27
|
async append(ref, inputs) {
|
|
18
|
-
await appendDurable(
|
|
28
|
+
await appendDurable(this.paths().active, `${JSON.stringify({ ref, inputs })}\n`);
|
|
29
|
+
// Handed to the ledger in memory: the decision that follows needs the
|
|
30
|
+
// principal for its index line and must not read this file back.
|
|
31
|
+
this.note?.(ref, inputs);
|
|
19
32
|
}
|
|
20
33
|
async get(ref) {
|
|
21
|
-
let text;
|
|
22
|
-
try {
|
|
23
|
-
text = await ledgerFs.readFile(join(this.dir, "inputs.jsonl"), "utf8");
|
|
24
|
-
}
|
|
25
|
-
catch (err) {
|
|
26
|
-
if (err.code === "ENOENT")
|
|
27
|
-
return null;
|
|
28
|
-
throw err;
|
|
29
|
-
}
|
|
30
34
|
let found = null;
|
|
31
|
-
for (const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
for (const path of this.paths().all) {
|
|
36
|
+
let text;
|
|
37
|
+
try {
|
|
38
|
+
text = await ledgerFs.readFile(path, "utf8");
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
if (err.code === "ENOENT")
|
|
42
|
+
continue;
|
|
43
|
+
throw err;
|
|
44
|
+
}
|
|
45
|
+
for (const line of text.split("\n")) {
|
|
46
|
+
if (line === "")
|
|
47
|
+
continue;
|
|
48
|
+
const row = JSON.parse(line);
|
|
49
|
+
if (row.ref === ref)
|
|
50
|
+
found = row.inputs;
|
|
51
|
+
}
|
|
37
52
|
}
|
|
38
53
|
return found;
|
|
39
54
|
}
|
|
@@ -43,7 +58,9 @@ export function inputsLogFor(ledger) {
|
|
|
43
58
|
const bag = ledger;
|
|
44
59
|
if (bag[STASH])
|
|
45
60
|
return bag[STASH];
|
|
46
|
-
const
|
|
61
|
+
const resolve = typeof bag.inputsPaths === "function" ? () => bag.inputsPaths() : undefined;
|
|
62
|
+
const note = typeof bag.noteInputs === "function" ? (ref, inputs) => bag.noteInputs(ref, inputs) : undefined;
|
|
63
|
+
const log = typeof bag.dir === "string" ? new FileInputsLog(bag.dir, resolve, note) : new MemoryInputsLog();
|
|
47
64
|
bag[STASH] = log;
|
|
48
65
|
return log;
|
|
49
66
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { open as fsOpen } from "node:fs/promises";
|
|
2
|
+
export type TailVisit = "take" | "skip" | "stop";
|
|
3
|
+
export type TailOpts = {
|
|
4
|
+
/**
|
|
5
|
+
* Bytes read per step from the end. Default 64 KiB: measured on a 194 MB
|
|
6
|
+
* file, 64 KiB, 256 KiB and 1 MiB read the whole of it in the same time,
|
|
7
|
+
* and the small step keeps a small window's cost small.
|
|
8
|
+
*/
|
|
9
|
+
chunkBytes?: number;
|
|
10
|
+
/** The file opener, so a seam or a test can count what was read. */
|
|
11
|
+
open?: typeof fsOpen;
|
|
12
|
+
/**
|
|
13
|
+
* Walk backwards from this byte instead of the file's end. A window that
|
|
14
|
+
* starts inside a piece uses this after a timestamp seek so the walk does
|
|
15
|
+
* not cross every later row.
|
|
16
|
+
*/
|
|
17
|
+
endOffset?: number;
|
|
18
|
+
};
|
|
19
|
+
export declare function readJsonlTail<T>(path: string, visit: (row: T) => TailVisit, opts?: TailOpts): Promise<T[]>;
|
|
20
|
+
/**
|
|
21
|
+
* The start of the first row whose stamp is at or after `targetMs`, or the
|
|
22
|
+
* file size when none is. Rows are treated as time-ordered (one writer, the
|
|
23
|
+
* clock's `now()`); a 60 s slack on the caller covers a clock set back.
|
|
24
|
+
* Each probe reads from the mid offset until the first complete line.
|
|
25
|
+
*/
|
|
26
|
+
export declare function seekOffsetByTimestamp<T>(path: string, targetMs: number, tsOf: (row: T) => number, opts?: TailOpts): Promise<number>;
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
// A JSON-lines file read from its end. The ledger files are append-only and
|
|
2
|
+
// in time order, so what a reader usually wants (the last day, the last N
|
|
3
|
+
// rows) sits at the end, and reading the whole file to get there costs the
|
|
4
|
+
// whole file: measured on 17 Sep 2026, a 100k-decision ledger answered a
|
|
5
|
+
// 24-hour window in 1.5 s, 40% of the time it took to answer everything.
|
|
6
|
+
//
|
|
7
|
+
// The file is walked backwards in chunks. Each row goes to a visitor, newest
|
|
8
|
+
// first, which says take, skip or stop. What was taken comes back in file
|
|
9
|
+
// order. A chunk is cut at its first newline: what lies before it is the tail
|
|
10
|
+
// of a row that continues further up and is carried to the next chunk; what
|
|
11
|
+
// lies after it is whole rows, decoded together and split. Newlines are
|
|
12
|
+
// single bytes that never occur inside a multibyte character, so a chunk
|
|
13
|
+
// boundary inside one changes nothing.
|
|
14
|
+
import { open as fsOpen } from "node:fs/promises";
|
|
15
|
+
const NEWLINE = 0x0a;
|
|
16
|
+
export async function readJsonlTail(path, visit, opts = {}) {
|
|
17
|
+
const chunkBytes = Math.max(1, Math.floor(opts.chunkBytes ?? 64 * 1024));
|
|
18
|
+
const openFile = opts.open ?? fsOpen;
|
|
19
|
+
let fh;
|
|
20
|
+
try {
|
|
21
|
+
fh = await openFile(path, "r");
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
if (err.code === "ENOENT")
|
|
25
|
+
return [];
|
|
26
|
+
throw err;
|
|
27
|
+
}
|
|
28
|
+
// Newest first while reading; reversed once at the end.
|
|
29
|
+
const taken = [];
|
|
30
|
+
let stopped = false;
|
|
31
|
+
const emitLines = (text) => {
|
|
32
|
+
const lines = text.split("\n");
|
|
33
|
+
for (let i = lines.length - 1; i >= 0 && !stopped; i -= 1) {
|
|
34
|
+
const line = lines[i];
|
|
35
|
+
if (line.trim() === "")
|
|
36
|
+
continue;
|
|
37
|
+
const row = JSON.parse(line);
|
|
38
|
+
const verdict = visit(row);
|
|
39
|
+
if (verdict === "take")
|
|
40
|
+
taken.push(row);
|
|
41
|
+
if (verdict === "stop")
|
|
42
|
+
stopped = true;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
try {
|
|
46
|
+
const { size } = await fh.stat();
|
|
47
|
+
const end = opts.endOffset === undefined ? size : Math.min(size, Math.max(0, Math.floor(opts.endOffset)));
|
|
48
|
+
let pos = end;
|
|
49
|
+
// The bytes before the first newline of the chunk just read: the tail of
|
|
50
|
+
// a row whose head is still further up the file.
|
|
51
|
+
let carry = Buffer.alloc(0);
|
|
52
|
+
while (pos > 0 && !stopped) {
|
|
53
|
+
const len = Math.min(chunkBytes, pos);
|
|
54
|
+
pos -= len;
|
|
55
|
+
const buf = Buffer.allocUnsafe(len);
|
|
56
|
+
let got = 0;
|
|
57
|
+
while (got < len) {
|
|
58
|
+
const r = await fh.read(buf, got, len - got, pos + got);
|
|
59
|
+
if (r.bytesRead === 0)
|
|
60
|
+
break;
|
|
61
|
+
got += r.bytesRead;
|
|
62
|
+
}
|
|
63
|
+
const data = carry.length > 0 ? Buffer.concat([buf.subarray(0, got), carry]) : buf.subarray(0, got);
|
|
64
|
+
const first = data.indexOf(NEWLINE);
|
|
65
|
+
if (first === -1) {
|
|
66
|
+
// No row ends in this chunk: all of it belongs to a row further up,
|
|
67
|
+
// unless this is the top of the file, where it is the first row.
|
|
68
|
+
if (pos === 0)
|
|
69
|
+
emitLines(data.toString("utf8"));
|
|
70
|
+
else
|
|
71
|
+
carry = Buffer.from(data);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
emitLines(data.toString("utf8", first + 1));
|
|
75
|
+
if (stopped)
|
|
76
|
+
break;
|
|
77
|
+
if (pos === 0) {
|
|
78
|
+
if (first > 0)
|
|
79
|
+
emitLines(data.toString("utf8", 0, first));
|
|
80
|
+
carry = Buffer.alloc(0);
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
carry = Buffer.from(data.subarray(0, first));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
await fh.close();
|
|
89
|
+
}
|
|
90
|
+
return taken.reverse();
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The start of the first row whose stamp is at or after `targetMs`, or the
|
|
94
|
+
* file size when none is. Rows are treated as time-ordered (one writer, the
|
|
95
|
+
* clock's `now()`); a 60 s slack on the caller covers a clock set back.
|
|
96
|
+
* Each probe reads from the mid offset until the first complete line.
|
|
97
|
+
*/
|
|
98
|
+
export async function seekOffsetByTimestamp(path, targetMs, tsOf, opts = {}) {
|
|
99
|
+
const chunkBytes = Math.max(1, Math.floor(opts.chunkBytes ?? 64 * 1024));
|
|
100
|
+
const openFile = opts.open ?? fsOpen;
|
|
101
|
+
let fh;
|
|
102
|
+
try {
|
|
103
|
+
fh = await openFile(path, "r");
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
if (err.code === "ENOENT")
|
|
107
|
+
return 0;
|
|
108
|
+
throw err;
|
|
109
|
+
}
|
|
110
|
+
try {
|
|
111
|
+
const { size } = await fh.stat();
|
|
112
|
+
if (size === 0)
|
|
113
|
+
return 0;
|
|
114
|
+
const readRange = async (from, len) => {
|
|
115
|
+
if (len <= 0)
|
|
116
|
+
return Buffer.alloc(0);
|
|
117
|
+
const buf = Buffer.allocUnsafe(len);
|
|
118
|
+
let got = 0;
|
|
119
|
+
while (got < len) {
|
|
120
|
+
const r = await fh.read(buf, got, len - got, from + got);
|
|
121
|
+
if (r.bytesRead === 0)
|
|
122
|
+
break;
|
|
123
|
+
got += r.bytesRead;
|
|
124
|
+
}
|
|
125
|
+
return buf.subarray(0, got);
|
|
126
|
+
};
|
|
127
|
+
const probe = Math.min(4096, chunkBytes);
|
|
128
|
+
const parseLine = (start, text, end) => {
|
|
129
|
+
if (text.trim() === "")
|
|
130
|
+
return null;
|
|
131
|
+
return { start, end, row: JSON.parse(text) };
|
|
132
|
+
};
|
|
133
|
+
const firstCompleteLine = async (mid) => {
|
|
134
|
+
const from = mid > 0 ? mid - 1 : 0;
|
|
135
|
+
let collected = await readRange(from, Math.min(probe, size - from));
|
|
136
|
+
if (collected.length === 0)
|
|
137
|
+
return null;
|
|
138
|
+
let start;
|
|
139
|
+
let rel = 0;
|
|
140
|
+
if (mid <= 0) {
|
|
141
|
+
start = 0;
|
|
142
|
+
rel = 0;
|
|
143
|
+
}
|
|
144
|
+
else if (collected[0] === NEWLINE) {
|
|
145
|
+
start = mid;
|
|
146
|
+
rel = 1;
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
let pos = from;
|
|
150
|
+
let buf = collected;
|
|
151
|
+
let found = false;
|
|
152
|
+
for (;;) {
|
|
153
|
+
const nl = buf.indexOf(NEWLINE, pos === from ? 1 : 0);
|
|
154
|
+
if (nl !== -1) {
|
|
155
|
+
start = pos + nl + 1;
|
|
156
|
+
collected = buf;
|
|
157
|
+
rel = nl + 1;
|
|
158
|
+
found = true;
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
161
|
+
pos += buf.length;
|
|
162
|
+
if (pos >= size)
|
|
163
|
+
return null;
|
|
164
|
+
buf = await readRange(pos, Math.min(probe, size - pos));
|
|
165
|
+
if (buf.length === 0)
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
if (!found)
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
if (start >= size)
|
|
172
|
+
return null;
|
|
173
|
+
const after = collected.subarray(rel);
|
|
174
|
+
const nl = after.indexOf(NEWLINE);
|
|
175
|
+
if (nl !== -1) {
|
|
176
|
+
const parsed = parseLine(start, after.subarray(0, nl).toString("utf8"), start + nl + 1);
|
|
177
|
+
return parsed ?? firstCompleteLine(start + nl + 1);
|
|
178
|
+
}
|
|
179
|
+
let acc = Buffer.from(after);
|
|
180
|
+
let pos = start + after.length;
|
|
181
|
+
while (pos < size) {
|
|
182
|
+
const chunk = await readRange(pos, Math.min(probe, size - pos));
|
|
183
|
+
if (chunk.length === 0)
|
|
184
|
+
break;
|
|
185
|
+
const n2 = chunk.indexOf(NEWLINE);
|
|
186
|
+
if (n2 !== -1) {
|
|
187
|
+
const parsed = parseLine(start, Buffer.concat([acc, chunk.subarray(0, n2)]).toString("utf8"), start + acc.length + n2 + 1);
|
|
188
|
+
return parsed ?? firstCompleteLine(start + acc.length + n2 + 1);
|
|
189
|
+
}
|
|
190
|
+
acc = Buffer.concat([acc, chunk]);
|
|
191
|
+
pos += chunk.length;
|
|
192
|
+
}
|
|
193
|
+
return parseLine(start, acc.toString("utf8"), size);
|
|
194
|
+
};
|
|
195
|
+
let lo = 0;
|
|
196
|
+
let hi = size;
|
|
197
|
+
let found = size;
|
|
198
|
+
while (lo < hi) {
|
|
199
|
+
const mid = lo + Math.floor((hi - lo) / 2);
|
|
200
|
+
const line = await firstCompleteLine(mid);
|
|
201
|
+
if (line === null || line.start >= hi) {
|
|
202
|
+
hi = mid;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
if (tsOf(line.row) >= targetMs) {
|
|
206
|
+
found = line.start;
|
|
207
|
+
hi = mid;
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
lo = line.end > lo ? line.end : lo + 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return found;
|
|
214
|
+
}
|
|
215
|
+
finally {
|
|
216
|
+
await fh.close();
|
|
217
|
+
}
|
|
218
|
+
}
|