@buildanagent/harness 0.0.0-stage → 0.1.0-alpha.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +54 -0
- package/LICENSE-APACHE +202 -0
- package/LICENSE-MIT +21 -0
- package/README.md +716 -2
- package/dist/ablation.d.ts +2 -0
- package/dist/ablation.js +3 -0
- package/dist/ablation.js.map +1 -0
- package/dist/completion/check.d.ts +64 -0
- package/dist/completion/check.js +152 -0
- package/dist/completion/check.js.map +1 -0
- package/dist/completion/checks.d.ts +45 -0
- package/dist/completion/checks.js +173 -0
- package/dist/completion/checks.js.map +1 -0
- package/dist/completion/flags.d.ts +25 -0
- package/dist/completion/flags.js +87 -0
- package/dist/completion/flags.js.map +1 -0
- package/dist/completion/index.d.ts +8 -0
- package/dist/completion/index.js +9 -0
- package/dist/completion/index.js.map +1 -0
- package/dist/completion/snapshot.d.ts +33 -0
- package/dist/completion/snapshot.js +83 -0
- package/dist/completion/snapshot.js.map +1 -0
- package/dist/context/cap.d.ts +32 -0
- package/dist/context/cap.js +50 -0
- package/dist/context/cap.js.map +1 -0
- package/dist/context/estimate.d.ts +18 -0
- package/dist/context/estimate.js +20 -0
- package/dist/context/estimate.js.map +1 -0
- package/dist/context/file-spill.d.ts +5 -0
- package/dist/context/file-spill.js +25 -0
- package/dist/context/file-spill.js.map +1 -0
- package/dist/context/index.d.ts +10 -0
- package/dist/context/index.js +10 -0
- package/dist/context/index.js.map +1 -0
- package/dist/context/mask.d.ts +27 -0
- package/dist/context/mask.js +71 -0
- package/dist/context/mask.js.map +1 -0
- package/dist/context/mid-turn.d.ts +40 -0
- package/dist/context/mid-turn.js +82 -0
- package/dist/context/mid-turn.js.map +1 -0
- package/dist/context/policy.d.ts +9 -0
- package/dist/context/policy.js +140 -0
- package/dist/context/policy.js.map +1 -0
- package/dist/context/spill-ledger.d.ts +25 -0
- package/dist/context/spill-ledger.js +54 -0
- package/dist/context/spill-ledger.js.map +1 -0
- package/dist/context/spill.d.ts +6 -0
- package/dist/context/spill.js +14 -0
- package/dist/context/spill.js.map +1 -0
- package/dist/context/summarize.d.ts +53 -0
- package/dist/context/summarize.js +204 -0
- package/dist/context/summarize.js.map +1 -0
- package/dist/context/summary-template.d.ts +29 -0
- package/dist/context/summary-template.js +69 -0
- package/dist/context/summary-template.js.map +1 -0
- package/dist/context/types.d.ts +150 -0
- package/dist/context/types.js +2 -0
- package/dist/context/types.js.map +1 -0
- package/dist/context/write-cap.d.ts +19 -0
- package/dist/context/write-cap.js +41 -0
- package/dist/context/write-cap.js.map +1 -0
- package/dist/create-harness.d.ts +7 -0
- package/dist/create-harness.js +21 -0
- package/dist/create-harness.js.map +1 -0
- package/dist/credentials/addresses.d.ts +14 -0
- package/dist/credentials/addresses.js +144 -0
- package/dist/credentials/addresses.js.map +1 -0
- package/dist/credentials/allowlist.d.ts +34 -0
- package/dist/credentials/allowlist.js +77 -0
- package/dist/credentials/allowlist.js.map +1 -0
- package/dist/credentials/egress-proxy.d.ts +70 -0
- package/dist/credentials/egress-proxy.js +482 -0
- package/dist/credentials/egress-proxy.js.map +1 -0
- package/dist/credentials/index.d.ts +9 -0
- package/dist/credentials/index.js +10 -0
- package/dist/credentials/index.js.map +1 -0
- package/dist/credentials/scrub.d.ts +16 -0
- package/dist/credentials/scrub.js +93 -0
- package/dist/credentials/scrub.js.map +1 -0
- package/dist/errors.d.ts +56 -0
- package/dist/errors.js +63 -0
- package/dist/errors.js.map +1 -0
- package/dist/extensions/compose.d.ts +27 -0
- package/dist/extensions/compose.js +83 -0
- package/dist/extensions/compose.js.map +1 -0
- package/dist/extensions/context-files-node.d.ts +28 -0
- package/dist/extensions/context-files-node.js +74 -0
- package/dist/extensions/context-files-node.js.map +1 -0
- package/dist/extensions/context-files.d.ts +68 -0
- package/dist/extensions/context-files.js +214 -0
- package/dist/extensions/context-files.js.map +1 -0
- package/dist/extensions/digest.d.ts +6 -0
- package/dist/extensions/digest.js +68 -0
- package/dist/extensions/digest.js.map +1 -0
- package/dist/extensions/index.d.ts +10 -0
- package/dist/extensions/index.js +10 -0
- package/dist/extensions/index.js.map +1 -0
- package/dist/extensions/load.d.ts +42 -0
- package/dist/extensions/load.js +211 -0
- package/dist/extensions/load.js.map +1 -0
- package/dist/extensions/lock.d.ts +8 -0
- package/dist/extensions/lock.js +95 -0
- package/dist/extensions/lock.js.map +1 -0
- package/dist/extensions/node.d.ts +10 -0
- package/dist/extensions/node.js +11 -0
- package/dist/extensions/node.js.map +1 -0
- package/dist/extensions/project-dirs.d.ts +24 -0
- package/dist/extensions/project-dirs.js +97 -0
- package/dist/extensions/project-dirs.js.map +1 -0
- package/dist/extensions/skills-node.d.ts +34 -0
- package/dist/extensions/skills-node.js +123 -0
- package/dist/extensions/skills-node.js.map +1 -0
- package/dist/extensions/skills.d.ts +51 -0
- package/dist/extensions/skills.js +175 -0
- package/dist/extensions/skills.js.map +1 -0
- package/dist/extensions/trust-file.d.ts +3 -0
- package/dist/extensions/trust-file.js +62 -0
- package/dist/extensions/trust-file.js.map +1 -0
- package/dist/extensions/trust-store-node.d.ts +10 -0
- package/dist/extensions/trust-store-node.js +33 -0
- package/dist/extensions/trust-store-node.js.map +1 -0
- package/dist/extensions/types.d.ts +42 -0
- package/dist/extensions/types.js +2 -0
- package/dist/extensions/types.js.map +1 -0
- package/dist/harness.d.ts +156 -0
- package/dist/harness.js +2 -0
- package/dist/harness.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/connection.d.ts +22 -0
- package/dist/mcp/connection.js +267 -0
- package/dist/mcp/connection.js.map +1 -0
- package/dist/mcp/content.d.ts +27 -0
- package/dist/mcp/content.js +177 -0
- package/dist/mcp/content.js.map +1 -0
- package/dist/mcp/errors.d.ts +19 -0
- package/dist/mcp/errors.js +21 -0
- package/dist/mcp/errors.js.map +1 -0
- package/dist/mcp/index.d.ts +10 -0
- package/dist/mcp/index.js +10 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/manager.d.ts +7 -0
- package/dist/mcp/manager.js +173 -0
- package/dist/mcp/manager.js.map +1 -0
- package/dist/mcp/record.d.ts +7 -0
- package/dist/mcp/record.js +20 -0
- package/dist/mcp/record.js.map +1 -0
- package/dist/mcp/stdio.d.ts +39 -0
- package/dist/mcp/stdio.js +382 -0
- package/dist/mcp/stdio.js.map +1 -0
- package/dist/mcp/types.d.ts +172 -0
- package/dist/mcp/types.js +2 -0
- package/dist/mcp/types.js.map +1 -0
- package/dist/memory/index.d.ts +2 -0
- package/dist/memory/index.js +3 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/permissions/default-rules.d.ts +7 -0
- package/dist/permissions/default-rules.js +42 -0
- package/dist/permissions/default-rules.js.map +1 -0
- package/dist/permissions/errors.d.ts +11 -0
- package/dist/permissions/errors.js +14 -0
- package/dist/permissions/errors.js.map +1 -0
- package/dist/permissions/index.d.ts +9 -0
- package/dist/permissions/index.js +9 -0
- package/dist/permissions/index.js.map +1 -0
- package/dist/permissions/match.d.ts +33 -0
- package/dist/permissions/match.js +287 -0
- package/dist/permissions/match.js.map +1 -0
- package/dist/permissions/pipeline.d.ts +9 -0
- package/dist/permissions/pipeline.js +235 -0
- package/dist/permissions/pipeline.js.map +1 -0
- package/dist/permissions/rules.d.ts +22 -0
- package/dist/permissions/rules.js +74 -0
- package/dist/permissions/rules.js.map +1 -0
- package/dist/permissions/shadow.d.ts +5 -0
- package/dist/permissions/shadow.js +22 -0
- package/dist/permissions/shadow.js.map +1 -0
- package/dist/permissions/types.d.ts +104 -0
- package/dist/permissions/types.js +2 -0
- package/dist/permissions/types.js.map +1 -0
- package/dist/sandbox/base.d.ts +35 -0
- package/dist/sandbox/base.js +60 -0
- package/dist/sandbox/base.js.map +1 -0
- package/dist/sandbox/docker/index.d.ts +35 -0
- package/dist/sandbox/docker/index.js +448 -0
- package/dist/sandbox/docker/index.js.map +1 -0
- package/dist/sandbox/errors.d.ts +16 -0
- package/dist/sandbox/errors.js +22 -0
- package/dist/sandbox/errors.js.map +1 -0
- package/dist/sandbox/host-files.d.ts +63 -0
- package/dist/sandbox/host-files.js +248 -0
- package/dist/sandbox/host-files.js.map +1 -0
- package/dist/sandbox/index.d.ts +8 -0
- package/dist/sandbox/index.js +8 -0
- package/dist/sandbox/index.js.map +1 -0
- package/dist/sandbox/none.d.ts +17 -0
- package/dist/sandbox/none.js +108 -0
- package/dist/sandbox/none.js.map +1 -0
- package/dist/sandbox/process.d.ts +61 -0
- package/dist/sandbox/process.js +253 -0
- package/dist/sandbox/process.js.map +1 -0
- package/dist/sandbox/spawn-env.d.ts +20 -0
- package/dist/sandbox/spawn-env.js +128 -0
- package/dist/sandbox/spawn-env.js.map +1 -0
- package/dist/sandbox/srt/index.d.ts +35 -0
- package/dist/sandbox/srt/index.js +264 -0
- package/dist/sandbox/srt/index.js.map +1 -0
- package/dist/sandbox/srt/settings.d.ts +48 -0
- package/dist/sandbox/srt/settings.js +31 -0
- package/dist/sandbox/srt/settings.js.map +1 -0
- package/dist/sandbox/types.d.ts +88 -0
- package/dist/sandbox/types.js +6 -0
- package/dist/sandbox/types.js.map +1 -0
- package/dist/server/answers.d.ts +16 -0
- package/dist/server/answers.js +43 -0
- package/dist/server/answers.js.map +1 -0
- package/dist/server/connection.d.ts +99 -0
- package/dist/server/connection.js +309 -0
- package/dist/server/connection.js.map +1 -0
- package/dist/server/dispatcher.d.ts +86 -0
- package/dist/server/dispatcher.js +800 -0
- package/dist/server/dispatcher.js.map +1 -0
- package/dist/server/index.d.ts +13 -0
- package/dist/server/index.js +12 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/local-transport.d.ts +44 -0
- package/dist/server/local-transport.js +103 -0
- package/dist/server/local-transport.js.map +1 -0
- package/dist/server/outbox.d.ts +31 -0
- package/dist/server/outbox.js +130 -0
- package/dist/server/outbox.js.map +1 -0
- package/dist/server/params.d.ts +25 -0
- package/dist/server/params.js +112 -0
- package/dist/server/params.js.map +1 -0
- package/dist/server/protocol.schema.json +2448 -0
- package/dist/server/rpc-session.d.ts +43 -0
- package/dist/server/rpc-session.js +167 -0
- package/dist/server/rpc-session.js.map +1 -0
- package/dist/server/stdio-streams.d.ts +67 -0
- package/dist/server/stdio-streams.js +218 -0
- package/dist/server/stdio-streams.js.map +1 -0
- package/dist/server/stdio.d.ts +59 -0
- package/dist/server/stdio.js +189 -0
- package/dist/server/stdio.js.map +1 -0
- package/dist/server/websocket.d.ts +106 -0
- package/dist/server/websocket.js +327 -0
- package/dist/server/websocket.js.map +1 -0
- package/dist/session/approvals.d.ts +81 -0
- package/dist/session/approvals.js +217 -0
- package/dist/session/approvals.js.map +1 -0
- package/dist/session/code-call-log.d.ts +42 -0
- package/dist/session/code-call-log.js +63 -0
- package/dist/session/code-call-log.js.map +1 -0
- package/dist/session/code-call-result.d.ts +19 -0
- package/dist/session/code-call-result.js +95 -0
- package/dist/session/code-call-result.js.map +1 -0
- package/dist/session/code-call.d.ts +49 -0
- package/dist/session/code-call.js +345 -0
- package/dist/session/code-call.js.map +1 -0
- package/dist/session/effect-scope.d.ts +11 -0
- package/dist/session/effect-scope.js +10 -0
- package/dist/session/effect-scope.js.map +1 -0
- package/dist/session/errors.d.ts +19 -0
- package/dist/session/errors.js +27 -0
- package/dist/session/errors.js.map +1 -0
- package/dist/session/file-index.d.ts +8 -0
- package/dist/session/file-index.js +70 -0
- package/dist/session/file-index.js.map +1 -0
- package/dist/session/file-key-store.d.ts +16 -0
- package/dist/session/file-key-store.js +169 -0
- package/dist/session/file-key-store.js.map +1 -0
- package/dist/session/file-lock.d.ts +52 -0
- package/dist/session/file-lock.js +324 -0
- package/dist/session/file-lock.js.map +1 -0
- package/dist/session/index.d.ts +22 -0
- package/dist/session/index.js +20 -0
- package/dist/session/index.js.map +1 -0
- package/dist/session/jsonl-store.d.ts +37 -0
- package/dist/session/jsonl-store.js +386 -0
- package/dist/session/jsonl-store.js.map +1 -0
- package/dist/session/jsonl.d.ts +9 -0
- package/dist/session/jsonl.js +10 -0
- package/dist/session/jsonl.js.map +1 -0
- package/dist/session/lock.d.ts +19 -0
- package/dist/session/lock.js +2 -0
- package/dist/session/lock.js.map +1 -0
- package/dist/session/memory-store.d.ts +12 -0
- package/dist/session/memory-store.js +60 -0
- package/dist/session/memory-store.js.map +1 -0
- package/dist/session/migrate.d.ts +41 -0
- package/dist/session/migrate.js +86 -0
- package/dist/session/migrate.js.map +1 -0
- package/dist/session/pointer.d.ts +15 -0
- package/dist/session/pointer.js +22 -0
- package/dist/session/pointer.js.map +1 -0
- package/dist/session/rebuild.d.ts +53 -0
- package/dist/session/rebuild.js +230 -0
- package/dist/session/rebuild.js.map +1 -0
- package/dist/session/resume.d.ts +26 -0
- package/dist/session/resume.js +128 -0
- package/dist/session/resume.js.map +1 -0
- package/dist/session/retention.d.ts +97 -0
- package/dist/session/retention.js +702 -0
- package/dist/session/retention.js.map +1 -0
- package/dist/session/runner.d.ts +156 -0
- package/dist/session/runner.js +656 -0
- package/dist/session/runner.js.map +1 -0
- package/dist/session/scheduler.d.ts +47 -0
- package/dist/session/scheduler.js +85 -0
- package/dist/session/scheduler.js.map +1 -0
- package/dist/session/secret-key-store.d.ts +47 -0
- package/dist/session/secret-key-store.js +79 -0
- package/dist/session/secret-key-store.js.map +1 -0
- package/dist/session/session.d.ts +106 -0
- package/dist/session/session.js +353 -0
- package/dist/session/session.js.map +1 -0
- package/dist/session/sticky.d.ts +48 -0
- package/dist/session/sticky.js +232 -0
- package/dist/session/sticky.js.map +1 -0
- package/dist/session/store.d.ts +108 -0
- package/dist/session/store.js +163 -0
- package/dist/session/store.js.map +1 -0
- package/dist/telemetry/index.d.ts +2 -0
- package/dist/telemetry/index.js +3 -0
- package/dist/telemetry/index.js.map +1 -0
- package/dist/testing/fake-sandbox.d.ts +34 -0
- package/dist/testing/fake-sandbox.js +221 -0
- package/dist/testing/fake-sandbox.js.map +1 -0
- package/dist/testing/index.d.ts +10 -0
- package/dist/testing/index.js +11 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/key-store-conformance.d.ts +22 -0
- package/dist/testing/key-store-conformance.js +97 -0
- package/dist/testing/key-store-conformance.js.map +1 -0
- package/dist/testing/run-cases.d.ts +21 -0
- package/dist/testing/run-cases.js +20 -0
- package/dist/testing/run-cases.js.map +1 -0
- package/dist/testing/sandbox-conformance.d.ts +64 -0
- package/dist/testing/sandbox-conformance.js +361 -0
- package/dist/testing/sandbox-conformance.js.map +1 -0
- package/dist/testing/scheduler-conformance.d.ts +32 -0
- package/dist/testing/scheduler-conformance.js +72 -0
- package/dist/testing/scheduler-conformance.js.map +1 -0
- package/dist/testing/store-conformance.d.ts +21 -0
- package/dist/testing/store-conformance.js +320 -0
- package/dist/testing/store-conformance.js.map +1 -0
- package/dist/tools/errors.d.ts +13 -0
- package/dist/tools/errors.js +17 -0
- package/dist/tools/errors.js.map +1 -0
- package/dist/tools/index.d.ts +8 -0
- package/dist/tools/index.js +8 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/registry.d.ts +3 -0
- package/dist/tools/registry.js +234 -0
- package/dist/tools/registry.js.map +1 -0
- package/dist/tools/tool-search.d.ts +30 -0
- package/dist/tools/tool-search.js +118 -0
- package/dist/tools/tool-search.js.map +1 -0
- package/dist/tools/types.d.ts +106 -0
- package/dist/tools/types.js +2 -0
- package/dist/tools/types.js.map +1 -0
- package/llms.txt +435 -0
- package/package.json +118 -3
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `jsonlStore()`: a `SessionStore` that keeps each session in one JSONL file, one entry per line, under a folder per
|
|
3
|
+
* user. It needs Node's file system, so it has its own entry, `@buildanagent/harness/session/jsonl`, and the portable
|
|
4
|
+
* `./session` entry stays free of it.
|
|
5
|
+
*/
|
|
6
|
+
import { Buffer } from "node:buffer";
|
|
7
|
+
import { mkdir, open, readdir, readFile, rm, stat, truncate } from "node:fs/promises";
|
|
8
|
+
import { basename, join } from "node:path";
|
|
9
|
+
import { isProtocolError, parseEntry } from "@buildanagent/protocol";
|
|
10
|
+
import { HarnessError } from "../errors.js";
|
|
11
|
+
import { ARCHIVE_DIR, ARCHIVE_EXTENSION, archivePath, checkRetention, expireSessions, readArchive, restoreArchive, } from "./retention.js";
|
|
12
|
+
import { byAge, checkKey, DEFAULT_MAX_ENTRY_BYTES, encodeName, extendState, planAppend, stateOf, summaryOf, } from "./store.js";
|
|
13
|
+
const EXTENSION = ".jsonl";
|
|
14
|
+
/** Reads a log: a line counts when it ends in a newline, and a last line that doesn't is torn. Sizes are bytes of the file. */
|
|
15
|
+
function parseLog(buffer, path) {
|
|
16
|
+
const end = buffer.lastIndexOf(0x0a) + 1;
|
|
17
|
+
const entries = [];
|
|
18
|
+
const lines = buffer.subarray(0, end).toString("utf8").split("\n");
|
|
19
|
+
lines.pop();
|
|
20
|
+
for (const [index, line] of lines.entries()) {
|
|
21
|
+
try {
|
|
22
|
+
entries.push(parseEntry(line));
|
|
23
|
+
}
|
|
24
|
+
catch (cause) {
|
|
25
|
+
// A header from another format isn't corruption: the file is whole and needs a migration or a newer harness.
|
|
26
|
+
const format = isProtocolError(cause) && cause.code === "unsupported_format_version";
|
|
27
|
+
throw new HarnessError(format ? "unsupported_format" : "session_corrupt", `${path} has ${format ? "a log in a format this harness doesn't read" : "a bad entry"} on line ${index + 1}`, { cause });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return { entries, goodBytes: end, tornBytes: buffer.length - end };
|
|
31
|
+
}
|
|
32
|
+
async function readLog(path) {
|
|
33
|
+
try {
|
|
34
|
+
return parseLog(await readFile(path), path);
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
if (error.code === "ENOENT")
|
|
38
|
+
return undefined;
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Writes `data` at the end of the file and flushes it to disk before returning. */
|
|
43
|
+
async function appendDurably(path, data, isNew) {
|
|
44
|
+
const file = await open(path, "a");
|
|
45
|
+
try {
|
|
46
|
+
await file.appendFile(data);
|
|
47
|
+
await file.sync();
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
await file.close();
|
|
51
|
+
}
|
|
52
|
+
if (isNew)
|
|
53
|
+
await syncFolder(join(path, ".."));
|
|
54
|
+
}
|
|
55
|
+
// A new file is only durable once its folder entry is. Some platforms refuse to open a folder, and then there's nothing to do.
|
|
56
|
+
async function syncFolder(dir) {
|
|
57
|
+
try {
|
|
58
|
+
const handle = await open(dir, "r");
|
|
59
|
+
try {
|
|
60
|
+
await handle.sync();
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
await handle.close();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
/* the platform can't flush a folder */
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Enough for the sessions a process is working on; a log that falls out is read again from its file on the next append.
|
|
71
|
+
const MAX_CACHED_LOGS = 64;
|
|
72
|
+
// An archived file never changes in place, so its summary is kept by path, size and time; this bounds the memory.
|
|
73
|
+
const MAX_CACHED_SUMMARIES = 1024;
|
|
74
|
+
// How long a failed expiry pass waits before a touch of the same user runs it again.
|
|
75
|
+
const RETRY_AFTER_MS = 60_000;
|
|
76
|
+
const busyName = (key) => JSON.stringify([key.userId, key.sessionId]);
|
|
77
|
+
class JsonlStore {
|
|
78
|
+
#dir;
|
|
79
|
+
#maxBytes;
|
|
80
|
+
#options;
|
|
81
|
+
#expiry = new Map();
|
|
82
|
+
#archived = new Map();
|
|
83
|
+
#cache = new Map();
|
|
84
|
+
#queues = new Map();
|
|
85
|
+
/** How many calls past expiry are using each file, by user and file id; a pass keeps their sessions. */
|
|
86
|
+
#busy = new Map();
|
|
87
|
+
constructor(options) {
|
|
88
|
+
this.#dir = options.dir;
|
|
89
|
+
this.#maxBytes = options.maxEntryBytes ?? DEFAULT_MAX_ENTRY_BYTES;
|
|
90
|
+
this.#options = options;
|
|
91
|
+
if (options.retention !== undefined)
|
|
92
|
+
checkRetention(options.retention);
|
|
93
|
+
}
|
|
94
|
+
#now() {
|
|
95
|
+
return (this.#options.now?.() ?? new Date()).getTime();
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Expires a user's sessions the first time this store touches them, and again after a failed pass once its wait is
|
|
99
|
+
* over. A failure is reported and never thrown, so the returned promise always resolves.
|
|
100
|
+
*/
|
|
101
|
+
#expire(userId) {
|
|
102
|
+
const { retention } = this.#options;
|
|
103
|
+
if (retention === undefined)
|
|
104
|
+
return Promise.resolve();
|
|
105
|
+
const known = this.#expiry.get(userId);
|
|
106
|
+
if (known !== undefined && (known.retryAt === undefined || this.#now() < known.retryAt))
|
|
107
|
+
return known.done;
|
|
108
|
+
const expiry = {
|
|
109
|
+
settled: false,
|
|
110
|
+
done: this.#prune(userId, retention).then((failed) => {
|
|
111
|
+
if (failed)
|
|
112
|
+
expiry.retryAt = this.#now() + RETRY_AFTER_MS;
|
|
113
|
+
expiry.settled = true;
|
|
114
|
+
}),
|
|
115
|
+
};
|
|
116
|
+
this.#expiry.set(userId, expiry);
|
|
117
|
+
return expiry.done;
|
|
118
|
+
}
|
|
119
|
+
/** One expiry pass. Resolves to whether it had an error, and never rejects. */
|
|
120
|
+
async #prune(userId, retention) {
|
|
121
|
+
try {
|
|
122
|
+
const options = {
|
|
123
|
+
dir: this.#dir,
|
|
124
|
+
userId,
|
|
125
|
+
retention,
|
|
126
|
+
now: () => new Date(this.#now()),
|
|
127
|
+
...(this.#options.lock === undefined ? {} : { lock: this.#options.lock }),
|
|
128
|
+
};
|
|
129
|
+
const result = await expireSessions(options, (fileId) => this.#busy.has(busyName({ userId, sessionId: fileId })));
|
|
130
|
+
for (const { error } of result.errors)
|
|
131
|
+
this.#report(error);
|
|
132
|
+
return result.errors.length > 0;
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
this.#report(error);
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Runs `task` once expiry for the key's user is over. A pass that another call started meanwhile runs first; once no
|
|
141
|
+
* pass runs, the file is marked busy in the same turn, so a pass that starts later keeps its session until `task` ends.
|
|
142
|
+
*/
|
|
143
|
+
async #afterExpiry(key, task) {
|
|
144
|
+
await this.#expire(key.userId);
|
|
145
|
+
for (let pass = this.#running(key.userId); pass !== undefined; pass = this.#running(key.userId))
|
|
146
|
+
await pass;
|
|
147
|
+
const name = busyName(key);
|
|
148
|
+
this.#busy.set(name, (this.#busy.get(name) ?? 0) + 1);
|
|
149
|
+
try {
|
|
150
|
+
return await task();
|
|
151
|
+
}
|
|
152
|
+
finally {
|
|
153
|
+
const left = (this.#busy.get(name) ?? 1) - 1;
|
|
154
|
+
if (left === 0)
|
|
155
|
+
this.#busy.delete(name);
|
|
156
|
+
else
|
|
157
|
+
this.#busy.set(name, left);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/** The pass running for the user, if one is. */
|
|
161
|
+
#running(userId) {
|
|
162
|
+
const expiry = this.#expiry.get(userId);
|
|
163
|
+
return expiry === undefined || expiry.settled ? undefined : expiry.done;
|
|
164
|
+
}
|
|
165
|
+
#report(error) {
|
|
166
|
+
try {
|
|
167
|
+
this.#options.onRetentionError?.(error);
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
/* the callback's own failure must not stop the session, nor every later call for the user */
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* The log in the live file, or else in its archive. A live file with no complete line holds nothing that was
|
|
175
|
+
* acknowledged, so it counts as missing.
|
|
176
|
+
*/
|
|
177
|
+
async #readAny(key, path) {
|
|
178
|
+
const live = await readLog(path);
|
|
179
|
+
if (live !== undefined && live.goodBytes > 0)
|
|
180
|
+
return live;
|
|
181
|
+
const archived = await readArchive(archivePath(this.#dir, key.userId, key.sessionId));
|
|
182
|
+
return archived === undefined ? live : parseLog(archived, path);
|
|
183
|
+
}
|
|
184
|
+
#userDir(userId) {
|
|
185
|
+
return join(this.#dir, encodeName(userId));
|
|
186
|
+
}
|
|
187
|
+
#pathOf(key) {
|
|
188
|
+
return join(this.#userDir(key.userId), `${encodeName(key.sessionId)}${EXTENSION}`);
|
|
189
|
+
}
|
|
190
|
+
/** Runs `task` after every earlier task on this file, so one process's appends, deletes and loads never interleave. */
|
|
191
|
+
#queued(path, task) {
|
|
192
|
+
const turn = (this.#queues.get(path) ?? Promise.resolve()).then(task);
|
|
193
|
+
const tail = turn.then(() => undefined, () => undefined);
|
|
194
|
+
this.#queues.set(path, tail);
|
|
195
|
+
// The entry is only needed while a task is waiting, so the map doesn't grow with every session ever touched.
|
|
196
|
+
void tail.then(() => {
|
|
197
|
+
if (this.#queues.get(path) === tail)
|
|
198
|
+
this.#queues.delete(path);
|
|
199
|
+
});
|
|
200
|
+
return turn;
|
|
201
|
+
}
|
|
202
|
+
#remember(path, size, state) {
|
|
203
|
+
this.#cache.delete(path);
|
|
204
|
+
this.#cache.set(path, { size, state });
|
|
205
|
+
const oldest = this.#cache.keys().next().value;
|
|
206
|
+
if (this.#cache.size > MAX_CACHED_LOGS && oldest !== undefined)
|
|
207
|
+
this.#cache.delete(oldest);
|
|
208
|
+
}
|
|
209
|
+
/** The log's state, from the cache when the file is the size it was left at, and from the file otherwise. */
|
|
210
|
+
async #read(path) {
|
|
211
|
+
const size = await stat(path).then((info) => info.size, () => -1);
|
|
212
|
+
const cached = this.#cache.get(path);
|
|
213
|
+
if (size >= 0 && cached?.size === size)
|
|
214
|
+
return { state: cached.state, size, torn: false, exists: true };
|
|
215
|
+
const log = size < 0 ? undefined : await readLog(path);
|
|
216
|
+
if (log === undefined)
|
|
217
|
+
return { state: stateOf([]), size: 0, torn: false, exists: false };
|
|
218
|
+
return { state: stateOf(log.entries), size: log.goodBytes, torn: log.tornBytes > 0, exists: true };
|
|
219
|
+
}
|
|
220
|
+
async #appendNow(key, inputs, options) {
|
|
221
|
+
const path = this.#pathOf(key);
|
|
222
|
+
let read = await this.#read(path);
|
|
223
|
+
// Writing an archived session brings its file back, so the log grows where the store looks for it. A live file with
|
|
224
|
+
// no complete line holds nothing that was acknowledged, so the archive replaces it.
|
|
225
|
+
if ((!read.exists || read.size === 0) && (await restoreArchive(this.#dir, key.userId, key.sessionId, path)))
|
|
226
|
+
read = await this.#read(path);
|
|
227
|
+
const { state, size, torn, exists } = read;
|
|
228
|
+
const plan = planAppend(state, inputs, options, this.#maxBytes);
|
|
229
|
+
if (plan.fresh.length === 0)
|
|
230
|
+
return structuredClone(plan.result);
|
|
231
|
+
const created = await mkdir(this.#userDir(key.userId), { recursive: true });
|
|
232
|
+
// A folder this call made is only durable once the folder above it is.
|
|
233
|
+
if (created !== undefined)
|
|
234
|
+
await Promise.all([syncFolder(join(created, "..")), syncFolder(this.#dir)]);
|
|
235
|
+
// The unterminated tail was never acknowledged, so cutting it off loses nothing, and keeps the next line apart from it.
|
|
236
|
+
if (torn)
|
|
237
|
+
await truncate(path, size);
|
|
238
|
+
const data = `${plan.fresh.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
|
|
239
|
+
await appendDurably(path, data, !exists);
|
|
240
|
+
this.#remember(path, size + Buffer.byteLength(data), extendState(state, plan.fresh));
|
|
241
|
+
return structuredClone(plan.result);
|
|
242
|
+
}
|
|
243
|
+
async append(key, inputs, options) {
|
|
244
|
+
checkKey(key);
|
|
245
|
+
const path = this.#pathOf(key);
|
|
246
|
+
// Expiry waits inside the queue, so adding it never changes the order in which calls to one session run.
|
|
247
|
+
return this.#queued(path, () => this.#afterExpiry(key, () => this.#appendNow(key, inputs, options)));
|
|
248
|
+
}
|
|
249
|
+
load(key, options) {
|
|
250
|
+
checkKey(key);
|
|
251
|
+
return this.#entries(key, options);
|
|
252
|
+
}
|
|
253
|
+
async *#entries(key, options) {
|
|
254
|
+
const path = this.#pathOf(key);
|
|
255
|
+
// A load waits for the appends already queued, so it never sees a line that is still being written.
|
|
256
|
+
const log = await this.#queued(path, () => this.#afterExpiry(key, () => this.#readAny(key, path)));
|
|
257
|
+
if (log === undefined)
|
|
258
|
+
return;
|
|
259
|
+
if (log.tornBytes > 0)
|
|
260
|
+
options?.onTornLine?.({ bytes: log.tornBytes });
|
|
261
|
+
const from = options?.fromSeq ?? 0;
|
|
262
|
+
for (const entry of log.entries)
|
|
263
|
+
if (entry.seq >= from)
|
|
264
|
+
yield entry;
|
|
265
|
+
}
|
|
266
|
+
loadRaw(key) {
|
|
267
|
+
checkKey(key);
|
|
268
|
+
return this.#raw(key);
|
|
269
|
+
}
|
|
270
|
+
async *#raw(key) {
|
|
271
|
+
const path = this.#pathOf(key);
|
|
272
|
+
const buffer = await this.#queued(path, () => this.#afterExpiry(key, async () => {
|
|
273
|
+
const live = await readFile(path).catch((error) => {
|
|
274
|
+
if (error.code !== "ENOENT")
|
|
275
|
+
throw error;
|
|
276
|
+
return undefined;
|
|
277
|
+
});
|
|
278
|
+
if (live?.includes(0x0a))
|
|
279
|
+
return live;
|
|
280
|
+
return (await readArchive(archivePath(this.#dir, key.userId, key.sessionId))) ?? live;
|
|
281
|
+
}));
|
|
282
|
+
if (buffer === undefined)
|
|
283
|
+
return;
|
|
284
|
+
const complete = buffer
|
|
285
|
+
.subarray(0, buffer.lastIndexOf(0x0a) + 1)
|
|
286
|
+
.toString("utf8")
|
|
287
|
+
.split("\n");
|
|
288
|
+
complete.pop();
|
|
289
|
+
for (const [index, line] of complete.entries()) {
|
|
290
|
+
try {
|
|
291
|
+
yield JSON.parse(line);
|
|
292
|
+
}
|
|
293
|
+
catch (cause) {
|
|
294
|
+
throw new HarnessError("session_corrupt", `${path} has a bad entry on line ${index + 1}`, { cause });
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
list(userId) {
|
|
299
|
+
checkKey({ userId, sessionId: "-" });
|
|
300
|
+
return this.#summaries(userId);
|
|
301
|
+
}
|
|
302
|
+
async *#summaries(userId) {
|
|
303
|
+
await this.#expire(userId);
|
|
304
|
+
const dir = this.#userDir(userId);
|
|
305
|
+
const archiveDir = join(this.#dir, ARCHIVE_DIR, encodeName(userId));
|
|
306
|
+
const live = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
307
|
+
const archived = await readdir(archiveDir, { withFileTypes: true }).catch(() => []);
|
|
308
|
+
const found = [];
|
|
309
|
+
const seen = new Set();
|
|
310
|
+
for (const [folder, files, extension] of [
|
|
311
|
+
[dir, live, EXTENSION],
|
|
312
|
+
[archiveDir, archived, ARCHIVE_EXTENSION],
|
|
313
|
+
]) {
|
|
314
|
+
for (const file of files.filter((entry) => entry.isFile() && entry.name.endsWith(extension))) {
|
|
315
|
+
const summary = await this.#summary(join(folder, file.name), extension, seen);
|
|
316
|
+
if (summary !== undefined)
|
|
317
|
+
found.push(summary);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
yield* found.sort(byAge);
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* One file's summary, or `undefined` for a file the store didn't write or can't read, so one bad file hides no other.
|
|
324
|
+
* A session that is both live and archived is mid-move, and the live file, which is read first, is whole. A live file
|
|
325
|
+
* with no complete line, or one that went after the folder was read, hides no archive, as `load()` reads past it too.
|
|
326
|
+
*/
|
|
327
|
+
async #summary(path, extension, seen) {
|
|
328
|
+
try {
|
|
329
|
+
const sessionId = decodeURIComponent(basename(path).slice(0, -extension.length));
|
|
330
|
+
if (seen.has(sessionId))
|
|
331
|
+
return undefined;
|
|
332
|
+
seen.add(sessionId);
|
|
333
|
+
if (extension === EXTENSION) {
|
|
334
|
+
const log = await readLog(path);
|
|
335
|
+
if (log === undefined || log.goodBytes === 0)
|
|
336
|
+
seen.delete(sessionId);
|
|
337
|
+
return summaryOf(sessionId, log?.entries ?? []);
|
|
338
|
+
}
|
|
339
|
+
return await this.#archivedSummary(sessionId, path);
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
return undefined;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
/** An archived file's summary, unpacked once and then kept while the file keeps its size and modification time. */
|
|
346
|
+
async #archivedSummary(sessionId, path) {
|
|
347
|
+
const info = await stat(path);
|
|
348
|
+
const cached = this.#archived.get(path);
|
|
349
|
+
if (cached !== undefined && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {
|
|
350
|
+
return structuredClone(cached.summary);
|
|
351
|
+
}
|
|
352
|
+
const bytes = await readArchive(path);
|
|
353
|
+
if (bytes === undefined)
|
|
354
|
+
return undefined;
|
|
355
|
+
const summary = summaryOf(sessionId, parseLog(bytes, path).entries);
|
|
356
|
+
this.#archived.delete(path);
|
|
357
|
+
this.#archived.set(path, { size: info.size, mtimeMs: info.mtimeMs, summary });
|
|
358
|
+
const oldest = this.#archived.keys().next().value;
|
|
359
|
+
if (this.#archived.size > MAX_CACHED_SUMMARIES && oldest !== undefined)
|
|
360
|
+
this.#archived.delete(oldest);
|
|
361
|
+
return structuredClone(summary);
|
|
362
|
+
}
|
|
363
|
+
async delete(key) {
|
|
364
|
+
checkKey(key);
|
|
365
|
+
const path = this.#pathOf(key);
|
|
366
|
+
// A delete waits for expiry like every other call, so a pass that is moving this file can't bring it back.
|
|
367
|
+
await this.#queued(path, () => this.#afterExpiry(key, async () => {
|
|
368
|
+
this.#cache.delete(path);
|
|
369
|
+
await rm(path, { force: true });
|
|
370
|
+
await rm(archivePath(this.#dir, key.userId, key.sessionId), { force: true });
|
|
371
|
+
}));
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* A store that keeps each session in `<dir>/<user>/<session>.jsonl`, one JSON entry per line. Each append is one write
|
|
376
|
+
* of whole lines followed by an `fsync`, so a crash leaves at most one unterminated last line, which `load()` skips and
|
|
377
|
+
* the next append cuts off before it writes. Names are percent-encoded, so no id can reach another user's folder. A
|
|
378
|
+
* log's state is cached by file size, so one process appends without rereading the file, and another process's write
|
|
379
|
+
* is seen on the next append. Appends to one session from one process run one at a time, and a second process needs the
|
|
380
|
+
* writer lock. With `retention`, expired sessions are gzipped into `<dir>/.archive/<user>/` or deleted, and without it
|
|
381
|
+
* no file is ever deleted or moved.
|
|
382
|
+
*/
|
|
383
|
+
export function jsonlStore(options) {
|
|
384
|
+
return new JsonlStore(options);
|
|
385
|
+
}
|
|
386
|
+
//# sourceMappingURL=jsonl-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl-store.js","sourceRoot":"","sources":["../../src/session/jsonl-store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAqB,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,cAAc,EACd,cAAc,EAEd,WAAW,EACX,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEL,KAAK,EACL,QAAQ,EACR,uBAAuB,EAEvB,UAAU,EACV,WAAW,EAGX,UAAU,EAIV,OAAO,EACP,SAAS,GACV,MAAM,YAAY,CAAC;AAEpB,MAAM,SAAS,GAAG,QAAQ,CAAC;AAmC3B,+HAA+H;AAC/H,SAAS,QAAQ,CAAC,MAAc,EAAE,IAAY;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnE,KAAK,CAAC,GAAG,EAAE,CAAC;IACZ,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,6GAA6G;YAC7G,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,CAAC;YACrF,MAAM,IAAI,YAAY,CACpB,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,iBAAiB,EACjD,GAAG,IAAI,QAAQ,MAAM,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,aAAa,YAAY,KAAK,GAAG,CAAC,EAAE,EAC5G,EAAE,KAAK,EAAE,CACV,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAY;IACjC,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QACzE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,oFAAoF;AACpF,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,IAAY,EAAE,KAAc;IACrE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IACD,IAAI,KAAK;QAAE,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,+HAA+H;AAC/H,KAAK,UAAU,UAAU,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,uCAAuC;IACzC,CAAC;AACH,CAAC;AAUD,wHAAwH;AACxH,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,kHAAkH;AAClH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,qFAAqF;AACrF,MAAM,cAAc,GAAG,MAAM,CAAC;AAU9B,MAAM,QAAQ,GAAG,CAAC,GAAe,EAAU,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AAE1F,MAAM,UAAU;IACL,IAAI,CAAS;IACb,SAAS,CAAS;IAClB,QAAQ,CAAoB;IAC5B,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,SAAS,GAAG,IAAI,GAAG,EAAkF,CAAC;IACtG,MAAM,GAAG,IAAI,GAAG,EAA6C,CAAC;IAC9D,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACvD,wGAAwG;IAC/F,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,YAAY,OAA0B;QACpC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC;QAClE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS;YAAE,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACzE,CAAC;IAED,IAAI;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IACzD,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,MAAc;QACpB,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QACpC,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC,IAAI,CAAC;QAC3G,MAAM,MAAM,GAAW;YACrB,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;gBACnD,IAAI,MAAM;oBAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,cAAc,CAAC;gBAC1D,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACxB,CAAC,CAAC;SACH,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,SAAoB;QAC/C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG;gBACd,GAAG,EAAE,IAAI,CAAC,IAAI;gBACd,MAAM;gBACN,SAAS;gBACT,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBAChC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;aAC1E,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YAClH,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM;gBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC3D,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAI,GAAe,EAAE,IAAsB;QAC3D,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/B,KAAK,IAAI,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,KAAK,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,CAAC;QAC5G,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,IAAI,KAAK,CAAC;gBAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;;gBACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,QAAQ,CAAC,MAAc;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1E,CAAC;IAED,OAAO,CAAC,KAAc;QACpB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,6FAA6F;QAC/F,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAe,EAAE,IAAY;QAC1C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACtF,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,QAAQ,CAAC,MAAc;QACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,CAAC,GAAe;QACrB,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,SAAS,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,uHAAuH;IACvH,OAAO,CAAI,IAAY,EAAE,IAAsB;QAC7C,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CACpB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7B,6GAA6G;QAC7G,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,IAAY,EAAE,IAAY,EAAE,KAAe;QACnD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,eAAe,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7F,CAAC;IAED,6GAA6G;IAC7G,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAChC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EACnB,GAAG,EAAE,CAAC,CAAC,CAAC,CACT,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,IAAI,CAAC,IAAI,MAAM,EAAE,IAAI,KAAK,IAAI;YAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACxG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC1F,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACrG,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAe,EAAE,MAA6B,EAAE,OAAkC;QACjG,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,oHAAoH;QACpH,oFAAoF;QACpF,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACzG,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAChE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,uEAAuE;QACvE,IAAI,OAAO,KAAK,SAAS;YAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvG,wHAAwH;QACxH,IAAI,IAAI;YAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAChF,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACrF,OAAO,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAe,EAAE,MAA6B,EAAE,OAAuB;QAClF,QAAQ,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,yGAAyG;QACzG,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACvG,CAAC;IAED,IAAI,CAAC,GAAe,EAAE,OAAqB;QACzC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAe,EAAE,OAAgC;QAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,oGAAoG;QACpG,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QACnG,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO;QAC9B,IAAI,GAAG,CAAC,SAAS,GAAG,CAAC;YAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,MAAM,IAAI,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QACnC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,OAAO;YAAE,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI;gBAAE,MAAM,KAAK,CAAC;IACtE,CAAC;IAED,OAAO,CAAC,GAAe;QACrB,QAAQ,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,GAAe;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,CAC3C,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;YAChC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAA4B,EAAE,EAAE;gBACvE,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;oBAAE,MAAM,KAAK,CAAC;gBACzC,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;YACH,IAAI,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YACtC,OAAO,CAAC,MAAM,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACxF,CAAC,CAAC,CACH,CAAC;QACF,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO;QACjC,MAAM,QAAQ,GAAG,MAAM;aACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACzC,QAAQ,CAAC,MAAM,CAAC;aAChB,KAAK,CAAC,IAAI,CAAC,CAAC;QACf,QAAQ,CAAC,GAAG,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,GAAG,IAAI,4BAA4B,KAAK,GAAG,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACvG,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAc;QACjB,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QACrC,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,CAAC,UAAU,CAAC,MAAc;QAC9B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI;YACvC,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC;YACtB,CAAC,UAAU,EAAE,QAAQ,EAAE,iBAAiB,CAAC;SACjC,EAAE,CAAC;YACX,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;gBAC7F,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;gBAC9E,IAAI,OAAO,KAAK,SAAS;oBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QACD,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,SAAiB,EAAE,IAAiB;QAC/D,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACjF,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,OAAO,SAAS,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACpB,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,CAAC;oBAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACrE,OAAO,SAAS,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,mHAAmH;IACnH,KAAK,CAAC,gBAAgB,CAAC,SAAiB,EAAE,IAAY;QACpD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YACzF,OAAO,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC1C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;QACpE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAClD,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,oBAAoB,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtG,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAe;QAC1B,QAAQ,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,2GAA2G;QAC3G,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,CAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;YAChC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzB,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChC,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CAAC,OAA0B;IACnD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC","sourcesContent":["/**\n * `jsonlStore()`: a `SessionStore` that keeps each session in one JSONL file, one entry per line, under a folder per\n * user. It needs Node's file system, so it has its own entry, `@buildanagent/harness/session/jsonl`, and the portable\n * `./session` entry stays free of it.\n */\nimport { Buffer } from \"node:buffer\";\nimport { mkdir, open, readdir, readFile, rm, stat, truncate } from \"node:fs/promises\";\nimport { basename, join } from \"node:path\";\nimport { isProtocolError, parseEntry, type SessionEntry } from \"@buildanagent/protocol\";\nimport { HarnessError } from \"../errors.js\";\nimport type { SessionLock } from \"./lock.js\";\nimport {\n ARCHIVE_DIR,\n ARCHIVE_EXTENSION,\n archivePath,\n checkRetention,\n expireSessions,\n type Retention,\n readArchive,\n restoreArchive,\n} from \"./retention.js\";\nimport {\n type AppendOptions,\n byAge,\n checkKey,\n DEFAULT_MAX_ENTRY_BYTES,\n type EntryInput,\n encodeName,\n extendState,\n type LoadOptions,\n type LogState,\n planAppend,\n type SessionKey,\n type SessionStore,\n type SessionSummary,\n stateOf,\n summaryOf,\n} from \"./store.js\";\n\nconst EXTENSION = \".jsonl\";\n\n/** The options of `jsonlStore()`. */\nexport interface JsonlStoreOptions {\n /** The folder that holds the sessions. It is created on the first write. */\n readonly dir: string;\n /** The bound on one entry's JSON, in bytes. Default 64 KiB. */\n readonly maxEntryBytes?: number;\n /**\n * How long to keep a session. Without it the store never deletes or moves a file. With it, the first time the store\n * touches a user's sessions, it archives or purges that user's expired ones (see `pruneSessions()`), and it never does\n * so in the background. That happens once per user for the life of the store: a long-running server calls\n * `pruneSessions()` on a schedule of its own. A pass that had an error runs again on the first touch at least a minute\n * after it ended. An archived session is still listed and loaded, and a write restores the file it writes to.\n */\n readonly retention?: Retention;\n /** The writer lock, so that expiry leaves a session whose writer is running. Only used with `retention`. */\n readonly lock?: SessionLock;\n /**\n * Called with each error of an expiry pass: one for each session that failed to expire, or the one that stopped the\n * pass. Expiry never stops the call that triggered it, and an error this callback throws is ignored.\n */\n readonly onRetentionError?: (error: unknown) => void;\n /** The clock expiry reads, for its cutoff and for when a failed pass may run again. Default `new Date()`. */\n readonly now?: () => Date;\n}\n\ninterface ReadLog {\n readonly entries: SessionEntry[];\n /** The size of the file up to its last complete line. */\n readonly goodBytes: number;\n /** The bytes of an unterminated last line, or 0. */\n readonly tornBytes: number;\n}\n\n/** Reads a log: a line counts when it ends in a newline, and a last line that doesn't is torn. Sizes are bytes of the file. */\nfunction parseLog(buffer: Buffer, path: string): ReadLog {\n const end = buffer.lastIndexOf(0x0a) + 1;\n const entries: SessionEntry[] = [];\n const lines = buffer.subarray(0, end).toString(\"utf8\").split(\"\\n\");\n lines.pop();\n for (const [index, line] of lines.entries()) {\n try {\n entries.push(parseEntry(line));\n } catch (cause) {\n // A header from another format isn't corruption: the file is whole and needs a migration or a newer harness.\n const format = isProtocolError(cause) && cause.code === \"unsupported_format_version\";\n throw new HarnessError(\n format ? \"unsupported_format\" : \"session_corrupt\",\n `${path} has ${format ? \"a log in a format this harness doesn't read\" : \"a bad entry\"} on line ${index + 1}`,\n { cause },\n );\n }\n }\n return { entries, goodBytes: end, tornBytes: buffer.length - end };\n}\n\nasync function readLog(path: string): Promise<ReadLog | undefined> {\n try {\n return parseLog(await readFile(path), path);\n } catch (error) {\n if ((error as NodeJS.ErrnoException).code === \"ENOENT\") return undefined;\n throw error;\n }\n}\n\n/** Writes `data` at the end of the file and flushes it to disk before returning. */\nasync function appendDurably(path: string, data: string, isNew: boolean): Promise<void> {\n const file = await open(path, \"a\");\n try {\n await file.appendFile(data);\n await file.sync();\n } finally {\n await file.close();\n }\n if (isNew) await syncFolder(join(path, \"..\"));\n}\n\n// A new file is only durable once its folder entry is. Some platforms refuse to open a folder, and then there's nothing to do.\nasync function syncFolder(dir: string): Promise<void> {\n try {\n const handle = await open(dir, \"r\");\n try {\n await handle.sync();\n } finally {\n await handle.close();\n }\n } catch {\n /* the platform can't flush a folder */\n }\n}\n\ninterface Read {\n readonly state: LogState;\n /** The size of the file up to its last complete line. */\n readonly size: number;\n readonly torn: boolean;\n readonly exists: boolean;\n}\n\n// Enough for the sessions a process is working on; a log that falls out is read again from its file on the next append.\nconst MAX_CACHED_LOGS = 64;\n// An archived file never changes in place, so its summary is kept by path, size and time; this bounds the memory.\nconst MAX_CACHED_SUMMARIES = 1024;\n// How long a failed expiry pass waits before a touch of the same user runs it again.\nconst RETRY_AFTER_MS = 60_000;\n\ninterface Expiry {\n readonly done: Promise<void>;\n /** Whether the pass has ended. */\n settled: boolean;\n /** When a failed pass may run again, set once the pass has ended with an error. */\n retryAt?: number;\n}\n\nconst busyName = (key: SessionKey): string => JSON.stringify([key.userId, key.sessionId]);\n\nclass JsonlStore implements SessionStore {\n readonly #dir: string;\n readonly #maxBytes: number;\n readonly #options: JsonlStoreOptions;\n readonly #expiry = new Map<string, Expiry>();\n readonly #archived = new Map<string, { size: number; mtimeMs: number; summary: SessionSummary | undefined }>();\n readonly #cache = new Map<string, { size: number; state: LogState }>();\n readonly #queues = new Map<string, Promise<unknown>>();\n /** How many calls past expiry are using each file, by user and file id; a pass keeps their sessions. */\n readonly #busy = new Map<string, number>();\n\n constructor(options: JsonlStoreOptions) {\n this.#dir = options.dir;\n this.#maxBytes = options.maxEntryBytes ?? DEFAULT_MAX_ENTRY_BYTES;\n this.#options = options;\n if (options.retention !== undefined) checkRetention(options.retention);\n }\n\n #now(): number {\n return (this.#options.now?.() ?? new Date()).getTime();\n }\n\n /**\n * Expires a user's sessions the first time this store touches them, and again after a failed pass once its wait is\n * over. A failure is reported and never thrown, so the returned promise always resolves.\n */\n #expire(userId: string): Promise<void> {\n const { retention } = this.#options;\n if (retention === undefined) return Promise.resolve();\n const known = this.#expiry.get(userId);\n if (known !== undefined && (known.retryAt === undefined || this.#now() < known.retryAt)) return known.done;\n const expiry: Expiry = {\n settled: false,\n done: this.#prune(userId, retention).then((failed) => {\n if (failed) expiry.retryAt = this.#now() + RETRY_AFTER_MS;\n expiry.settled = true;\n }),\n };\n this.#expiry.set(userId, expiry);\n return expiry.done;\n }\n\n /** One expiry pass. Resolves to whether it had an error, and never rejects. */\n async #prune(userId: string, retention: Retention): Promise<boolean> {\n try {\n const options = {\n dir: this.#dir,\n userId,\n retention,\n now: () => new Date(this.#now()),\n ...(this.#options.lock === undefined ? {} : { lock: this.#options.lock }),\n };\n const result = await expireSessions(options, (fileId) => this.#busy.has(busyName({ userId, sessionId: fileId })));\n for (const { error } of result.errors) this.#report(error);\n return result.errors.length > 0;\n } catch (error) {\n this.#report(error);\n return true;\n }\n }\n\n /**\n * Runs `task` once expiry for the key's user is over. A pass that another call started meanwhile runs first; once no\n * pass runs, the file is marked busy in the same turn, so a pass that starts later keeps its session until `task` ends.\n */\n async #afterExpiry<T>(key: SessionKey, task: () => Promise<T>): Promise<T> {\n await this.#expire(key.userId);\n for (let pass = this.#running(key.userId); pass !== undefined; pass = this.#running(key.userId)) await pass;\n const name = busyName(key);\n this.#busy.set(name, (this.#busy.get(name) ?? 0) + 1);\n try {\n return await task();\n } finally {\n const left = (this.#busy.get(name) ?? 1) - 1;\n if (left === 0) this.#busy.delete(name);\n else this.#busy.set(name, left);\n }\n }\n\n /** The pass running for the user, if one is. */\n #running(userId: string): Promise<void> | undefined {\n const expiry = this.#expiry.get(userId);\n return expiry === undefined || expiry.settled ? undefined : expiry.done;\n }\n\n #report(error: unknown): void {\n try {\n this.#options.onRetentionError?.(error);\n } catch {\n /* the callback's own failure must not stop the session, nor every later call for the user */\n }\n }\n\n /**\n * The log in the live file, or else in its archive. A live file with no complete line holds nothing that was\n * acknowledged, so it counts as missing.\n */\n async #readAny(key: SessionKey, path: string): Promise<ReadLog | undefined> {\n const live = await readLog(path);\n if (live !== undefined && live.goodBytes > 0) return live;\n const archived = await readArchive(archivePath(this.#dir, key.userId, key.sessionId));\n return archived === undefined ? live : parseLog(archived, path);\n }\n\n #userDir(userId: string): string {\n return join(this.#dir, encodeName(userId));\n }\n\n #pathOf(key: SessionKey): string {\n return join(this.#userDir(key.userId), `${encodeName(key.sessionId)}${EXTENSION}`);\n }\n\n /** Runs `task` after every earlier task on this file, so one process's appends, deletes and loads never interleave. */\n #queued<T>(path: string, task: () => Promise<T>): Promise<T> {\n const turn = (this.#queues.get(path) ?? Promise.resolve()).then(task);\n const tail = turn.then(\n () => undefined,\n () => undefined,\n );\n this.#queues.set(path, tail);\n // The entry is only needed while a task is waiting, so the map doesn't grow with every session ever touched.\n void tail.then(() => {\n if (this.#queues.get(path) === tail) this.#queues.delete(path);\n });\n return turn;\n }\n\n #remember(path: string, size: number, state: LogState): void {\n this.#cache.delete(path);\n this.#cache.set(path, { size, state });\n const oldest = this.#cache.keys().next().value;\n if (this.#cache.size > MAX_CACHED_LOGS && oldest !== undefined) this.#cache.delete(oldest);\n }\n\n /** The log's state, from the cache when the file is the size it was left at, and from the file otherwise. */\n async #read(path: string): Promise<Read> {\n const size = await stat(path).then(\n (info) => info.size,\n () => -1,\n );\n const cached = this.#cache.get(path);\n if (size >= 0 && cached?.size === size) return { state: cached.state, size, torn: false, exists: true };\n const log = size < 0 ? undefined : await readLog(path);\n if (log === undefined) return { state: stateOf([]), size: 0, torn: false, exists: false };\n return { state: stateOf(log.entries), size: log.goodBytes, torn: log.tornBytes > 0, exists: true };\n }\n\n async #appendNow(key: SessionKey, inputs: readonly EntryInput[], options: AppendOptions | undefined) {\n const path = this.#pathOf(key);\n let read = await this.#read(path);\n // Writing an archived session brings its file back, so the log grows where the store looks for it. A live file with\n // no complete line holds nothing that was acknowledged, so the archive replaces it.\n if ((!read.exists || read.size === 0) && (await restoreArchive(this.#dir, key.userId, key.sessionId, path)))\n read = await this.#read(path);\n const { state, size, torn, exists } = read;\n const plan = planAppend(state, inputs, options, this.#maxBytes);\n if (plan.fresh.length === 0) return structuredClone(plan.result);\n const created = await mkdir(this.#userDir(key.userId), { recursive: true });\n // A folder this call made is only durable once the folder above it is.\n if (created !== undefined) await Promise.all([syncFolder(join(created, \"..\")), syncFolder(this.#dir)]);\n // The unterminated tail was never acknowledged, so cutting it off loses nothing, and keeps the next line apart from it.\n if (torn) await truncate(path, size);\n const data = `${plan.fresh.map((entry) => JSON.stringify(entry)).join(\"\\n\")}\\n`;\n await appendDurably(path, data, !exists);\n this.#remember(path, size + Buffer.byteLength(data), extendState(state, plan.fresh));\n return structuredClone(plan.result);\n }\n\n async append(key: SessionKey, inputs: readonly EntryInput[], options?: AppendOptions): Promise<SessionEntry[]> {\n checkKey(key);\n const path = this.#pathOf(key);\n // Expiry waits inside the queue, so adding it never changes the order in which calls to one session run.\n return this.#queued(path, () => this.#afterExpiry(key, () => this.#appendNow(key, inputs, options)));\n }\n\n load(key: SessionKey, options?: LoadOptions): AsyncGenerator<SessionEntry> {\n checkKey(key);\n return this.#entries(key, options);\n }\n\n async *#entries(key: SessionKey, options: LoadOptions | undefined): AsyncGenerator<SessionEntry> {\n const path = this.#pathOf(key);\n // A load waits for the appends already queued, so it never sees a line that is still being written.\n const log = await this.#queued(path, () => this.#afterExpiry(key, () => this.#readAny(key, path)));\n if (log === undefined) return;\n if (log.tornBytes > 0) options?.onTornLine?.({ bytes: log.tornBytes });\n const from = options?.fromSeq ?? 0;\n for (const entry of log.entries) if (entry.seq >= from) yield entry;\n }\n\n loadRaw(key: SessionKey): AsyncGenerator<unknown> {\n checkKey(key);\n return this.#raw(key);\n }\n\n async *#raw(key: SessionKey): AsyncGenerator<unknown> {\n const path = this.#pathOf(key);\n const buffer = await this.#queued(path, () =>\n this.#afterExpiry(key, async () => {\n const live = await readFile(path).catch((error: NodeJS.ErrnoException) => {\n if (error.code !== \"ENOENT\") throw error;\n return undefined;\n });\n if (live?.includes(0x0a)) return live;\n return (await readArchive(archivePath(this.#dir, key.userId, key.sessionId))) ?? live;\n }),\n );\n if (buffer === undefined) return;\n const complete = buffer\n .subarray(0, buffer.lastIndexOf(0x0a) + 1)\n .toString(\"utf8\")\n .split(\"\\n\");\n complete.pop();\n for (const [index, line] of complete.entries()) {\n try {\n yield JSON.parse(line);\n } catch (cause) {\n throw new HarnessError(\"session_corrupt\", `${path} has a bad entry on line ${index + 1}`, { cause });\n }\n }\n }\n\n list(userId: string): AsyncGenerator<SessionSummary> {\n checkKey({ userId, sessionId: \"-\" });\n return this.#summaries(userId);\n }\n\n async *#summaries(userId: string): AsyncGenerator<SessionSummary> {\n await this.#expire(userId);\n const dir = this.#userDir(userId);\n const archiveDir = join(this.#dir, ARCHIVE_DIR, encodeName(userId));\n const live = await readdir(dir, { withFileTypes: true }).catch(() => []);\n const archived = await readdir(archiveDir, { withFileTypes: true }).catch(() => []);\n const found: SessionSummary[] = [];\n const seen = new Set<string>();\n for (const [folder, files, extension] of [\n [dir, live, EXTENSION],\n [archiveDir, archived, ARCHIVE_EXTENSION],\n ] as const) {\n for (const file of files.filter((entry) => entry.isFile() && entry.name.endsWith(extension))) {\n const summary = await this.#summary(join(folder, file.name), extension, seen);\n if (summary !== undefined) found.push(summary);\n }\n }\n yield* found.sort(byAge);\n }\n\n /**\n * One file's summary, or `undefined` for a file the store didn't write or can't read, so one bad file hides no other.\n * A session that is both live and archived is mid-move, and the live file, which is read first, is whole. A live file\n * with no complete line, or one that went after the folder was read, hides no archive, as `load()` reads past it too.\n */\n async #summary(path: string, extension: string, seen: Set<string>): Promise<SessionSummary | undefined> {\n try {\n const sessionId = decodeURIComponent(basename(path).slice(0, -extension.length));\n if (seen.has(sessionId)) return undefined;\n seen.add(sessionId);\n if (extension === EXTENSION) {\n const log = await readLog(path);\n if (log === undefined || log.goodBytes === 0) seen.delete(sessionId);\n return summaryOf(sessionId, log?.entries ?? []);\n }\n return await this.#archivedSummary(sessionId, path);\n } catch {\n return undefined;\n }\n }\n\n /** An archived file's summary, unpacked once and then kept while the file keeps its size and modification time. */\n async #archivedSummary(sessionId: string, path: string): Promise<SessionSummary | undefined> {\n const info = await stat(path);\n const cached = this.#archived.get(path);\n if (cached !== undefined && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {\n return structuredClone(cached.summary);\n }\n const bytes = await readArchive(path);\n if (bytes === undefined) return undefined;\n const summary = summaryOf(sessionId, parseLog(bytes, path).entries);\n this.#archived.delete(path);\n this.#archived.set(path, { size: info.size, mtimeMs: info.mtimeMs, summary });\n const oldest = this.#archived.keys().next().value;\n if (this.#archived.size > MAX_CACHED_SUMMARIES && oldest !== undefined) this.#archived.delete(oldest);\n return structuredClone(summary);\n }\n\n async delete(key: SessionKey): Promise<void> {\n checkKey(key);\n const path = this.#pathOf(key);\n // A delete waits for expiry like every other call, so a pass that is moving this file can't bring it back.\n await this.#queued(path, () =>\n this.#afterExpiry(key, async () => {\n this.#cache.delete(path);\n await rm(path, { force: true });\n await rm(archivePath(this.#dir, key.userId, key.sessionId), { force: true });\n }),\n );\n }\n}\n\n/**\n * A store that keeps each session in `<dir>/<user>/<session>.jsonl`, one JSON entry per line. Each append is one write\n * of whole lines followed by an `fsync`, so a crash leaves at most one unterminated last line, which `load()` skips and\n * the next append cuts off before it writes. Names are percent-encoded, so no id can reach another user's folder. A\n * log's state is cached by file size, so one process appends without rereading the file, and another process's write\n * is seen on the next append. Appends to one session from one process run one at a time, and a second process needs the\n * writer lock. With `retention`, expired sessions are gzipped into `<dir>/.archive/<user>/` or deleted, and without it\n * no file is ever deleted or moved.\n */\nexport function jsonlStore(options: JsonlStoreOptions): SessionStore {\n return new JsonlStore(options);\n}\n"]}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@buildanagent/harness/session/jsonl`: the file-backed session store and writer lock, kept apart from the portable
|
|
3
|
+
* session entry.
|
|
4
|
+
*/
|
|
5
|
+
export { type FileIndexOptions, fileIndex } from "./file-index.js";
|
|
6
|
+
export { defaultKeyPath, fileKeyStore } from "./file-key-store.js";
|
|
7
|
+
export { type FileLockOptions, fileLock, type LockRecord, type RemovedLock, unlockSession } from "./file-lock.js";
|
|
8
|
+
export { type JsonlStoreOptions, jsonlStore } from "./jsonl-store.js";
|
|
9
|
+
export { type KeptReason, type PruneOptions, type PruneResult, pruneSessions, type Retention } from "./retention.js";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@buildanagent/harness/session/jsonl`: the file-backed session store and writer lock, kept apart from the portable
|
|
3
|
+
* session entry.
|
|
4
|
+
*/
|
|
5
|
+
export { fileIndex } from "./file-index.js";
|
|
6
|
+
export { defaultKeyPath, fileKeyStore } from "./file-key-store.js";
|
|
7
|
+
export { fileLock, unlockSession } from "./file-lock.js";
|
|
8
|
+
export { jsonlStore } from "./jsonl-store.js";
|
|
9
|
+
export { pruneSessions } from "./retention.js";
|
|
10
|
+
//# sourceMappingURL=jsonl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/session/jsonl.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAyB,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAwB,QAAQ,EAAqC,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAClH,OAAO,EAA0B,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAwD,aAAa,EAAkB,MAAM,gBAAgB,CAAC","sourcesContent":["/**\n * `@buildanagent/harness/session/jsonl`: the file-backed session store and writer lock, kept apart from the portable\n * session entry.\n */\n\nexport { type FileIndexOptions, fileIndex } from \"./file-index.js\";\nexport { defaultKeyPath, fileKeyStore } from \"./file-key-store.js\";\nexport { type FileLockOptions, fileLock, type LockRecord, type RemovedLock, unlockSession } from \"./file-lock.js\";\nexport { type JsonlStoreOptions, jsonlStore } from \"./jsonl-store.js\";\nexport { type KeptReason, type PruneOptions, type PruneResult, pruneSessions, type Retention } from \"./retention.js\";\n"]}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** The session lock's interface: one writer per session. The file-backed implementation is in `./file-lock.ts`. */
|
|
2
|
+
import type { SessionKey } from "./store.js";
|
|
3
|
+
/** What `acquire()` returns to the writer that got the lock. */
|
|
4
|
+
export interface LockHandle {
|
|
5
|
+
/** A random value that names this hold, so a release or a renewal only ever touches its own lock. */
|
|
6
|
+
readonly token: string;
|
|
7
|
+
/**
|
|
8
|
+
* Whether this writer still holds the lock. It turns `false` when a renewal finds the lock gone or holding another writer's
|
|
9
|
+
* token, for example after the lease lapsed and someone took over. A writer that sees `false` must stop writing.
|
|
10
|
+
*/
|
|
11
|
+
held(): boolean;
|
|
12
|
+
/** Gives the lock up and stops renewing it. It is safe to call again, and it never frees another writer's lock. */
|
|
13
|
+
release(): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
/** Keeps two writers from extending one session's log at once. Readers never lock. */
|
|
16
|
+
export interface SessionLock {
|
|
17
|
+
/** The lock for this session, or `"would_block"` at once when another writer holds it. It never waits. */
|
|
18
|
+
acquire(key: SessionKey): Promise<LockHandle | "would_block">;
|
|
19
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lock.js","sourceRoot":"","sources":["../../src/session/lock.ts"],"names":[],"mappings":"","sourcesContent":["/** The session lock's interface: one writer per session. The file-backed implementation is in `./file-lock.ts`. */\nimport type { SessionKey } from \"./store.js\";\n\n/** What `acquire()` returns to the writer that got the lock. */\nexport interface LockHandle {\n /** A random value that names this hold, so a release or a renewal only ever touches its own lock. */\n readonly token: string;\n /**\n * Whether this writer still holds the lock. It turns `false` when a renewal finds the lock gone or holding another writer's\n * token, for example after the lease lapsed and someone took over. A writer that sees `false` must stop writing.\n */\n held(): boolean;\n /** Gives the lock up and stops renewing it. It is safe to call again, and it never frees another writer's lock. */\n release(): Promise<void>;\n}\n\n/** Keeps two writers from extending one session's log at once. Readers never lock. */\nexport interface SessionLock {\n /** The lock for this session, or `\"would_block\"` at once when another writer holds it. It never waits. */\n acquire(key: SessionKey): Promise<LockHandle | \"would_block\">;\n}\n"]}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type SessionStore } from "./store.js";
|
|
2
|
+
/** The options of `memoryStore()`. */
|
|
3
|
+
export interface MemoryStoreOptions {
|
|
4
|
+
/** The bound on one entry's JSON, in bytes. Default 64 KiB. */
|
|
5
|
+
readonly maxEntryBytes?: number;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* A store that keeps sessions in memory, for tests and for a process that doesn't need them after it ends. It follows
|
|
9
|
+
* every rule of the interface and runs anywhere, workerd included. Entries are copied on the way in and out, so no
|
|
10
|
+
* caller can change a stored one.
|
|
11
|
+
*/
|
|
12
|
+
export declare function memoryStore(options?: MemoryStoreOptions): SessionStore;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { byAge, checkKey, DEFAULT_MAX_ENTRY_BYTES, extendState, planAppend, stateOf, summaryOf, } from "./store.js";
|
|
2
|
+
/**
|
|
3
|
+
* A store that keeps sessions in memory, for tests and for a process that doesn't need them after it ends. It follows
|
|
4
|
+
* every rule of the interface and runs anywhere, workerd included. Entries are copied on the way in and out, so no
|
|
5
|
+
* caller can change a stored one.
|
|
6
|
+
*/
|
|
7
|
+
export function memoryStore(options = {}) {
|
|
8
|
+
const maxBytes = options.maxEntryBytes ?? DEFAULT_MAX_ENTRY_BYTES;
|
|
9
|
+
const users = new Map();
|
|
10
|
+
const sessionsOf = (userId) => users.get(userId);
|
|
11
|
+
async function* entriesOf(key, loadOptions) {
|
|
12
|
+
const from = loadOptions?.fromSeq ?? 0;
|
|
13
|
+
// A snapshot, as the JSONL store's load is, so an append made while a caller iterates doesn't extend the loop.
|
|
14
|
+
const snapshot = (sessionsOf(key.userId)?.get(key.sessionId)?.entries ?? []).filter((entry) => entry.seq >= from);
|
|
15
|
+
for (const entry of structuredClone(snapshot))
|
|
16
|
+
yield entry;
|
|
17
|
+
}
|
|
18
|
+
async function* summaries(userId) {
|
|
19
|
+
const found = [];
|
|
20
|
+
for (const [sessionId, log] of sessionsOf(userId) ?? []) {
|
|
21
|
+
const summary = summaryOf(sessionId, log.entries);
|
|
22
|
+
if (summary !== undefined)
|
|
23
|
+
found.push(summary);
|
|
24
|
+
}
|
|
25
|
+
yield* found.sort(byAge);
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
async append(key, inputs, appendOptions) {
|
|
29
|
+
checkKey(key);
|
|
30
|
+
const log = sessionsOf(key.userId)?.get(key.sessionId) ?? { entries: [], state: stateOf([]) };
|
|
31
|
+
const plan = planAppend(log.state, inputs, appendOptions, maxBytes);
|
|
32
|
+
if (plan.fresh.length > 0) {
|
|
33
|
+
const stored = structuredClone(plan.fresh);
|
|
34
|
+
log.entries.push(...stored);
|
|
35
|
+
log.state = extendState(log.state, stored);
|
|
36
|
+
const sessions = sessionsOf(key.userId) ?? new Map();
|
|
37
|
+
users.set(key.userId, sessions);
|
|
38
|
+
sessions.set(key.sessionId, log);
|
|
39
|
+
}
|
|
40
|
+
return structuredClone(plan.result);
|
|
41
|
+
},
|
|
42
|
+
load(key, loadOptions) {
|
|
43
|
+
checkKey(key);
|
|
44
|
+
return entriesOf(key, loadOptions);
|
|
45
|
+
},
|
|
46
|
+
list(userId) {
|
|
47
|
+
checkKey({ userId, sessionId: "-" });
|
|
48
|
+
return summaries(userId);
|
|
49
|
+
},
|
|
50
|
+
loadRaw(key) {
|
|
51
|
+
checkKey(key);
|
|
52
|
+
return entriesOf(key, undefined);
|
|
53
|
+
},
|
|
54
|
+
async delete(key) {
|
|
55
|
+
checkKey(key);
|
|
56
|
+
sessionsOf(key.userId)?.delete(key.sessionId);
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=memory-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-store.js","sourceRoot":"","sources":["../../src/session/memory-store.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,EACL,QAAQ,EACR,uBAAuB,EACvB,WAAW,EAGX,UAAU,EAIV,OAAO,EACP,SAAS,GACV,MAAM,YAAY,CAAC;AAcpB;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,OAAO,GAAuB,EAAE;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzD,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,GAAe,EAAE,WAAoC;QAC7E,MAAM,IAAI,GAAG,WAAW,EAAE,OAAO,IAAI,CAAC,CAAC;QACvC,+GAA+G;QAC/G,MAAM,QAAQ,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;QAClH,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,QAAQ,CAAC;YAAE,MAAM,KAAK,CAAC;IAC7D,CAAC;IACD,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,MAAc;QACtC,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,OAAO,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;QACD,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO;QACL,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa;YACrC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC9F,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YACpE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAC5B,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC3C,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;gBACrE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,OAAO,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,WAAW;YACnB,QAAQ,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,MAAM;YACT,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YACrC,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,GAAG;YACT,QAAQ,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACnC,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG;YACd,QAAQ,CAAC,GAAG,CAAC,CAAC;YACd,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { SessionEntry } from \"@buildanagent/protocol\";\nimport {\n byAge,\n checkKey,\n DEFAULT_MAX_ENTRY_BYTES,\n extendState,\n type LoadOptions,\n type LogState,\n planAppend,\n type SessionKey,\n type SessionStore,\n type SessionSummary,\n stateOf,\n summaryOf,\n} from \"./store.js\";\n\n/** One session's entries and the index over them. */\ninterface Stored {\n readonly entries: SessionEntry[];\n state: LogState;\n}\n\n/** The options of `memoryStore()`. */\nexport interface MemoryStoreOptions {\n /** The bound on one entry's JSON, in bytes. Default 64 KiB. */\n readonly maxEntryBytes?: number;\n}\n\n/**\n * A store that keeps sessions in memory, for tests and for a process that doesn't need them after it ends. It follows\n * every rule of the interface and runs anywhere, workerd included. Entries are copied on the way in and out, so no\n * caller can change a stored one.\n */\nexport function memoryStore(options: MemoryStoreOptions = {}): SessionStore {\n const maxBytes = options.maxEntryBytes ?? DEFAULT_MAX_ENTRY_BYTES;\n const users = new Map<string, Map<string, Stored>>();\n const sessionsOf = (userId: string) => users.get(userId);\n async function* entriesOf(key: SessionKey, loadOptions: LoadOptions | undefined) {\n const from = loadOptions?.fromSeq ?? 0;\n // A snapshot, as the JSONL store's load is, so an append made while a caller iterates doesn't extend the loop.\n const snapshot = (sessionsOf(key.userId)?.get(key.sessionId)?.entries ?? []).filter((entry) => entry.seq >= from);\n for (const entry of structuredClone(snapshot)) yield entry;\n }\n async function* summaries(userId: string) {\n const found: SessionSummary[] = [];\n for (const [sessionId, log] of sessionsOf(userId) ?? []) {\n const summary = summaryOf(sessionId, log.entries);\n if (summary !== undefined) found.push(summary);\n }\n yield* found.sort(byAge);\n }\n return {\n async append(key, inputs, appendOptions) {\n checkKey(key);\n const log = sessionsOf(key.userId)?.get(key.sessionId) ?? { entries: [], state: stateOf([]) };\n const plan = planAppend(log.state, inputs, appendOptions, maxBytes);\n if (plan.fresh.length > 0) {\n const stored = structuredClone(plan.fresh);\n log.entries.push(...stored);\n log.state = extendState(log.state, stored);\n const sessions = sessionsOf(key.userId) ?? new Map<string, Stored>();\n users.set(key.userId, sessions);\n sessions.set(key.sessionId, log);\n }\n return structuredClone(plan.result);\n },\n load(key, loadOptions) {\n checkKey(key);\n return entriesOf(key, loadOptions);\n },\n list(userId) {\n checkKey({ userId, sessionId: \"-\" });\n return summaries(userId);\n },\n loadRaw(key) {\n checkKey(key);\n return entriesOf(key, undefined);\n },\n async delete(key) {\n checkKey(key);\n sessionsOf(key.userId)?.delete(key.sessionId);\n },\n };\n}\n"]}
|