owenloop 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +144 -35
- package/bin/owenloop.mjs +49 -1
- package/dist/packages/work/src/agent/brief.d.ts +29 -4
- package/dist/packages/work/src/agent/brief.js +49 -4
- package/dist/packages/work/src/agent/capability-model.d.ts +123 -0
- package/dist/packages/work/src/agent/capability-model.js +148 -0
- package/dist/packages/work/src/agent/loop.d.ts +56 -7
- package/dist/packages/work/src/agent/loop.js +166 -9
- package/dist/packages/work/src/bundle/cache.d.ts +7 -5
- package/dist/packages/work/src/bundle/cache.js +20 -6
- package/dist/packages/work/src/bundle/fetch.d.ts +7 -5
- package/dist/packages/work/src/bundle/fetch.js +27 -7
- package/dist/packages/work/src/bundle/types.d.ts +8 -4
- package/dist/packages/work/src/consumed-verifier.d.ts +45 -0
- package/dist/packages/work/src/consumed-verifier.js +205 -0
- package/dist/packages/work/src/exec/instructions.d.ts +99 -0
- package/dist/packages/work/src/exec/instructions.js +323 -0
- package/dist/packages/work/src/exec/loop.d.ts +29 -1
- package/dist/packages/work/src/exec/loop.js +313 -35
- package/dist/packages/work/src/exec/payload.d.ts +29 -0
- package/dist/packages/work/src/exec/payload.js +54 -0
- package/dist/packages/work/src/exec/receipt.d.ts +9 -6
- package/dist/packages/work/src/exec/receipt.js +12 -1
- package/dist/packages/work/src/exec/runner.d.ts +7 -1
- package/dist/packages/work/src/exec/runner.js +84 -3
- package/dist/packages/work/src/harness/child-env.js +7 -0
- package/dist/packages/work/src/harness/claude.d.ts +20 -8
- package/dist/packages/work/src/harness/claude.js +260 -58
- package/dist/packages/work/src/harness/codex.d.ts +5 -4
- package/dist/packages/work/src/harness/codex.js +169 -41
- package/dist/packages/work/src/harness/contract.d.ts +36 -11
- package/dist/packages/work/src/harness/fake.js +1 -0
- package/dist/packages/work/src/harness/jsonrpc-stdio.js +7 -0
- package/dist/packages/work/src/harness/permissions.d.ts +16 -3
- package/dist/packages/work/src/harness/permissions.js +102 -13
- package/dist/packages/work/src/harness/session-store.d.ts +30 -10
- package/dist/packages/work/src/harness/session-store.js +317 -89
- package/dist/packages/work/src/hold/mcp.d.ts +25 -5
- package/dist/packages/work/src/hold/mcp.js +134 -12
- package/dist/packages/work/src/hub/client.d.ts +8 -1
- package/dist/packages/work/src/hub/client.js +15 -1
- package/dist/packages/work/src/hub/types.d.ts +137 -33
- package/dist/packages/work/src/hub/types.js +8 -10
- package/dist/packages/work/src/mcp/server.js +1 -1
- package/dist/packages/work/src/owenloop-bin.d.ts +1 -0
- package/dist/packages/work/src/owenloop-bin.js +18 -0
- package/dist/packages/work/src/roles/agent-run.d.ts +12 -6
- package/dist/packages/work/src/roles/agent-run.js +115 -34
- package/dist/packages/work/src/roles/exec.d.ts +8 -0
- package/dist/packages/work/src/roles/exec.js +27 -2
- package/dist/packages/work/src/roles/hold.d.ts +6 -0
- package/dist/packages/work/src/roles/hold.js +83 -13
- package/dist/packages/work/src/roles/lint.d.ts +3 -4
- package/dist/packages/work/src/roles/lint.js +41 -7
- package/dist/packages/work/src/roles/settings.js +19 -2
- package/dist/packages/work/src/roles/shift.d.ts +1 -0
- package/dist/packages/work/src/roles/shift.js +36 -2
- package/dist/packages/work/src/settings/settings.d.ts +70 -1
- package/dist/packages/work/src/settings/settings.js +90 -4
- package/dist/packages/work/src/shift/logretention.d.ts +253 -0
- package/dist/packages/work/src/shift/logretention.js +407 -0
- package/dist/packages/work/src/shift/logsink.d.ts +14 -0
- package/dist/packages/work/src/shift/logsink.js +98 -0
- package/dist/packages/work/src/shift/loop.d.ts +32 -4
- package/dist/packages/work/src/shift/loop.js +687 -166
- package/dist/packages/work/src/shift/protocol.d.ts +128 -1
- package/dist/packages/work/src/shift/protocol.js +14 -0
- package/dist/packages/work/src/shift/runtime.d.ts +16 -0
- package/dist/packages/work/src/shift/runtime.js +291 -5
- package/dist/packages/work/src/shift/server.d.ts +13 -0
- package/dist/packages/work/src/shift/server.js +61 -36
- package/dist/packages/work/src/shift/spawn.d.ts +115 -19
- package/dist/packages/work/src/shift/spawn.js +234 -30
- package/dist/packages/work/src/shift/state.d.ts +76 -49
- package/dist/packages/work/src/shift/state.js +350 -103
- package/dist/packages/work/src/shift/truncate.d.ts +35 -0
- package/dist/packages/work/src/shift/truncate.js +73 -0
- package/dist/packages/work/src/submit-proof.d.ts +61 -0
- package/dist/packages/work/src/submit-proof.js +119 -0
- package/dist/packages/work/src/submit-value.d.ts +48 -0
- package/dist/packages/work/src/submit-value.js +118 -0
- package/dist/packages/work/src/usage.d.ts +1 -1
- package/dist/packages/work/src/usage.js +11 -0
- package/dist/src/add.d.ts +64 -259
- package/dist/src/add.js +81 -615
- package/dist/src/archive.d.ts +97 -0
- package/dist/src/archive.js +591 -0
- package/dist/src/bundle/index.d.ts +64 -0
- package/dist/src/bundle/index.js +508 -0
- package/dist/src/bundle/manifest.d.ts +50 -0
- package/dist/src/bundle/manifest.js +435 -0
- package/dist/src/bundle/runtime.d.ts +26 -0
- package/dist/src/bundle/runtime.js +64 -0
- package/dist/src/bundle/tar.d.ts +62 -0
- package/dist/src/bundle/tar.js +238 -0
- package/dist/src/bundle/types.d.ts +105 -0
- package/dist/src/bundle/types.js +21 -0
- package/dist/src/capabilities.d.ts +93 -0
- package/dist/src/capabilities.js +101 -0
- package/dist/src/cli.d.ts +90 -20
- package/dist/src/cli.js +2421 -374
- package/dist/src/credentials.d.ts +4 -3
- package/dist/src/credentials.js +21 -19
- package/dist/src/crypto/allowed-signers.d.ts +68 -0
- package/dist/src/crypto/allowed-signers.js +307 -0
- package/dist/src/crypto/canonical.d.ts +8 -0
- package/dist/src/crypto/canonical.js +92 -0
- package/dist/src/crypto/chain.d.ts +65 -0
- package/dist/src/crypto/chain.js +377 -0
- package/dist/src/crypto/dsse.d.ts +209 -0
- package/dist/src/crypto/dsse.js +282 -0
- package/dist/src/crypto/enrollment.d.ts +78 -0
- package/dist/src/crypto/enrollment.js +186 -0
- package/dist/src/crypto/index.d.ts +48 -0
- package/dist/src/crypto/index.js +44 -0
- package/dist/src/crypto/keys.d.ts +258 -0
- package/dist/src/crypto/keys.js +908 -0
- package/dist/src/crypto/org-root.d.ts +33 -0
- package/dist/src/crypto/org-root.js +78 -0
- package/dist/src/crypto/origin-rules.d.ts +56 -0
- package/dist/src/crypto/origin-rules.js +143 -0
- package/dist/src/crypto/policy-floor.d.ts +98 -0
- package/dist/src/crypto/policy-floor.js +353 -0
- package/dist/src/crypto/records.d.ts +319 -0
- package/dist/src/crypto/records.js +168 -0
- package/dist/src/crypto/scope.d.ts +49 -0
- package/dist/src/crypto/scope.js +83 -0
- package/dist/src/crypto/ssh.d.ts +171 -0
- package/dist/src/crypto/ssh.js +396 -0
- package/dist/src/crypto/submission.d.ts +33 -0
- package/dist/src/crypto/submission.js +47 -0
- package/dist/src/crypto/trust-roots.d.ts +29 -0
- package/dist/src/crypto/trust-roots.js +39 -0
- package/dist/src/crypto/verify-consumed.d.ts +67 -0
- package/dist/src/crypto/verify-consumed.js +259 -0
- package/dist/src/crypto/verify-origin.d.ts +50 -0
- package/dist/src/crypto/verify-origin.js +151 -0
- package/dist/src/crypto/verify-publication.d.ts +60 -0
- package/dist/src/crypto/verify-publication.js +150 -0
- package/dist/src/defs.d.ts +24 -1
- package/dist/src/defs.js +335 -27
- package/dist/src/engine.d.ts +211 -6
- package/dist/src/engine.js +459 -59
- package/dist/src/factory.d.ts +16 -0
- package/dist/src/factory.js +16 -4
- package/dist/src/global-config.d.ts +59 -0
- package/dist/src/global-config.js +84 -0
- package/dist/src/hub.d.ts +102 -32
- package/dist/src/hub.js +147 -58
- package/dist/src/index.d.ts +19 -2
- package/dist/src/index.js +23 -2
- package/dist/src/install.d.ts +538 -0
- package/dist/src/install.js +1981 -0
- package/dist/src/lock.d.ts +15 -57
- package/dist/src/lock.js +251 -143
- package/dist/src/mcp/serve.d.ts +46 -22
- package/dist/src/mcp/serve.js +174 -44
- package/dist/src/mcp/server.js +12 -1
- package/dist/src/model.d.ts +51 -0
- package/dist/src/model.js +157 -1
- package/dist/src/order-resolver.d.ts +184 -0
- package/dist/src/order-resolver.js +252 -0
- package/dist/src/package-version.d.ts +10 -0
- package/dist/src/package-version.js +42 -0
- package/dist/src/paths.d.ts +14 -1
- package/dist/src/paths.js +18 -0
- package/dist/src/schemas/enrollment-grant.v1.schema.json +90 -0
- package/dist/src/schemas/index.d.ts +1433 -0
- package/dist/src/schemas/index.js +31 -0
- package/dist/src/schemas/order.v1.schema.json +120 -0
- package/dist/src/schemas/origin.v1.schema.json +68 -0
- package/dist/src/schemas/policy-floor.v1.schema.json +29 -0
- package/dist/src/schemas/publication.v1.schema.json +31 -0
- package/dist/src/schemas/revocation.v1.schema.json +39 -0
- package/dist/src/schemas/submission.v1.schema.json +61 -0
- package/dist/src/store/def-source.d.ts +52 -0
- package/dist/src/store/def-source.js +335 -0
- package/dist/src/store/index-file.d.ts +69 -0
- package/dist/src/store/index-file.js +151 -0
- package/dist/src/store/index.d.ts +27 -0
- package/dist/src/store/index.js +22 -0
- package/dist/src/store/ingestor.d.ts +43 -0
- package/dist/src/store/ingestor.js +211 -0
- package/dist/src/store/install.d.ts +228 -0
- package/dist/src/store/install.js +555 -0
- package/dist/src/store/instruction-source.d.ts +49 -0
- package/dist/src/store/instruction-source.js +298 -0
- package/dist/src/store/origin-evidence.d.ts +21 -0
- package/dist/src/store/origin-evidence.js +66 -0
- package/dist/src/store/pre-commit-verifier.d.ts +68 -0
- package/dist/src/store/pre-commit-verifier.js +418 -0
- package/dist/src/store/resolve.d.ts +138 -0
- package/dist/src/store/resolve.js +363 -0
- package/dist/src/store/types.d.ts +288 -0
- package/dist/src/store/types.js +356 -0
- package/dist/src/store.d.ts +33 -1
- package/dist/src/store.js +57 -4
- package/dist/src/types.d.ts +146 -12
- package/dist/src/untar.d.ts +8 -23
- package/dist/src/untar.js +10 -136
- package/docs/agent-runner.md +86 -0
- package/docs/authoring.md +64 -6
- package/docs/bundles.md +350 -0
- package/docs/cli.md +1298 -106
- package/docs/crypto.md +832 -0
- package/docs/design.md +342 -20
- package/docs/embedding.md +132 -25
- package/docs/shift-logs.md +642 -0
- package/docs/wire-contracts.md +395 -0
- package/package.json +6 -1
- package/plugins/claude-code/.claude-plugin/marketplace.json +14 -0
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +8 -0
- package/plugins/claude-code/plugin/.mcp.json +14 -0
- package/plugins/claude-code/plugin/hooks/hooks.json +26 -0
- package/plugins/claude-code/plugin/hooks/session-end.sh +40 -0
- package/plugins/claude-code/plugin/hooks/session-start.sh +52 -0
- package/plugins/claude-code/plugin/skills/author/SKILL.md +181 -0
- package/plugins/claude-code/plugin/skills/conduct/SKILL.md +106 -0
- package/plugins/claude-code/plugin/skills/shift/SKILL.md +105 -0
- package/plugins/codex/.agents/plugins/marketplace.json +21 -0
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +7 -0
- package/plugins/codex/plugins/owenloop/.mcp.json +20 -0
- package/plugins/codex/plugins/owenloop/hooks/hooks.json +26 -0
- package/plugins/codex/plugins/owenloop/hooks/session-end.sh +40 -0
- package/plugins/codex/plugins/owenloop/hooks/session-start.sh +52 -0
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +181 -0
- package/plugins/codex/plugins/owenloop/skills/conduct/SKILL.md +106 -0
- package/plugins/codex/plugins/owenloop/skills/shift/SKILL.md +105 -0
|
@@ -40,21 +40,38 @@ export interface SessionRecord {
|
|
|
40
40
|
updatedAt: number;
|
|
41
41
|
}
|
|
42
42
|
export interface SessionStoreOptions {
|
|
43
|
-
/** Called once per skipped corrupt
|
|
43
|
+
/** Called once per skipped line — either corrupt bytes that would not parse
|
|
44
|
+
* ("skipping corrupt record") or a parsed object that failed the schema
|
|
45
|
+
* ("skipping invalid record"). Default writes to stderr. */
|
|
44
46
|
warn?: (line: string) => void;
|
|
47
|
+
/** Test barrier invoked under the writer lock after a retirement snapshot. */
|
|
48
|
+
afterWriterSnapshot?: () => void;
|
|
45
49
|
/** Compaction threshold in bytes, checked after each append. Default 2 MB. */
|
|
46
50
|
maxBytes?: number;
|
|
51
|
+
/** Clock used to detect an abandoned unterminated tail. Default `Date.now`. */
|
|
52
|
+
now?: () => number;
|
|
53
|
+
/** Unchanged-tail grace before one warning is emitted. Default 5 seconds. */
|
|
54
|
+
unterminatedTailGraceMs?: number;
|
|
55
|
+
/** Injectable durability primitive for focused tests. Default `fsyncSync`. */
|
|
56
|
+
sync?: (fd: number) => void;
|
|
47
57
|
}
|
|
48
58
|
/** Default compaction threshold — plan §3's 2 MB. */
|
|
49
59
|
export declare const DEFAULT_MAX_BYTES = 2000000;
|
|
60
|
+
/** Grace for a concurrent writer to finish an unterminated JSONL append. */
|
|
61
|
+
export declare const DEFAULT_UNTERMINATED_TAIL_GRACE_MS = 5000;
|
|
50
62
|
/** `<cacheDir>/sessions.jsonl`. */
|
|
51
63
|
export declare function sessionsPath(cacheDir: string): string;
|
|
52
64
|
/** The composite order id `<workflow>/<run>` — the one place it is built. */
|
|
53
65
|
export declare function orderId(workflow: string, run: string): string;
|
|
54
66
|
/**
|
|
55
67
|
* Every valid record in `file`, oldest first. A missing/unreadable file reads as
|
|
56
|
-
* `[]`.
|
|
57
|
-
*
|
|
68
|
+
* `[]`. A trailing newline is the record-commit marker: an unterminated final
|
|
69
|
+
* tail may be a concurrent writer still appending. File modification time starts
|
|
70
|
+
* the grace, so a recently modified tail is ignored silently while an already-old
|
|
71
|
+
* tail warns on the first read even in a fresh process. File identity, size, tail,
|
|
72
|
+
* or modification changes restart the observation; a newline clears it. Blank lines are
|
|
73
|
+
* skipped silently; every other unusable COMPLETE line is skipped and reported
|
|
74
|
+
* through `warn` with its 1-indexed line number. Never throws.
|
|
58
75
|
*/
|
|
59
76
|
export declare function readSessions(file: string, opts?: SessionStoreOptions): SessionRecord[];
|
|
60
77
|
/**
|
|
@@ -67,7 +84,8 @@ export declare function latestFor(file: string, workflow: string, run: string, s
|
|
|
67
84
|
* Rewrite `file` with only the last record per `(workflow, run, step)`,
|
|
68
85
|
* preserving FIRST-SEEN key order, via temp file + rename. Corrupt lines are
|
|
69
86
|
* dropped in the process (they were already unreadable). A missing file is a
|
|
70
|
-
* no-op.
|
|
87
|
+
* no-op. The complete read/rewrite/rename transaction shares the same lock as
|
|
88
|
+
* append, so a successful append cannot be replaced by this snapshot.
|
|
71
89
|
*/
|
|
72
90
|
export declare function compact(file: string, opts?: SessionStoreOptions): void;
|
|
73
91
|
/**
|
|
@@ -82,8 +100,8 @@ export declare function compact(file: string, opts?: SessionStoreOptions): void;
|
|
|
82
100
|
*
|
|
83
101
|
* WHY A NEW ROW RATHER THAN A REWRITE: the store is append-only and last-wins per
|
|
84
102
|
* `(workflow, run, step)`, so appending a copy with `status: 'dead'` shadows the
|
|
85
|
-
* live row without rewriting history
|
|
86
|
-
*
|
|
103
|
+
* live row without rewriting history. The shared writer lock serializes each
|
|
104
|
+
* appended retirement with append/compaction transactions from other processes.
|
|
87
105
|
*
|
|
88
106
|
* IDEMPOTENT: a key whose newest row is already `dead` is skipped, so a sweep
|
|
89
107
|
* that reaps the same run twice appends nothing the second time.
|
|
@@ -155,9 +173,11 @@ export declare function reconcileActiveSessions(file: string, liveRunIds: Readon
|
|
|
155
173
|
/**
|
|
156
174
|
* Append one record, then compact when the file has grown past `maxBytes`.
|
|
157
175
|
*
|
|
158
|
-
* The
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
176
|
+
* The cross-process writer lock covers the WHOLE transaction: abandoned-tail
|
|
177
|
+
* quarantine, append, size check, and optional compaction rename. If a crashed
|
|
178
|
+
* writer left bytes without the JSONL commit newline, a leading newline commits
|
|
179
|
+
* only that fragment as corrupt before the new complete record. Append and lock
|
|
180
|
+
* failures PROPAGATE; post-append compaction remains best-effort because the
|
|
181
|
+
* appended line is already committed by its trailing JSONL newline in the original log.
|
|
162
182
|
*/
|
|
163
183
|
export declare function appendSession(file: string, rec: SessionRecord, opts?: SessionStoreOptions): void;
|
|
@@ -18,14 +18,19 @@
|
|
|
18
18
|
*
|
|
19
19
|
* FAILURE STANCE, three different answers on purpose:
|
|
20
20
|
* - READ is fail-open: a missing file reads as `[]` (mirrors `readChildRecords`
|
|
21
|
-
* in `src/shift/state.ts`), and
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* in `src/shift/state.ts`), and an unusable line — corrupt bytes that will
|
|
22
|
+
* not parse, or a parsed object that fails the schema — is skipped and
|
|
23
|
+
* reported through the injectable `warn` callback, never thrown. A store that
|
|
24
|
+
* cannot be parsed must degrade a resume into a replay, not break the worker.
|
|
24
25
|
* - APPEND PROPAGATES. Unlike the shift's advisory metering records, a lost
|
|
25
26
|
* session token silently degrades a Phase 4 resume into a cold replay — real
|
|
26
27
|
* work thrown away — so the caller must see the failure.
|
|
27
|
-
* -
|
|
28
|
-
*
|
|
28
|
+
* - `active` is the safety-critical boundary. Its complete JSONL row is fsynced
|
|
29
|
+
* before provider work may proceed. Other lifecycle rows retain ordinary
|
|
30
|
+
* append durability; losing one degrades to conservative retirement/replay.
|
|
31
|
+
* - COMPACTION is best-effort after a committed append, but a successful
|
|
32
|
+
* compaction fsyncs its replacement before rename so it cannot erase a
|
|
33
|
+
* previously durable `active` row across a host crash.
|
|
29
34
|
*
|
|
30
35
|
* DEVIATION FROM PLAN §3, stated deliberately: plan §3 says "compact on load if
|
|
31
36
|
* > 2 MB". This module compacts on the WRITE path instead — `appendSession`
|
|
@@ -34,20 +39,26 @@
|
|
|
34
39
|
* read-only filesystem), and a small `maxBytes` lets tests exercise compaction
|
|
35
40
|
* without writing 2 MB. The 2 MB default is unchanged.
|
|
36
41
|
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
42
|
+
* WRITER SERIALIZATION: every append, retirement, and compaction takes one
|
|
43
|
+
* cross-process writer lock for its complete read/append/size-check/rename
|
|
44
|
+
* transaction. A completed append can therefore never be replaced or shadowed
|
|
45
|
+
* by another process's stale snapshot. The shared SQLite lock database persists;
|
|
46
|
+
* process exit or crash releases its kernel lock without pathname deletion.
|
|
41
47
|
*
|
|
42
48
|
* Logging follows the house rule (there are zero `console.*` calls anywhere in
|
|
43
49
|
* `src/`): the library takes an injectable callback and the default writes to
|
|
44
50
|
* stderr, so tests stay silent and assertable.
|
|
45
51
|
*/
|
|
46
|
-
import {
|
|
52
|
+
import { closeSync, fstatSync, fsyncSync, mkdirSync, openSync, readFileSync, readSync, renameSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
47
53
|
import { join } from 'node:path';
|
|
54
|
+
import { acquireFileLockSync, releaseFileLock } from "../../../../src/lock.js";
|
|
48
55
|
const STATUSES = new Set(['active', 'turn-ended', 'submitted', 'dead']);
|
|
49
56
|
/** Default compaction threshold — plan §3's 2 MB. */
|
|
50
57
|
export const DEFAULT_MAX_BYTES = 2_000_000;
|
|
58
|
+
/** Grace for a concurrent writer to finish an unterminated JSONL append. */
|
|
59
|
+
export const DEFAULT_UNTERMINATED_TAIL_GRACE_MS = 5_000;
|
|
60
|
+
/** Process-local observations; session files remain the only persisted state. */
|
|
61
|
+
const tailObservations = new Map();
|
|
51
62
|
/** `<cacheDir>/sessions.jsonl`. */
|
|
52
63
|
export function sessionsPath(cacheDir) {
|
|
53
64
|
return join(cacheDir, 'sessions.jsonl');
|
|
@@ -69,40 +80,94 @@ function isNonEmptyString(v) {
|
|
|
69
80
|
return typeof v === 'string' && v !== '';
|
|
70
81
|
}
|
|
71
82
|
/**
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
83
|
+
* The non-empty-string fields `isSessionRecord` requires, in the order it checks
|
|
84
|
+
* them. Declared once and kept stable so a record failing several checks always
|
|
85
|
+
* reports the SAME field, which keeps the skip message deterministic.
|
|
86
|
+
* `status` is not here — its check is a membership test, not a string test — and
|
|
87
|
+
* is applied after these, preserving the original check order.
|
|
76
88
|
*/
|
|
77
|
-
|
|
89
|
+
const SESSION_RECORD_STRING_FIELDS = ['workflow', 'run', 'step', 'harness', 'token'];
|
|
90
|
+
/**
|
|
91
|
+
* Name of the first field of `v` that fails the `SessionRecord` schema, or
|
|
92
|
+
* `null` when the record is valid. `'<root>'` means `v` is not a plain object at
|
|
93
|
+
* all, so no field could be read from it.
|
|
94
|
+
*
|
|
95
|
+
* THE SINGLE SOURCE OF TRUTH for record validity: `isSessionRecord` is defined
|
|
96
|
+
* in terms of this function, so the accept/reject decision and the reported
|
|
97
|
+
* field name can never drift apart.
|
|
98
|
+
*
|
|
99
|
+
* Checks exactly the fields whose absence would make the record unusable for a
|
|
100
|
+
* resume: the three key fields plus the harness and token it exists to carry,
|
|
101
|
+
* and a `status` inside the four literals. Everything else is carried verbatim.
|
|
102
|
+
*/
|
|
103
|
+
function firstInvalidSessionRecordField(v) {
|
|
78
104
|
if (typeof v !== 'object' || v === null || Array.isArray(v))
|
|
79
|
-
return
|
|
105
|
+
return '<root>';
|
|
80
106
|
const r = v;
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if (!isNonEmptyString(r['step']))
|
|
86
|
-
return false;
|
|
87
|
-
if (!isNonEmptyString(r['harness']))
|
|
88
|
-
return false;
|
|
89
|
-
if (!isNonEmptyString(r['token']))
|
|
90
|
-
return false;
|
|
107
|
+
for (const field of SESSION_RECORD_STRING_FIELDS) {
|
|
108
|
+
if (!isNonEmptyString(r[field]))
|
|
109
|
+
return field;
|
|
110
|
+
}
|
|
91
111
|
if (typeof r['status'] !== 'string' || !STATUSES.has(r['status']))
|
|
92
|
-
return
|
|
93
|
-
return
|
|
112
|
+
return 'status';
|
|
113
|
+
return null;
|
|
94
114
|
}
|
|
95
|
-
/**
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
|
|
99
|
-
function
|
|
115
|
+
/**
|
|
116
|
+
* Validate one parsed line. A type predicate, so a `true` return narrows `v` to
|
|
117
|
+
* `SessionRecord` at the call site.
|
|
118
|
+
*/
|
|
119
|
+
function isSessionRecord(v) {
|
|
120
|
+
return firstInvalidSessionRecordField(v) === null;
|
|
121
|
+
}
|
|
122
|
+
/** Flush a directory entry update where the platform exposes directory fsync. */
|
|
123
|
+
function syncDirectory(dir, sync = fsyncSync) {
|
|
124
|
+
if (process.platform === 'win32')
|
|
125
|
+
return;
|
|
126
|
+
const fd = openSync(dir, 'r');
|
|
127
|
+
try {
|
|
128
|
+
sync(fd);
|
|
129
|
+
}
|
|
130
|
+
finally {
|
|
131
|
+
closeSync(fd);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** Atomic durable write used by compaction. The replacement bytes are fsynced
|
|
135
|
+
* before rename, then the directory entry is fsynced. Fs errors propagate to
|
|
136
|
+
* the caller; `compactIfNeededUnlocked` decides whether best-effort compaction
|
|
137
|
+
* may swallow them. */
|
|
138
|
+
function atomicWrite(filePath, content, opts) {
|
|
100
139
|
const dir = join(filePath, '..');
|
|
140
|
+
const sync = opts.sync ?? fsyncSync;
|
|
101
141
|
mkdirSync(dir, { recursive: true });
|
|
102
142
|
const tmp = join(dir, `.${Math.random().toString(36).slice(2)}-${process.pid}-${Date.now()}.tmp`);
|
|
103
|
-
|
|
143
|
+
const fd = openSync(tmp, 'wx');
|
|
144
|
+
let failure;
|
|
145
|
+
try {
|
|
146
|
+
writeFileSync(fd, content);
|
|
147
|
+
sync(fd);
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
failure = error;
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
closeSync(fd);
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
if (failure === undefined)
|
|
157
|
+
failure = error;
|
|
158
|
+
}
|
|
159
|
+
if (failure !== undefined) {
|
|
160
|
+
try {
|
|
161
|
+
unlinkSync(tmp);
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
// Preserve the original write or close failure.
|
|
165
|
+
}
|
|
166
|
+
throw failure;
|
|
167
|
+
}
|
|
104
168
|
try {
|
|
105
169
|
renameSync(tmp, filePath);
|
|
170
|
+
syncDirectory(dir, sync);
|
|
106
171
|
}
|
|
107
172
|
catch (err) {
|
|
108
173
|
try {
|
|
@@ -116,24 +181,93 @@ function atomicWrite(filePath, content) {
|
|
|
116
181
|
}
|
|
117
182
|
/**
|
|
118
183
|
* Every valid record in `file`, oldest first. A missing/unreadable file reads as
|
|
119
|
-
* `[]`.
|
|
120
|
-
*
|
|
184
|
+
* `[]`. A trailing newline is the record-commit marker: an unterminated final
|
|
185
|
+
* tail may be a concurrent writer still appending. File modification time starts
|
|
186
|
+
* the grace, so a recently modified tail is ignored silently while an already-old
|
|
187
|
+
* tail warns on the first read even in a fresh process. File identity, size, tail,
|
|
188
|
+
* or modification changes restart the observation; a newline clears it. Blank lines are
|
|
189
|
+
* skipped silently; every other unusable COMPLETE line is skipped and reported
|
|
190
|
+
* through `warn` with its 1-indexed line number. Never throws.
|
|
121
191
|
*/
|
|
122
192
|
export function readSessions(file, opts = {}) {
|
|
123
193
|
const warn = opts.warn ?? defaultWarn;
|
|
124
194
|
let raw;
|
|
195
|
+
let metadata;
|
|
196
|
+
let fd;
|
|
125
197
|
try {
|
|
126
|
-
|
|
198
|
+
fd = openSync(file, 'r');
|
|
199
|
+
raw = readFileSync(fd, 'utf8');
|
|
200
|
+
const stat = fstatSync(fd);
|
|
201
|
+
metadata = {
|
|
202
|
+
identity: `${stat.dev}:${stat.ino}`,
|
|
203
|
+
size: stat.size,
|
|
204
|
+
mtimeMs: stat.mtimeMs,
|
|
205
|
+
};
|
|
127
206
|
}
|
|
128
207
|
catch {
|
|
208
|
+
tailObservations.delete(file);
|
|
129
209
|
return []; // fail-open: no store yet is the normal first-run case
|
|
130
210
|
}
|
|
211
|
+
finally {
|
|
212
|
+
if (fd !== undefined) {
|
|
213
|
+
try {
|
|
214
|
+
closeSync(fd);
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
// The bytes and metadata were already captured; a close failure cannot
|
|
218
|
+
// make a resumable record safer, so preserve the read path's fail-open stance.
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
131
222
|
const out = [];
|
|
132
223
|
const lines = raw.split('\n');
|
|
224
|
+
const hasUncommittedTail = raw !== '' && !raw.endsWith('\n');
|
|
225
|
+
if (!hasUncommittedTail) {
|
|
226
|
+
tailObservations.delete(file);
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
const tail = lines.at(-1) ?? '';
|
|
230
|
+
if (tail.trim() === '') {
|
|
231
|
+
tailObservations.delete(file);
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
const now = (opts.now ?? Date.now)();
|
|
235
|
+
const graceMs = opts.unterminatedTailGraceMs ?? DEFAULT_UNTERMINATED_TAIL_GRACE_MS;
|
|
236
|
+
const observed = tailObservations.get(file);
|
|
237
|
+
const sameObservation = observed !== undefined
|
|
238
|
+
&& observed.identity === metadata.identity
|
|
239
|
+
&& observed.size === metadata.size
|
|
240
|
+
&& observed.mtimeMs === metadata.mtimeMs
|
|
241
|
+
&& observed.tail === tail;
|
|
242
|
+
if (!sameObservation) {
|
|
243
|
+
// mtime is persisted by the filesystem, so an abandoned tail can be
|
|
244
|
+
// recognized on the first read of a fresh one-shot process. Clamp a
|
|
245
|
+
// future timestamp to `now` for clock skew and injected test clocks.
|
|
246
|
+
const firstSeenAt = Math.min(now, metadata.mtimeMs);
|
|
247
|
+
const next = {
|
|
248
|
+
...metadata,
|
|
249
|
+
tail,
|
|
250
|
+
firstSeenAt,
|
|
251
|
+
warned: false,
|
|
252
|
+
};
|
|
253
|
+
tailObservations.set(file, next);
|
|
254
|
+
if (now - firstSeenAt >= graceMs) {
|
|
255
|
+
warn(`owenloop work sessions: persistent unterminated record at ${file}:${lines.length}`);
|
|
256
|
+
next.warned = true;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
else if (observed !== undefined && !observed.warned && now - observed.firstSeenAt >= graceMs) {
|
|
260
|
+
warn(`owenloop work sessions: persistent unterminated record at ${file}:${lines.length}`);
|
|
261
|
+
observed.warned = true;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
133
265
|
for (let i = 0; i < lines.length; i++) {
|
|
134
266
|
const line = lines[i];
|
|
135
267
|
if (line === undefined)
|
|
136
268
|
continue; // noUncheckedIndexedAccess
|
|
269
|
+
if (hasUncommittedTail && i === lines.length - 1)
|
|
270
|
+
continue;
|
|
137
271
|
if (line.trim() === '')
|
|
138
272
|
continue; // blank lines are not corruption
|
|
139
273
|
let parsed;
|
|
@@ -145,7 +279,14 @@ export function readSessions(file, opts = {}) {
|
|
|
145
279
|
continue;
|
|
146
280
|
}
|
|
147
281
|
if (!isSessionRecord(parsed)) {
|
|
148
|
-
|
|
282
|
+
// DISTINCT WORDING ON PURPOSE — "invalid", not "corrupt". These bytes
|
|
283
|
+
// parsed fine; they just failed the schema. 36 records once whose only
|
|
284
|
+
// fault was an empty `token` were all reported as "corrupt", which sent
|
|
285
|
+
// debugging after a file-integrity problem that did not exist. The second
|
|
286
|
+
// pass runs only on this failure branch, so valid records pay nothing.
|
|
287
|
+
// The field NAME only, never its value — `token` is credential-shaped.
|
|
288
|
+
const badField = firstInvalidSessionRecordField(parsed) ?? '<unknown>';
|
|
289
|
+
warn(`owenloop work sessions: skipping invalid record at ${file}:${i + 1}: field "${badField}" failed schema check`);
|
|
149
290
|
continue;
|
|
150
291
|
}
|
|
151
292
|
out.push(parsed);
|
|
@@ -166,13 +307,40 @@ export function latestFor(file, workflow, run, step) {
|
|
|
166
307
|
}
|
|
167
308
|
return best;
|
|
168
309
|
}
|
|
169
|
-
/**
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
310
|
+
/** The lock shared by append and compaction for one session log. */
|
|
311
|
+
function writerLockPath(file) {
|
|
312
|
+
return `${file}.lock`;
|
|
313
|
+
}
|
|
314
|
+
/** Run one complete writer transaction under the cross-process session lock. */
|
|
315
|
+
function withWriterLock(file, operation) {
|
|
316
|
+
const lock = acquireFileLockSync(writerLockPath(file), {
|
|
317
|
+
waitMs: 30_000,
|
|
318
|
+
label: 'owenloop session-store write',
|
|
319
|
+
});
|
|
320
|
+
try {
|
|
321
|
+
return operation();
|
|
322
|
+
}
|
|
323
|
+
finally {
|
|
324
|
+
releaseFileLock(lock);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
/** True when an existing nonempty log does not end at a JSONL commit marker. */
|
|
328
|
+
function needsRecordSeparator(file) {
|
|
329
|
+
const stat = statSync(file, { throwIfNoEntry: false });
|
|
330
|
+
if (stat === undefined || stat.size === 0)
|
|
331
|
+
return false;
|
|
332
|
+
const fd = openSync(file, 'r');
|
|
333
|
+
try {
|
|
334
|
+
const last = Buffer.allocUnsafe(1);
|
|
335
|
+
const read = readSync(fd, last, 0, 1, stat.size - 1);
|
|
336
|
+
return read !== 1 || last[0] !== 0x0a;
|
|
337
|
+
}
|
|
338
|
+
finally {
|
|
339
|
+
closeSync(fd);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
/** Lock must already be held by the caller. */
|
|
343
|
+
function compactUnlocked(file, opts) {
|
|
176
344
|
const records = readSessions(file, opts);
|
|
177
345
|
if (records.length === 0)
|
|
178
346
|
return;
|
|
@@ -183,7 +351,58 @@ export function compact(file, opts = {}) {
|
|
|
183
351
|
byKey.set(keyOf(rec.workflow, rec.run, rec.step), rec);
|
|
184
352
|
}
|
|
185
353
|
const body = [...byKey.values()].map((r) => JSON.stringify(r)).join('\n');
|
|
186
|
-
atomicWrite(file, `${body}\n
|
|
354
|
+
atomicWrite(file, `${body}\n`, opts);
|
|
355
|
+
}
|
|
356
|
+
/** Lock must already be held. Append exactly one committed JSONL row.
|
|
357
|
+
* `active` is fsynced before returning because provider work is gated on this
|
|
358
|
+
* call. Later statuses use ordinary close-to-flush durability. */
|
|
359
|
+
function appendSessionUnlocked(file, rec, opts) {
|
|
360
|
+
const separator = needsRecordSeparator(file) ? '\n' : '';
|
|
361
|
+
const existed = statSync(file, { throwIfNoEntry: false }) !== undefined;
|
|
362
|
+
const fd = openSync(file, 'a');
|
|
363
|
+
let failure;
|
|
364
|
+
try {
|
|
365
|
+
writeFileSync(fd, `${separator}${JSON.stringify(rec)}\n`);
|
|
366
|
+
if (rec.status === 'active') {
|
|
367
|
+
const sync = opts.sync ?? fsyncSync;
|
|
368
|
+
sync(fd);
|
|
369
|
+
if (!existed)
|
|
370
|
+
syncDirectory(join(file, '..'), sync);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
catch (error) {
|
|
374
|
+
failure = error;
|
|
375
|
+
}
|
|
376
|
+
try {
|
|
377
|
+
closeSync(fd);
|
|
378
|
+
}
|
|
379
|
+
catch (error) {
|
|
380
|
+
if (failure === undefined)
|
|
381
|
+
failure = error;
|
|
382
|
+
}
|
|
383
|
+
if (failure !== undefined)
|
|
384
|
+
throw failure;
|
|
385
|
+
}
|
|
386
|
+
/** Lock must already be held. Compaction remains best-effort after committed JSONL appends. */
|
|
387
|
+
function compactIfNeededUnlocked(file, opts) {
|
|
388
|
+
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
389
|
+
try {
|
|
390
|
+
if (statSync(file).size > maxBytes)
|
|
391
|
+
compactUnlocked(file, opts);
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
// An un-compacted (or unstatable) file still contains the committed rows.
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Rewrite `file` with only the last record per `(workflow, run, step)`,
|
|
399
|
+
* preserving FIRST-SEEN key order, via temp file + rename. Corrupt lines are
|
|
400
|
+
* dropped in the process (they were already unreadable). A missing file is a
|
|
401
|
+
* no-op. The complete read/rewrite/rename transaction shares the same lock as
|
|
402
|
+
* append, so a successful append cannot be replaced by this snapshot.
|
|
403
|
+
*/
|
|
404
|
+
export function compact(file, opts = {}) {
|
|
405
|
+
withWriterLock(file, () => compactUnlocked(file, opts));
|
|
187
406
|
}
|
|
188
407
|
/**
|
|
189
408
|
* PHASE 4 — retire every session of one RUN by appending a `dead` row per step.
|
|
@@ -197,8 +416,8 @@ export function compact(file, opts = {}) {
|
|
|
197
416
|
*
|
|
198
417
|
* WHY A NEW ROW RATHER THAN A REWRITE: the store is append-only and last-wins per
|
|
199
418
|
* `(workflow, run, step)`, so appending a copy with `status: 'dead'` shadows the
|
|
200
|
-
* live row without rewriting history
|
|
201
|
-
*
|
|
419
|
+
* live row without rewriting history. The shared writer lock serializes each
|
|
420
|
+
* appended retirement with append/compaction transactions from other processes.
|
|
202
421
|
*
|
|
203
422
|
* IDEMPOTENT: a key whose newest row is already `dead` is skipped, so a sweep
|
|
204
423
|
* that reaps the same run twice appends nothing the second time.
|
|
@@ -215,20 +434,26 @@ export function compact(file, opts = {}) {
|
|
|
215
434
|
* Returns the step names actually marked, so a sweep can log a count.
|
|
216
435
|
*/
|
|
217
436
|
export function markRunSessionsDead(file, workflow, run, now, opts = {}) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
437
|
+
mkdirSync(join(file, '..'), { recursive: true });
|
|
438
|
+
return withWriterLock(file, () => {
|
|
439
|
+
const newest = new Map();
|
|
440
|
+
for (const rec of readSessions(file, opts)) {
|
|
441
|
+
if (rec.workflow !== workflow || rec.run !== run)
|
|
442
|
+
continue;
|
|
443
|
+
newest.set(keyOf(rec.workflow, rec.run, rec.step), rec);
|
|
444
|
+
}
|
|
445
|
+
opts.afterWriterSnapshot?.();
|
|
446
|
+
const marked = [];
|
|
447
|
+
for (const rec of newest.values()) {
|
|
448
|
+
if (rec.status === 'dead')
|
|
449
|
+
continue; // already retired — nothing to append
|
|
450
|
+
appendSessionUnlocked(file, { ...rec, status: 'dead', updatedAt: now }, opts);
|
|
451
|
+
marked.push(rec.step);
|
|
452
|
+
}
|
|
453
|
+
if (marked.length > 0)
|
|
454
|
+
compactIfNeededUnlocked(file, opts);
|
|
455
|
+
return marked;
|
|
456
|
+
});
|
|
232
457
|
}
|
|
233
458
|
/**
|
|
234
459
|
* PHASE 6, ITEM 4 — retire the `active` sessions of runs that are no longer
|
|
@@ -282,38 +507,41 @@ export function markRunSessionsDead(file, workflow, run, now, opts = {}) {
|
|
|
282
507
|
* caller can log what it took away.
|
|
283
508
|
*/
|
|
284
509
|
export function reconcileActiveSessions(file, liveRunIds, now, opts = {}) {
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
newest
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
510
|
+
mkdirSync(join(file, '..'), { recursive: true });
|
|
511
|
+
return withWriterLock(file, () => {
|
|
512
|
+
const newest = new Map();
|
|
513
|
+
for (const rec of readSessions(file, opts)) {
|
|
514
|
+
newest.set(keyOf(rec.workflow, rec.run, rec.step), rec);
|
|
515
|
+
}
|
|
516
|
+
opts.afterWriterSnapshot?.();
|
|
517
|
+
const retired = [];
|
|
518
|
+
for (const rec of newest.values()) {
|
|
519
|
+
if (rec.status !== 'active')
|
|
520
|
+
continue; // a completed turn is still resumable
|
|
521
|
+
if (liveRunIds.has(rec.run))
|
|
522
|
+
continue; // its worker is alive; the row is true
|
|
523
|
+
appendSessionUnlocked(file, { ...rec, status: 'dead', updatedAt: now }, opts);
|
|
524
|
+
retired.push(rec);
|
|
525
|
+
}
|
|
526
|
+
if (retired.length > 0)
|
|
527
|
+
compactIfNeededUnlocked(file, opts);
|
|
528
|
+
return retired;
|
|
529
|
+
});
|
|
299
530
|
}
|
|
300
531
|
/**
|
|
301
532
|
* Append one record, then compact when the file has grown past `maxBytes`.
|
|
302
533
|
*
|
|
303
|
-
* The
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
534
|
+
* The cross-process writer lock covers the WHOLE transaction: abandoned-tail
|
|
535
|
+
* quarantine, append, size check, and optional compaction rename. If a crashed
|
|
536
|
+
* writer left bytes without the JSONL commit newline, a leading newline commits
|
|
537
|
+
* only that fragment as corrupt before the new complete record. Append and lock
|
|
538
|
+
* failures PROPAGATE; post-append compaction remains best-effort because the
|
|
539
|
+
* appended line is already committed by its trailing JSONL newline in the original log.
|
|
307
540
|
*/
|
|
308
541
|
export function appendSession(file, rec, opts = {}) {
|
|
309
542
|
mkdirSync(join(file, '..'), { recursive: true });
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
compact(file, opts);
|
|
315
|
-
}
|
|
316
|
-
catch {
|
|
317
|
-
// best-effort: an un-compacted (or unstatable) file is still correct.
|
|
318
|
-
}
|
|
543
|
+
withWriterLock(file, () => {
|
|
544
|
+
appendSessionUnlocked(file, rec, opts);
|
|
545
|
+
compactIfNeededUnlocked(file, opts);
|
|
546
|
+
});
|
|
319
547
|
}
|