owenloop 0.5.9 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +77 -37
- package/bin/owenloop.mjs +16 -1
- package/dist/packages/work/src/agent/loop.js +7 -0
- package/dist/packages/work/src/bundle/pull.d.ts +21 -0
- package/dist/packages/work/src/bundle/pull.js +118 -0
- package/dist/packages/work/src/exec/instructions.d.ts +4 -1
- package/dist/packages/work/src/exec/instructions.js +3 -0
- package/dist/packages/work/src/exec/loop.d.ts +1 -1
- package/dist/packages/work/src/exec/loop.js +92 -14
- package/dist/packages/work/src/harness/claude.js +16 -0
- package/dist/packages/work/src/harness/codex.d.ts +8 -0
- package/dist/packages/work/src/harness/codex.js +27 -1
- package/dist/packages/work/src/harness/contract.d.ts +9 -0
- package/dist/packages/work/src/roles/agent-run.js +38 -10
- package/dist/packages/work/src/roles/exec.js +25 -8
- package/dist/packages/work/src/shift/loop.js +118 -4
- package/dist/packages/work/src/shift/runtime.d.ts +3 -0
- package/dist/packages/work/src/shift/runtime.js +10 -1
- package/dist/packages/work/src/submit-proof.d.ts +11 -8
- package/dist/packages/work/src/submit-proof.js +11 -8
- package/dist/src/capabilities.d.ts +70 -6
- package/dist/src/capabilities.js +49 -6
- package/dist/src/capability-mapping-client.d.ts +29 -0
- package/dist/src/capability-mapping-client.js +160 -0
- package/dist/src/cli.d.ts +2 -2
- package/dist/src/cli.js +1196 -164
- package/dist/src/defs.d.ts +32 -1
- package/dist/src/defs.js +391 -79
- package/dist/src/engine.d.ts +62 -19
- package/dist/src/engine.js +180 -70
- package/dist/src/hub.d.ts +77 -0
- package/dist/src/hub.js +162 -0
- package/dist/src/implements.d.ts +56 -0
- package/dist/src/implements.js +790 -0
- package/dist/src/index.d.ts +8 -6
- package/dist/src/index.js +4 -3
- package/dist/src/mcp/serve.js +254 -45
- package/dist/src/mcp/server.d.ts +10 -7
- package/dist/src/mcp/server.js +53 -8
- package/dist/src/model.d.ts +46 -10
- package/dist/src/model.js +220 -74
- package/dist/src/store/def-source.d.ts +19 -0
- package/dist/src/store/def-source.js +87 -17
- package/dist/src/store/gc-recovery.d.ts +46 -0
- package/dist/src/store/gc-recovery.js +258 -0
- package/dist/src/store/gc.d.ts +94 -0
- package/dist/src/store/gc.js +562 -0
- package/dist/src/store/index.d.ts +3 -1
- package/dist/src/store/index.js +2 -0
- package/dist/src/store/install.d.ts +34 -5
- package/dist/src/store/install.js +73 -11
- package/dist/src/store/pre-commit-verifier.js +42 -2
- package/dist/src/store/snapshot-guard.d.ts +18 -0
- package/dist/src/store/snapshot-guard.js +94 -0
- package/dist/src/store.d.ts +64 -4
- package/dist/src/store.js +211 -8
- package/dist/src/types.d.ts +61 -17
- package/dist/src/util.js +4 -1
- package/docs/authoring.md +145 -8
- package/docs/bundles.md +19 -0
- package/docs/cli.md +270 -30
- package/docs/crypto.md +13 -7
- package/docs/design.md +107 -46
- package/docs/evals/mcp-charter-baseline.json +428 -0
- package/docs/mcp-charter-eval.md +124 -0
- package/docs/shift-logs.md +18 -0
- package/docs/wire-contracts.md +28 -7
- package/examples/workflows/compile-dev-playbook.yaml +1 -1
- package/examples/workflows/ship.yaml +28 -2
- package/package.json +2 -1
- package/plugins/claude-code/plugin/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-code/plugin/.mcp.json +1 -1
- package/plugins/claude-code/plugin/skills/author/SKILL.md +22 -0
- package/plugins/claude-code/plugin/skills/ephemeral/SKILL.md +95 -0
- package/plugins/claude-code/plugin/skills/graduate/SKILL.md +156 -0
- package/plugins/claude-code/plugin/skills/plan/SKILL.md +236 -0
- package/plugins/codex/plugins/owenloop/.codex-plugin/plugin.json +1 -1
- package/plugins/codex/plugins/owenloop/.mcp.json +1 -1
- package/plugins/codex/plugins/owenloop/skills/author/SKILL.md +22 -0
- package/plugins/codex/plugins/owenloop/skills/ephemeral/SKILL.md +95 -0
- package/plugins/codex/plugins/owenloop/skills/graduate/SKILL.md +156 -0
- package/plugins/codex/plugins/owenloop/skills/plan/SKILL.md +236 -0
|
@@ -26,6 +26,16 @@ function isRuntimeIncompatible(error) {
|
|
|
26
26
|
return error instanceof Error
|
|
27
27
|
&& error.message.includes('incompatible with this Owenloop runtime');
|
|
28
28
|
}
|
|
29
|
+
function setBundleStoreRoots(def, roots) {
|
|
30
|
+
// This is live loader provenance, not definition content. Keep it available
|
|
31
|
+
// to snapshot writers without changing hashDef or the persisted snapshot.
|
|
32
|
+
Object.defineProperty(def, 'bundleStoreRoots', {
|
|
33
|
+
value: roots,
|
|
34
|
+
writable: true,
|
|
35
|
+
configurable: true,
|
|
36
|
+
enumerable: false,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
29
39
|
function failureMessage(error) {
|
|
30
40
|
return error instanceof Error ? error.message : String(error);
|
|
31
41
|
}
|
|
@@ -53,9 +63,9 @@ function readIndexedCoordinates(root, level, tolerant, warn) {
|
|
|
53
63
|
}
|
|
54
64
|
}
|
|
55
65
|
/** Load and verify every definition in one immutable object. */
|
|
56
|
-
function loadObjectDefs(objectDir, bundleDigest, level) {
|
|
66
|
+
function loadObjectDefs(root, objectDir, bundleDigest, level, requireHardenedModes = true) {
|
|
57
67
|
try {
|
|
58
|
-
verifyWorkflowObjectSync(objectDir, bundleDigest, { coordinateRepair: false });
|
|
68
|
+
verifyWorkflowObjectSync(objectDir, bundleDigest, { coordinateRepair: false, requireHardenedModes });
|
|
59
69
|
const manifest = parseManifestBytes(readFileSync(join(objectDir, 'bundle.yaml')));
|
|
60
70
|
const defs = new Map();
|
|
61
71
|
// Sorted, not YAML key order. `defs` is handed to callers as a Map, whose
|
|
@@ -76,7 +86,7 @@ function loadObjectDefs(objectDir, bundleDigest, level) {
|
|
|
76
86
|
def.bundleLock = { ...manifest.lock };
|
|
77
87
|
defs.set(def.name, def);
|
|
78
88
|
}
|
|
79
|
-
return { bundleDigest, manifest, defs, level };
|
|
89
|
+
return { bundleDigest, manifest, defs, level, objectRoot: root };
|
|
80
90
|
}
|
|
81
91
|
catch (error) {
|
|
82
92
|
if (isRuntimeIncompatible(error)) {
|
|
@@ -94,7 +104,7 @@ function loadObjectIfPresent(root, digest, level) {
|
|
|
94
104
|
if (probeObjectDir(objectDir, digest, level) !== 'dir') {
|
|
95
105
|
throw new CasObjectAbsentDuringCoordinatedRead();
|
|
96
106
|
}
|
|
97
|
-
return loadObjectDefs(objectDir, digest, level);
|
|
107
|
+
return loadObjectDefs(root, objectDir, digest, level);
|
|
98
108
|
});
|
|
99
109
|
}
|
|
100
110
|
catch (error) {
|
|
@@ -105,16 +115,26 @@ function loadObjectIfPresent(root, digest, level) {
|
|
|
105
115
|
throw new StoreIntegrityError('object-corrupt', digest, failureMessage(error));
|
|
106
116
|
}
|
|
107
117
|
}
|
|
108
|
-
function resolveObject(indexed, globalRoot, globalDigests) {
|
|
118
|
+
function resolveObject(indexed, globalRoot, globalDigests, repairReplacement) {
|
|
109
119
|
const digest = defDigest(indexed.entry.digest);
|
|
110
|
-
const
|
|
120
|
+
const loadAtRoot = (root, level) => {
|
|
121
|
+
if (repairReplacement !== undefined
|
|
122
|
+
&& root === repairReplacement.root
|
|
123
|
+
&& digest === repairReplacement.digest) {
|
|
124
|
+
// Staging is hardened only after the swap; canonical bytes and
|
|
125
|
+
// definition semantics are still verified here before revalidation.
|
|
126
|
+
return loadObjectDefs(root, repairReplacement.objectDir, digest, level, false);
|
|
127
|
+
}
|
|
128
|
+
return loadObjectIfPresent(root, digest, level);
|
|
129
|
+
};
|
|
130
|
+
const primary = loadAtRoot(indexed.root, indexed.level);
|
|
111
131
|
if (primary !== undefined)
|
|
112
132
|
return primary;
|
|
113
133
|
// An indexed project object may fall through only to the exact same digest,
|
|
114
134
|
// and only when that digest is also indexed globally. A different global
|
|
115
135
|
// digest for the coordinate is never considered.
|
|
116
136
|
if (indexed.level === 'project' && globalDigests.has(digest)) {
|
|
117
|
-
const fallback =
|
|
137
|
+
const fallback = loadAtRoot(globalRoot, 'global');
|
|
118
138
|
if (fallback !== undefined)
|
|
119
139
|
return fallback;
|
|
120
140
|
}
|
|
@@ -172,7 +192,7 @@ function shadowedWorkflowKey(candidate) {
|
|
|
172
192
|
* project pin still outranks a higher global version even when the project's
|
|
173
193
|
* own object bytes were missing and had to be read from the global store.
|
|
174
194
|
*/
|
|
175
|
-
function selectWorkflowRegistrations(objects, warn) {
|
|
195
|
+
function selectWorkflowRegistrations(objects, warn, verbose) {
|
|
176
196
|
const byQualified = new Map();
|
|
177
197
|
const ordered = [...objects].sort((a, b) => compareStoreText(a.loaded.bundleDigest, b.loaded.bundleDigest));
|
|
178
198
|
for (const { loaded, indexedLevel } of ordered) {
|
|
@@ -195,6 +215,7 @@ function selectWorkflowRegistrations(objects, warn) {
|
|
|
195
215
|
}
|
|
196
216
|
}
|
|
197
217
|
const registrations = [];
|
|
218
|
+
const suppressedDigests = new Set();
|
|
198
219
|
for (const qualified of [...byQualified.keys()].sort(compareStoreText)) {
|
|
199
220
|
const candidates = byQualified.get(qualified);
|
|
200
221
|
const selection = selectLatestVersion(candidates);
|
|
@@ -225,16 +246,20 @@ function selectWorkflowRegistrations(objects, warn) {
|
|
|
225
246
|
// index that named the object. The registration's own `level` field
|
|
226
247
|
// reports the different question of which store the bytes came from, and
|
|
227
248
|
// the two legitimately disagree under exact-digest fallback.
|
|
228
|
-
|
|
249
|
+
const message = `warning: workflow '${qualified}' from ${candidate.level}-indexed bundle ${candidate.digest} ` +
|
|
229
250
|
`(version ${candidate.version}) does not hold that name — ${selection.winner.level}-indexed ` +
|
|
230
251
|
`bundle ${selection.winner.digest} (version ${selection.winner.version}) is the selected ` +
|
|
231
|
-
`version; this copy stays reachable as '${key}'
|
|
252
|
+
`version; this copy stays reachable as '${key}'`;
|
|
253
|
+
if (verbose)
|
|
254
|
+
warn(message);
|
|
255
|
+
else
|
|
256
|
+
suppressedDigests.add(candidate.digest);
|
|
232
257
|
registrations.push(candidateRegistration(candidate, key));
|
|
233
258
|
}
|
|
234
259
|
}
|
|
235
|
-
return registrations;
|
|
260
|
+
return { registrations, suppressedDigests };
|
|
236
261
|
}
|
|
237
|
-
function discoverCasDefs(args, tolerant) {
|
|
262
|
+
function discoverCasDefs(args, tolerant, repairReplacement) {
|
|
238
263
|
const projectRoot = args.projectRoot === undefined ? undefined : projectStoreRoot(args.projectRoot);
|
|
239
264
|
const globalRoot = projectStoreRoot(args.globalRoot);
|
|
240
265
|
const sameRoot = projectRoot === undefined || projectRoot === globalRoot;
|
|
@@ -259,16 +284,26 @@ function discoverCasDefs(args, tolerant) {
|
|
|
259
284
|
for (const item of global.entries)
|
|
260
285
|
globalDigests.add(defDigest(item.entry.digest));
|
|
261
286
|
const registrations = [];
|
|
262
|
-
|
|
287
|
+
// Physical validation is scoped to the INDEXING root, not merely to content
|
|
288
|
+
// identity. A project index may borrow an exact digest from global, but a
|
|
289
|
+
// global index row must independently prove that the global copy exists. If
|
|
290
|
+
// this cache were digest-only, the project-first walk could let valid project
|
|
291
|
+
// bytes mask a missing/corrupt global object with the same digest.
|
|
292
|
+
const loadedByRootAndDigest = new Map();
|
|
263
293
|
const registeredObjects = new Map();
|
|
264
294
|
const registeredCoordinateKeys = new Set();
|
|
265
295
|
for (const indexed of selected) {
|
|
266
296
|
try {
|
|
267
297
|
const digest = defDigest(indexed.entry.digest);
|
|
268
|
-
let
|
|
298
|
+
let loadedAtRoot = loadedByRootAndDigest.get(indexed.root);
|
|
299
|
+
if (loadedAtRoot === undefined) {
|
|
300
|
+
loadedAtRoot = new Map();
|
|
301
|
+
loadedByRootAndDigest.set(indexed.root, loadedAtRoot);
|
|
302
|
+
}
|
|
303
|
+
let loaded = loadedAtRoot.get(digest);
|
|
269
304
|
if (loaded === undefined) {
|
|
270
|
-
loaded = resolveObject(indexed, globalRoot, globalDigests);
|
|
271
|
-
|
|
305
|
+
loaded = resolveObject(indexed, globalRoot, globalDigests, repairReplacement);
|
|
306
|
+
loadedAtRoot.set(digest, loaded);
|
|
272
307
|
}
|
|
273
308
|
const target = coordinateTarget(indexed, loaded);
|
|
274
309
|
// `indexed.level`, not `loaded.level`: the index that NAMED the object
|
|
@@ -279,6 +314,17 @@ function discoverCasDefs(args, tolerant) {
|
|
|
279
314
|
if (alreadyRegistered?.indexedLevel !== 'project') {
|
|
280
315
|
registeredObjects.set(loaded.bundleDigest, { loaded, indexedLevel: indexed.level });
|
|
281
316
|
}
|
|
317
|
+
// Snapshot writers must coordinate with every store root that can affect
|
|
318
|
+
// this digest's discoverability. This includes the index that named it
|
|
319
|
+
// and, for project exact-digest fallback, the global root that supplied
|
|
320
|
+
// its verified bytes.
|
|
321
|
+
for (const def of loaded.defs.values()) {
|
|
322
|
+
setBundleStoreRoots(def, [...new Set([
|
|
323
|
+
...(def.bundleStoreRoots ?? []),
|
|
324
|
+
indexed.root,
|
|
325
|
+
loaded.objectRoot,
|
|
326
|
+
])].sort(compareStoreText));
|
|
327
|
+
}
|
|
282
328
|
if (target === undefined) {
|
|
283
329
|
if (indexed.registerAlias) {
|
|
284
330
|
args.warn(`warning: coordinate '${indexed.coordinate}' is not callable because the bundle exports ` +
|
|
@@ -322,13 +368,37 @@ function discoverCasDefs(args, tolerant) {
|
|
|
322
368
|
args.warn(`warning: incomplete ${indexed.level} workflow coordinate '${indexed.coordinate}': ${failureMessage(error)}`);
|
|
323
369
|
}
|
|
324
370
|
}
|
|
325
|
-
|
|
371
|
+
const selectedWorkflows = selectWorkflowRegistrations([...registeredObjects.values()], args.warn, args.verbose === true);
|
|
372
|
+
registrations.push(...selectedWorkflows.registrations);
|
|
373
|
+
if (args.verbose !== true && selectedWorkflows.suppressedDigests.size > 0) {
|
|
374
|
+
const count = selectedWorkflows.suppressedDigests.size;
|
|
375
|
+
args.warn(`note: ${count} superseded bundle ${count === 1 ? 'version' : 'versions'} hidden; --verbose to list them`);
|
|
376
|
+
}
|
|
326
377
|
return { registrations, complete };
|
|
327
378
|
}
|
|
328
379
|
/** Strict executable discovery. Any indexed integrity failure aborts. */
|
|
329
380
|
export function loadCasDefs(args) {
|
|
330
381
|
return discoverCasDefs(args, false).registrations;
|
|
331
382
|
}
|
|
383
|
+
/**
|
|
384
|
+
* Strict combined-store discovery for a same-digest repair transaction.
|
|
385
|
+
*
|
|
386
|
+
* The installer calls this only after independently verifying the staged
|
|
387
|
+
* replacement. Substituting those bytes for the repairing root/digest lets its
|
|
388
|
+
* exact reinstall reach the repair swap without removing the indexed digest
|
|
389
|
+
* that project fallback legitimately depends on. Every dependency, unrelated
|
|
390
|
+
* object, and same-digest copy at another root still goes through the ordinary
|
|
391
|
+
* strict execution-time discovery path.
|
|
392
|
+
* This transaction-only seam is deliberately not re-exported from the store
|
|
393
|
+
* barrel: executable discovery must never suppress a corrupt object.
|
|
394
|
+
*/
|
|
395
|
+
export function loadCasDefsWithRepairReplacement(args, replacement) {
|
|
396
|
+
return discoverCasDefs(args, false, {
|
|
397
|
+
root: projectStoreRoot(replacement.root),
|
|
398
|
+
digest: defDigest(replacement.digest),
|
|
399
|
+
objectDir: replacement.objectDir,
|
|
400
|
+
}).registrations;
|
|
401
|
+
}
|
|
332
402
|
/** Tolerant read-only inspection. Never use this result for execution. */
|
|
333
403
|
export function inspectCasDefs(args) {
|
|
334
404
|
return discoverCasDefs(args, true);
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Crash recovery for GC object parking.
|
|
3
|
+
*
|
|
4
|
+
* APFS refuses to rename a hardened 0555 directory, so GC must briefly add
|
|
5
|
+
* owner-write to the object root before moving it out of the canonical CAS
|
|
6
|
+
* path. The journal below makes that otherwise-dangerous interval recoverable
|
|
7
|
+
* by every writer that shares the workflow-store lock. The pruned index is the
|
|
8
|
+
* durable commit point: recovery restores a parked object when the current
|
|
9
|
+
* index still names it, and treats it as staging debris only when the current
|
|
10
|
+
* index corroborates that the digest is no longer reachable by coordinate.
|
|
11
|
+
*/
|
|
12
|
+
import type { DefDigest } from './types.ts';
|
|
13
|
+
export declare const WORKFLOW_STORE_GC_JOURNAL_FILENAME = "gc.journal";
|
|
14
|
+
export interface WorkflowStoreGcRecoveryArgs {
|
|
15
|
+
root: string;
|
|
16
|
+
stateDir: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ParkWorkflowStoreGcObjectArgs extends WorkflowStoreGcRecoveryArgs {
|
|
19
|
+
digest: DefDigest;
|
|
20
|
+
parkedName: string;
|
|
21
|
+
afterLiveObjectMadeWritable?: (path: string) => void;
|
|
22
|
+
afterObjectParked?: (path: string) => void;
|
|
23
|
+
}
|
|
24
|
+
export interface RemoveParkedWorkflowStoreGcObjectArgs extends WorkflowStoreGcRecoveryArgs {
|
|
25
|
+
parked: string;
|
|
26
|
+
remove?: (path: string) => void;
|
|
27
|
+
}
|
|
28
|
+
export declare function workflowStoreGcJournalPath(stateDir: string): string;
|
|
29
|
+
/** Persist a directory-entry transition, not merely the moved directory inode. */
|
|
30
|
+
export declare function fsyncWorkflowStoreDirectory(path: string): void;
|
|
31
|
+
/** Make an atomic metadata write durable before a destructive follow-on step. */
|
|
32
|
+
export declare function fsyncWorkflowStoreFileAndParent(path: string): void;
|
|
33
|
+
/**
|
|
34
|
+
* Recover one interrupted GC park while the caller holds this root's add lock.
|
|
35
|
+
* Returns true when journal evidence was present.
|
|
36
|
+
*/
|
|
37
|
+
export declare function recoverInterruptedWorkflowStoreGc(args: WorkflowStoreGcRecoveryArgs): boolean;
|
|
38
|
+
/**
|
|
39
|
+
* Park one already-unindexed object under durable recovery evidence. The
|
|
40
|
+
* caller holds the root lock and must pass the returned path to
|
|
41
|
+
* removeParkedWorkflowStoreGcObject(). The journal remains durable until that
|
|
42
|
+
* path and its staging-directory entry have been removed successfully.
|
|
43
|
+
*/
|
|
44
|
+
export declare function parkWorkflowStoreGcObject(args: ParkWorkflowStoreGcObjectArgs): string;
|
|
45
|
+
/** Remove one journal-authenticated parked object and only then clear evidence. */
|
|
46
|
+
export declare function removeParkedWorkflowStoreGcObject(args: RemoveParkedWorkflowStoreGcObjectArgs): void;
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Crash recovery for GC object parking.
|
|
3
|
+
*
|
|
4
|
+
* APFS refuses to rename a hardened 0555 directory, so GC must briefly add
|
|
5
|
+
* owner-write to the object root before moving it out of the canonical CAS
|
|
6
|
+
* path. The journal below makes that otherwise-dangerous interval recoverable
|
|
7
|
+
* by every writer that shares the workflow-store lock. The pruned index is the
|
|
8
|
+
* durable commit point: recovery restores a parked object when the current
|
|
9
|
+
* index still names it, and treats it as staging debris only when the current
|
|
10
|
+
* index corroborates that the digest is no longer reachable by coordinate.
|
|
11
|
+
*/
|
|
12
|
+
import { chmodSync, closeSync, constants as fsConstants, fsyncSync, lstatSync, openSync, rmSync, } from 'node:fs';
|
|
13
|
+
import { dirname, join } from 'node:path';
|
|
14
|
+
import { guardStateFile, lockfilePathViolation, probeDirectoryPath, readRegularFileNoFollow, renameDirRestoringWrite, rmRecursiveForce, STAGING_DIRNAME, writeJsonAtomic, } from "../install.js";
|
|
15
|
+
import { readWorkflowStoreIndex } from "./index-file.js";
|
|
16
|
+
import { verifyWorkflowObjectSync } from "./ingestor.js";
|
|
17
|
+
import { storeIndexPath } from "./resolve.js";
|
|
18
|
+
import { compareStoreText, defDigest, objectDirForDigest } from "./types.js";
|
|
19
|
+
export const WORKFLOW_STORE_GC_JOURNAL_FILENAME = 'gc.journal';
|
|
20
|
+
export function workflowStoreGcJournalPath(stateDir) {
|
|
21
|
+
return join(stateDir, WORKFLOW_STORE_GC_JOURNAL_FILENAME);
|
|
22
|
+
}
|
|
23
|
+
function fsyncPath(path) {
|
|
24
|
+
const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
25
|
+
try {
|
|
26
|
+
fsyncSync(fd);
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
closeSync(fd);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/** Persist a directory-entry transition, not merely the moved directory inode. */
|
|
33
|
+
export function fsyncWorkflowStoreDirectory(path) {
|
|
34
|
+
const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
35
|
+
try {
|
|
36
|
+
fsyncSync(fd);
|
|
37
|
+
}
|
|
38
|
+
finally {
|
|
39
|
+
closeSync(fd);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Make an atomic metadata write durable before a destructive follow-on step. */
|
|
43
|
+
export function fsyncWorkflowStoreFileAndParent(path) {
|
|
44
|
+
fsyncPath(path);
|
|
45
|
+
fsyncWorkflowStoreDirectory(dirname(path));
|
|
46
|
+
}
|
|
47
|
+
function corruptGcJournal(path, detail) {
|
|
48
|
+
return new Error(`invalid workflow-store GC journal at ${path}: ${detail} — ` +
|
|
49
|
+
'inspect the target store and remove the journal manually before retrying');
|
|
50
|
+
}
|
|
51
|
+
function readGcJournal(path) {
|
|
52
|
+
const bytes = readRegularFileNoFollow(path, 'workflow-store GC journal');
|
|
53
|
+
if (bytes === undefined)
|
|
54
|
+
return undefined;
|
|
55
|
+
let parsed;
|
|
56
|
+
try {
|
|
57
|
+
parsed = JSON.parse(Buffer.from(bytes).toString('utf8'));
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
throw corruptGcJournal(path, `invalid JSON: ${error.message}`);
|
|
61
|
+
}
|
|
62
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
63
|
+
throw corruptGcJournal(path, 'expected an object');
|
|
64
|
+
}
|
|
65
|
+
const raw = parsed;
|
|
66
|
+
const keys = Object.keys(raw).sort(compareStoreText);
|
|
67
|
+
const expectedKeys = ['digest', 'originalMode', 'parkedName', 'version'];
|
|
68
|
+
if (JSON.stringify(keys) !== JSON.stringify(expectedKeys)) {
|
|
69
|
+
throw corruptGcJournal(path, `unexpected fields ${JSON.stringify(keys)}`);
|
|
70
|
+
}
|
|
71
|
+
if (raw.version !== 1)
|
|
72
|
+
throw corruptGcJournal(path, `unsupported version ${JSON.stringify(raw.version)}`);
|
|
73
|
+
if (typeof raw.digest !== 'string')
|
|
74
|
+
throw corruptGcJournal(path, 'digest must be a string');
|
|
75
|
+
let digest;
|
|
76
|
+
try {
|
|
77
|
+
digest = defDigest(raw.digest);
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
throw corruptGcJournal(path, error.message);
|
|
81
|
+
}
|
|
82
|
+
if (typeof raw.parkedName !== 'string' || lockfilePathViolation(raw.parkedName) !== undefined) {
|
|
83
|
+
throw corruptGcJournal(path, 'parkedName is not a safe path segment');
|
|
84
|
+
}
|
|
85
|
+
if (!new RegExp(`^gc-${digest}-park_[0-9a-f]{24}$`, 'u').test(raw.parkedName)) {
|
|
86
|
+
throw corruptGcJournal(path, 'parkedName does not match the recorded digest and GC transaction shape');
|
|
87
|
+
}
|
|
88
|
+
if (raw.originalMode !== 0o555) {
|
|
89
|
+
throw corruptGcJournal(path, `originalMode must be the hardened object mode 0555, got ${String(raw.originalMode)}`);
|
|
90
|
+
}
|
|
91
|
+
return { version: 1, digest, parkedName: raw.parkedName, originalMode: raw.originalMode };
|
|
92
|
+
}
|
|
93
|
+
function writeGcJournal(path, journal) {
|
|
94
|
+
guardStateFile(path, 'workflow-store GC journal');
|
|
95
|
+
writeJsonAtomic(path, journal);
|
|
96
|
+
// Stable evidence must precede the temporary live-object chmod.
|
|
97
|
+
fsyncWorkflowStoreFileAndParent(path);
|
|
98
|
+
}
|
|
99
|
+
function removeGcJournal(path, stateDir) {
|
|
100
|
+
guardStateFile(path, 'workflow-store GC journal');
|
|
101
|
+
rmSync(path, { force: true });
|
|
102
|
+
// Persist the unlink before staging cleanup. A crash can then only leak safe,
|
|
103
|
+
// unindexed staging debris; it cannot resurrect evidence for missing bytes.
|
|
104
|
+
fsyncWorkflowStoreDirectory(stateDir);
|
|
105
|
+
}
|
|
106
|
+
function transactionPaths(root, journal) {
|
|
107
|
+
const stagingRoot = join(root, STAGING_DIRNAME);
|
|
108
|
+
return {
|
|
109
|
+
live: objectDirForDigest(root, journal.digest),
|
|
110
|
+
parked: join(stagingRoot, journal.parkedName),
|
|
111
|
+
stagingRoot,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function realDirectoryState(path, label, boundary) {
|
|
115
|
+
const state = probeDirectoryPath(path, label, boundary);
|
|
116
|
+
if (state === 'absent')
|
|
117
|
+
return undefined;
|
|
118
|
+
const stat = lstatSync(path);
|
|
119
|
+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
120
|
+
throw new Error(`${label} '${path}' is not a real directory`);
|
|
121
|
+
}
|
|
122
|
+
return stat;
|
|
123
|
+
}
|
|
124
|
+
function restoreAndVerify(path, journal) {
|
|
125
|
+
const mode = lstatSync(path).mode & 0o7777;
|
|
126
|
+
const temporaryMode = journal.originalMode | 0o200;
|
|
127
|
+
if (mode !== journal.originalMode && mode !== temporaryMode) {
|
|
128
|
+
throw new Error(`transaction object '${path}' has contradictory mode ${mode.toString(8)} ` +
|
|
129
|
+
`(expected ${journal.originalMode.toString(8)} or ${temporaryMode.toString(8)})`);
|
|
130
|
+
}
|
|
131
|
+
if (mode !== journal.originalMode)
|
|
132
|
+
chmodSync(path, journal.originalMode);
|
|
133
|
+
fsyncPath(path);
|
|
134
|
+
verifyWorkflowObjectSync(path, journal.digest, { coordinateRepair: false });
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Recover one interrupted GC park while the caller holds this root's add lock.
|
|
138
|
+
* Returns true when journal evidence was present.
|
|
139
|
+
*/
|
|
140
|
+
export function recoverInterruptedWorkflowStoreGc(args) {
|
|
141
|
+
const journalPath = workflowStoreGcJournalPath(args.stateDir);
|
|
142
|
+
const journal = readGcJournal(journalPath);
|
|
143
|
+
if (journal === undefined)
|
|
144
|
+
return false;
|
|
145
|
+
const { live, parked, stagingRoot } = transactionPaths(args.root, journal);
|
|
146
|
+
const liveState = realDirectoryState(live, 'workflow-store GC live object', args.root);
|
|
147
|
+
const stagingState = probeDirectoryPath(stagingRoot, 'workflow store staging directory', args.root);
|
|
148
|
+
const parkedState = stagingState === 'dir'
|
|
149
|
+
? realDirectoryState(parked, 'workflow-store GC parked object', stagingRoot)
|
|
150
|
+
: undefined;
|
|
151
|
+
const index = readWorkflowStoreIndex(storeIndexPath(args.root));
|
|
152
|
+
const indexed = Object.values(index.entries).some((entry) => entry.digest === journal.digest);
|
|
153
|
+
if (liveState !== undefined && parkedState !== undefined) {
|
|
154
|
+
throw corruptGcJournal(journalPath, 'both the live and parked object paths exist');
|
|
155
|
+
}
|
|
156
|
+
if (liveState === undefined && parkedState === undefined) {
|
|
157
|
+
if (indexed) {
|
|
158
|
+
throw corruptGcJournal(journalPath, 'neither the live nor parked object path exists for an indexed digest');
|
|
159
|
+
}
|
|
160
|
+
// Deletion and its staging-directory fsync completed, but the process
|
|
161
|
+
// stopped before unlinking the journal. The index corroborates that no
|
|
162
|
+
// live bytes are owed, so finish the durable cleanup.
|
|
163
|
+
fsyncWorkflowStoreDirectory(stagingState === 'dir' ? stagingRoot : args.root);
|
|
164
|
+
removeGcJournal(journalPath, args.stateDir);
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
if (liveState !== undefined) {
|
|
168
|
+
// The interruption preceded (or the filesystem rolled back) the rename.
|
|
169
|
+
// Whether indexed or orphaned, the canonical path must not remain writable.
|
|
170
|
+
restoreAndVerify(live, journal);
|
|
171
|
+
fsyncWorkflowStoreDirectory(dirname(live));
|
|
172
|
+
removeGcJournal(journalPath, args.stateDir);
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
if (indexed) {
|
|
176
|
+
// The index is authoritative: parked bytes are still live state, so restore
|
|
177
|
+
// them before any shared-staging cleanup can discard them.
|
|
178
|
+
restoreAndVerify(parked, journal);
|
|
179
|
+
renameDirRestoringWrite(parked, live, journal.originalMode);
|
|
180
|
+
fsyncPath(live);
|
|
181
|
+
fsyncWorkflowStoreDirectory(stagingRoot);
|
|
182
|
+
fsyncWorkflowStoreDirectory(dirname(live));
|
|
183
|
+
verifyWorkflowObjectSync(live, journal.digest, { coordinateRepair: false });
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
// The journal authenticates this contained parked path, and the committed
|
|
187
|
+
// index proves it is doomed. A prior deletion attempt may already have
|
|
188
|
+
// removed files or made directories writable, so never require the debris
|
|
189
|
+
// to remain a complete, hardened bundle before retrying its removal.
|
|
190
|
+
rmRecursiveForce(parked);
|
|
191
|
+
fsyncWorkflowStoreDirectory(stagingRoot);
|
|
192
|
+
}
|
|
193
|
+
removeGcJournal(journalPath, args.stateDir);
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Park one already-unindexed object under durable recovery evidence. The
|
|
198
|
+
* caller holds the root lock and must pass the returned path to
|
|
199
|
+
* removeParkedWorkflowStoreGcObject(). The journal remains durable until that
|
|
200
|
+
* path and its staging-directory entry have been removed successfully.
|
|
201
|
+
*/
|
|
202
|
+
export function parkWorkflowStoreGcObject(args) {
|
|
203
|
+
if (lockfilePathViolation(args.parkedName) !== undefined) {
|
|
204
|
+
throw new Error(`refusing unsafe workflow-store GC parked name '${args.parkedName}'`);
|
|
205
|
+
}
|
|
206
|
+
const journal = {
|
|
207
|
+
version: 1,
|
|
208
|
+
digest: args.digest,
|
|
209
|
+
parkedName: args.parkedName,
|
|
210
|
+
originalMode: 0o555,
|
|
211
|
+
};
|
|
212
|
+
if (!new RegExp(`^gc-${args.digest}-park_[0-9a-f]{24}$`, 'u').test(args.parkedName)) {
|
|
213
|
+
throw new Error(`refusing malformed workflow-store GC parked name '${args.parkedName}'`);
|
|
214
|
+
}
|
|
215
|
+
const { live, parked, stagingRoot } = transactionPaths(args.root, journal);
|
|
216
|
+
const liveState = realDirectoryState(live, 'workflow-store GC live object', args.root);
|
|
217
|
+
if (liveState === undefined) {
|
|
218
|
+
throw new Error(`refusing to park workflow-store object '${live}': it disappeared`);
|
|
219
|
+
}
|
|
220
|
+
const originalMode = liveState.mode & 0o7777;
|
|
221
|
+
if (originalMode !== journal.originalMode) {
|
|
222
|
+
throw new Error(`refusing to park workflow-store object '${live}': expected hardened mode 0555`);
|
|
223
|
+
}
|
|
224
|
+
if (lstatSync(parked, { throwIfNoEntry: false }) !== undefined) {
|
|
225
|
+
throw new Error(`refusing to park workflow-store object: destination '${parked}' already exists`);
|
|
226
|
+
}
|
|
227
|
+
verifyWorkflowObjectSync(live, args.digest, { coordinateRepair: false });
|
|
228
|
+
const journalPath = workflowStoreGcJournalPath(args.stateDir);
|
|
229
|
+
writeGcJournal(journalPath, journal);
|
|
230
|
+
chmodSync(live, originalMode | 0o200);
|
|
231
|
+
args.afterLiveObjectMadeWritable?.(live);
|
|
232
|
+
renameDirRestoringWrite(live, parked, originalMode);
|
|
233
|
+
fsyncPath(parked);
|
|
234
|
+
// The rename is not durable until BOTH directory entries are stable. Keep the
|
|
235
|
+
// journal until source disappearance and destination appearance are fsynced.
|
|
236
|
+
fsyncWorkflowStoreDirectory(dirname(live));
|
|
237
|
+
fsyncWorkflowStoreDirectory(stagingRoot);
|
|
238
|
+
args.afterObjectParked?.(parked);
|
|
239
|
+
return parked;
|
|
240
|
+
}
|
|
241
|
+
/** Remove one journal-authenticated parked object and only then clear evidence. */
|
|
242
|
+
export function removeParkedWorkflowStoreGcObject(args) {
|
|
243
|
+
const journalPath = workflowStoreGcJournalPath(args.stateDir);
|
|
244
|
+
const journal = readGcJournal(journalPath);
|
|
245
|
+
if (journal === undefined) {
|
|
246
|
+
throw new Error(`refusing workflow-store GC parked cleanup '${args.parked}': journal is missing`);
|
|
247
|
+
}
|
|
248
|
+
const { parked, stagingRoot } = transactionPaths(args.root, journal);
|
|
249
|
+
if (args.parked !== parked) {
|
|
250
|
+
throw new Error(`refusing workflow-store GC parked cleanup '${args.parked}': journal authenticates '${parked}'`);
|
|
251
|
+
}
|
|
252
|
+
(args.remove ?? rmRecursiveForce)(parked);
|
|
253
|
+
if (lstatSync(parked, { throwIfNoEntry: false }) !== undefined) {
|
|
254
|
+
throw new Error(`workflow-store GC parked cleanup '${parked}' returned without removing the path`);
|
|
255
|
+
}
|
|
256
|
+
fsyncWorkflowStoreDirectory(stagingRoot);
|
|
257
|
+
removeGcJournal(journalPath, args.stateDir);
|
|
258
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reachability-based garbage collection for the content-addressed workflow
|
|
3
|
+
* store. Planning is synchronous and side-effect free. Applying recomputes the
|
|
4
|
+
* plan under the store's existing writer lock, commits the pruned index first,
|
|
5
|
+
* then parks and removes unreferenced immutable objects.
|
|
6
|
+
*/
|
|
7
|
+
import type { LedgerLookup } from '../install.ts';
|
|
8
|
+
import type { RuntimeSnapshotBundlePins } from '../store.ts';
|
|
9
|
+
import type { StoreLevel, WorkflowStoreIndex } from './types.ts';
|
|
10
|
+
export interface WorkflowStoreGcObject {
|
|
11
|
+
digest: string;
|
|
12
|
+
bytes: number;
|
|
13
|
+
coordinates: string[];
|
|
14
|
+
}
|
|
15
|
+
export interface WorkflowStoreGcReport {
|
|
16
|
+
ok: true;
|
|
17
|
+
dryRun: boolean;
|
|
18
|
+
level: StoreLevel;
|
|
19
|
+
root: string;
|
|
20
|
+
keep: number;
|
|
21
|
+
/** Number of unique object directories selected for removal. */
|
|
22
|
+
count: number;
|
|
23
|
+
/** Sum of regular-file logical sizes in selected object directories. */
|
|
24
|
+
bytes: number;
|
|
25
|
+
/** Target-index coordinates selected for removal. */
|
|
26
|
+
coordinates: string[];
|
|
27
|
+
objects: WorkflowStoreGcObject[];
|
|
28
|
+
}
|
|
29
|
+
export interface PlanWorkflowStoreGcArgs {
|
|
30
|
+
projectRoot: string;
|
|
31
|
+
globalRoot: string;
|
|
32
|
+
level: StoreLevel;
|
|
33
|
+
keep: number;
|
|
34
|
+
snapshotPins: readonly RuntimeSnapshotBundlePins[];
|
|
35
|
+
/** Exact versioned calls from currently loadable non-CAS definitions. */
|
|
36
|
+
exactCalls?: readonly string[];
|
|
37
|
+
}
|
|
38
|
+
export interface WorkflowStoreGcPlan {
|
|
39
|
+
report: WorkflowStoreGcReport;
|
|
40
|
+
/** Canonical target index to commit before deleting objects. */
|
|
41
|
+
nextIndex: WorkflowStoreIndex;
|
|
42
|
+
}
|
|
43
|
+
export interface CollectWorkflowStoreGcArgs {
|
|
44
|
+
projectRoot: string;
|
|
45
|
+
globalRoot: string;
|
|
46
|
+
level: StoreLevel;
|
|
47
|
+
keep: number;
|
|
48
|
+
yes: boolean;
|
|
49
|
+
/** Re-run inside the writer lock when `yes` is true. */
|
|
50
|
+
readSnapshotPins: () => readonly RuntimeSnapshotBundlePins[];
|
|
51
|
+
/** Re-read exact versioned calls from current project/add definitions under the writer locks. */
|
|
52
|
+
readExactCalls?: () => readonly string[];
|
|
53
|
+
/** Project mode supplies the same legacy installed-ledger reader as bundle installation. */
|
|
54
|
+
readLedger?: () => LedgerLookup;
|
|
55
|
+
/**
|
|
56
|
+
* Project GitHub-add transaction state sharing this root's staging tree.
|
|
57
|
+
* When supplied, GC takes this lock and recovers this journal before it may
|
|
58
|
+
* clear staging entries not owned by the current GC transaction.
|
|
59
|
+
*/
|
|
60
|
+
legacyRecovery?: {
|
|
61
|
+
lockPath: string;
|
|
62
|
+
journalPath: string;
|
|
63
|
+
lockfilePath: string;
|
|
64
|
+
readLedger: () => LedgerLookup;
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* Read-only coordination for non-CAS project/add definitions when their root
|
|
68
|
+
* is not the GC target. A pending journal is refused rather than recovered so
|
|
69
|
+
* global GC never mutates the non-target project tree.
|
|
70
|
+
*/
|
|
71
|
+
legacyReadBarrier?: {
|
|
72
|
+
lockPath: string;
|
|
73
|
+
journalPath: string;
|
|
74
|
+
};
|
|
75
|
+
recoveryMarkerDir?: string;
|
|
76
|
+
/** Narrow failure-injection seams for commit-order regression tests. */
|
|
77
|
+
hooks?: {
|
|
78
|
+
afterSnapshotPinsRead?: () => void;
|
|
79
|
+
afterIndexWrite?: () => void;
|
|
80
|
+
/** Process-interruption seam after durable evidence and the temporary live chmod. */
|
|
81
|
+
afterLiveObjectMadeWritable?: (path: string) => void;
|
|
82
|
+
/** Process-interruption seam after the object rename and parent fsyncs. */
|
|
83
|
+
afterObjectParked?: (path: string) => void;
|
|
84
|
+
removeParkedObject?: (path: string) => void;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Compute one deterministic, fail-closed GC plan without creating any path.
|
|
89
|
+
* `keep=2` at the CLI means the current version plus one immediate rollback
|
|
90
|
+
* version: bounded history without giving up the cheapest recovery option.
|
|
91
|
+
*/
|
|
92
|
+
export declare function planWorkflowStoreGc(args: PlanWorkflowStoreGcArgs): WorkflowStoreGcPlan;
|
|
93
|
+
/** Plan, or recompute under both store writer locks and apply, one target-root GC. */
|
|
94
|
+
export declare function collectWorkflowStoreGarbage(args: CollectWorkflowStoreGcArgs): Promise<WorkflowStoreGcReport>;
|