@bitkyc08/opencodex 2.19.0 → 2.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gui/dist/assets/index-BOFeam5a.js +76 -0
- package/gui/dist/assets/index-Xq49CY8F.css +1 -0
- package/gui/dist/index.html +2 -2
- package/package.json +1 -1
- package/src/adapters/anthropic.ts +2 -28
- package/src/adapters/google.ts +31 -3
- package/src/adapters/mimo-free.ts +17 -0
- package/src/adapters/openai-chat.ts +392 -40
- package/src/adapters/registry.ts +144 -0
- package/src/adapters/responses-tool-schema.ts +67 -0
- package/src/bridge.ts +15 -2
- package/src/chat/inbound.ts +13 -7
- package/src/claude/gateway-cache.ts +41 -4
- package/src/cli/claude.ts +3 -2
- package/src/cli/combo.ts +3 -0
- package/src/cli/dispatch.ts +8 -0
- package/src/cli/export-command.ts +2 -2
- package/src/cli/help.ts +2 -0
- package/src/cli/index.ts +3 -2
- package/src/cli/lab.ts +135 -1
- package/src/cli/minimax.ts +491 -0
- package/src/cli/models-runtime.ts +22 -1
- package/src/cli/models.ts +67 -2
- package/src/cli/opencode.ts +2 -1
- package/src/cli/registry.ts +22 -2
- package/src/clients/config-export.ts +125 -7
- package/src/codex/app-server-processes.ts +57 -2
- package/src/codex/app-server-restart-service.ts +232 -0
- package/src/codex/catalog/aggregation.ts +10 -1
- package/src/codex/catalog/effort.ts +15 -3
- package/src/codex/catalog/parsing.ts +3 -1
- package/src/codex/catalog/provider-fetch.ts +45 -5
- package/src/codex/catalog/sync.ts +74 -4
- package/src/codex/convergence.ts +2 -0
- package/src/combos/index.ts +1 -0
- package/src/combos/request.ts +30 -0
- package/src/combos/types.ts +6 -0
- package/src/config.ts +52 -0
- package/src/generated/compatibility-version.json +241 -85
- package/src/images/loop.ts +26 -6
- package/src/integrations/registry.ts +7 -0
- package/src/lab/conformance/jcs.ts +42 -2
- package/src/lab/conformance/negative-controls.ts +6 -2
- package/src/lab/conformance/runner.ts +16 -5
- package/src/lab/fabric/observe.ts +49 -14
- package/src/lab/index.ts +16 -0
- package/src/lab/ledger/purge.ts +152 -83
- package/src/lab/ledger/store.ts +168 -54
- package/src/lab/observe/from-conformance.ts +8 -6
- package/src/lab/observe/from-live.ts +8 -2
- package/src/lab/paths.ts +23 -0
- package/src/lab/public/bundle.ts +217 -0
- package/src/lab/public/community-authority.ts +175 -0
- package/src/lab/public/community-files.ts +29 -0
- package/src/lab/public/community.ts +479 -0
- package/src/lab/public/file-safety.ts +155 -0
- package/src/lab/public/ids.ts +26 -0
- package/src/lab/public/index.ts +16 -0
- package/src/lab/public/mutation-lock.ts +424 -0
- package/src/lab/public/operator.ts +353 -0
- package/src/lab/public/origin-purge.ts +79 -0
- package/src/lab/public/origin.ts +203 -0
- package/src/lab/public/privacy.ts +143 -0
- package/src/lab/public/private-file.ts +261 -0
- package/src/lab/public/project.ts +124 -0
- package/src/lab/public/purge-test-fault.ts +21 -0
- package/src/lab/public/purge.ts +223 -0
- package/src/lab/public/registry.ts +44 -0
- package/src/lab/public/revocation.ts +252 -0
- package/src/lab/public/signature.ts +219 -0
- package/src/lab/public/storage.ts +105 -0
- package/src/lab/public/strict-json.ts +206 -0
- package/src/lab/public/time.ts +26 -0
- package/src/lab/public/types.ts +172 -0
- package/src/lab/public/validate.ts +391 -0
- package/src/lib/codex-restart-contract.ts +120 -0
- package/src/lib/lab-activation.ts +109 -47
- package/src/lib/lab-live-pinned-sender.ts +16 -5
- package/src/lib/pinned-http.ts +70 -16
- package/src/lib/self-launch-argv.ts +15 -0
- package/src/lib/state-store-registrations.ts +2 -0
- package/src/lib/upstream-reachability.ts +4 -0
- package/src/lib/windows-elevation.ts +10 -1
- package/src/providers/derive.ts +24 -4
- package/src/providers/registry.ts +14 -4
- package/src/providers/request-pacing.ts +310 -0
- package/src/providers/service-tier.ts +143 -0
- package/src/providers/static-model-discovery.ts +86 -0
- package/src/reasoning-effort.ts +27 -1
- package/src/responses/custom-tool-compat.ts +4 -1
- package/src/responses/parser.ts +7 -1
- package/src/responses/provider-opaque-metadata.ts +73 -0
- package/src/responses/schema.ts +6 -0
- package/src/router.ts +23 -6
- package/src/routing/capability.ts +4 -2
- package/src/routing/compatibility/behavior.ts +5 -1
- package/src/server/adapter-resolve.ts +2 -32
- package/src/server/auth-cors.ts +50 -6
- package/src/server/chat-completions.ts +74 -36
- package/src/server/chat-native-sse.ts +331 -0
- package/src/server/chat-native.ts +371 -0
- package/src/server/management/combo-routes.ts +16 -2
- package/src/server/management/config-routes.ts +6 -4
- package/src/server/management/context.ts +17 -0
- package/src/server/management/lab-routes.ts +181 -19
- package/src/server/management/model-routes.ts +76 -2
- package/src/server/management/model-rows.ts +8 -0
- package/src/server/management/provider-capability-config.ts +48 -0
- package/src/server/management/provider-routes.ts +76 -4
- package/src/server/management/system-restart.ts +4 -2
- package/src/server/management/system-routes.ts +38 -0
- package/src/server/relay.ts +17 -3
- package/src/server/responses/compact.ts +4 -1
- package/src/server/responses/core.ts +257 -46
- package/src/server/responses/empty-completion-guard.ts +276 -0
- package/src/server/responses/fetch-helpers.ts +35 -4
- package/src/server/responses/pacing-overload.ts +13 -0
- package/src/server/responses/policy-fallback.ts +16 -2
- package/src/server/responses/terminal-guard.ts +1 -1
- package/src/server/responses/upstream-error.ts +5 -0
- package/src/server/responses.ts +17 -2
- package/src/server/system-env.ts +1 -1
- package/src/types.ts +84 -5
- package/src/update/index.ts +6 -5
- package/src/update/job.ts +5 -6
- package/src/update/notify.ts +5 -3
- package/src/usage/log.ts +11 -1
- package/src/web-search/loop.ts +21 -5
- package/gui/dist/assets/index-CQ7bIKee.css +0 -1
- package/gui/dist/assets/index-D_JUZLEC.js +0 -76
package/src/lab/ledger/purge.ts
CHANGED
|
@@ -18,10 +18,11 @@ import {
|
|
|
18
18
|
expandSensitiveArtifactEventTargets,
|
|
19
19
|
} from "./artifact-refs";
|
|
20
20
|
import { buildInvalidationIndex } from "./invalidation";
|
|
21
|
-
import {
|
|
21
|
+
import { withLedgerMutation } from "./store";
|
|
22
22
|
import { ensureLabDirs } from "../paths";
|
|
23
23
|
import { rebuildLabProjection } from "../projection/rebuild";
|
|
24
24
|
import { jcsStringify } from "../digest";
|
|
25
|
+
import { purgeLocalPublicEvidenceCopies } from "../public/purge";
|
|
25
26
|
import {
|
|
26
27
|
closeSync,
|
|
27
28
|
existsSync,
|
|
@@ -34,7 +35,7 @@ import {
|
|
|
34
35
|
writeSync,
|
|
35
36
|
} from "node:fs";
|
|
36
37
|
import { dirname, join } from "node:path";
|
|
37
|
-
|
|
38
|
+
|
|
38
39
|
export class PurgeError extends Error {
|
|
39
40
|
readonly code: string;
|
|
40
41
|
readonly completedActions: string[];
|
|
@@ -45,7 +46,7 @@ export class PurgeError extends Error {
|
|
|
45
46
|
this.completedActions = [...completedActions];
|
|
46
47
|
}
|
|
47
48
|
}
|
|
48
|
-
|
|
49
|
+
|
|
49
50
|
export interface SensitivePurgeRequest {
|
|
50
51
|
configDir?: string;
|
|
51
52
|
targetEventIds?: string[];
|
|
@@ -54,7 +55,7 @@ export interface SensitivePurgeRequest {
|
|
|
54
55
|
recordedAt?: number;
|
|
55
56
|
producerVersion?: string;
|
|
56
57
|
}
|
|
57
|
-
|
|
58
|
+
|
|
58
59
|
function writeAll(fd: number, bytes: Uint8Array): void {
|
|
59
60
|
let offset = 0;
|
|
60
61
|
while (offset < bytes.byteLength) {
|
|
@@ -63,7 +64,7 @@ function writeAll(fd: number, bytes: Uint8Array): void {
|
|
|
63
64
|
offset += n;
|
|
64
65
|
}
|
|
65
66
|
}
|
|
66
|
-
|
|
67
|
+
|
|
67
68
|
function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void {
|
|
68
69
|
const body = events.map((e) => jcsStringify(e)).join("\n") + (events.length ? "\n" : "");
|
|
69
70
|
const bytes = new TextEncoder().encode(body);
|
|
@@ -90,7 +91,7 @@ function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void {
|
|
|
90
91
|
}
|
|
91
92
|
throw err;
|
|
92
93
|
}
|
|
93
|
-
|
|
94
|
+
|
|
94
95
|
if (process.platform !== "win32") {
|
|
95
96
|
try {
|
|
96
97
|
const dirFd = openSync(parent, "r");
|
|
@@ -100,8 +101,6 @@ function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void {
|
|
|
100
101
|
closeSync(dirFd);
|
|
101
102
|
}
|
|
102
103
|
} catch (err) {
|
|
103
|
-
// The rename is already committed and visible. Report durability failure
|
|
104
|
-
// without pretending the ledger action can be rolled back.
|
|
105
104
|
throw new PurgeError(
|
|
106
105
|
"ledger_durability_failed",
|
|
107
106
|
`ledger rewrite committed but directory fsync failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -110,7 +109,7 @@ function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void {
|
|
|
110
109
|
}
|
|
111
110
|
}
|
|
112
111
|
}
|
|
113
|
-
|
|
112
|
+
|
|
114
113
|
function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void {
|
|
115
114
|
const errors: string[] = [];
|
|
116
115
|
for (const digest of digests) {
|
|
@@ -125,7 +124,7 @@ function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]):
|
|
|
125
124
|
throw new PurgeError("artifact_delete_failed", errors.join("; "));
|
|
126
125
|
}
|
|
127
126
|
}
|
|
128
|
-
|
|
127
|
+
|
|
129
128
|
function purgeBoundedDirectory(dirPath: string): void {
|
|
130
129
|
if (!existsSync(dirPath)) return;
|
|
131
130
|
const entries = readdirSync(dirPath, { withFileTypes: true });
|
|
@@ -141,7 +140,41 @@ function purgeBoundedDirectory(dirPath: string): void {
|
|
|
141
140
|
}
|
|
142
141
|
}
|
|
143
142
|
}
|
|
144
|
-
|
|
143
|
+
|
|
144
|
+
function normalizePurgeError(err: unknown, completed: readonly string[]): PurgeError {
|
|
145
|
+
if (err instanceof PurgeError) {
|
|
146
|
+
return new PurgeError(
|
|
147
|
+
err.code,
|
|
148
|
+
err.message,
|
|
149
|
+
[...new Set([...completed, ...err.completedActions])],
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
return new PurgeError(
|
|
153
|
+
"purge_failed",
|
|
154
|
+
err instanceof Error ? err.message : String(err),
|
|
155
|
+
[...completed],
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function buildPurgeTombstone(
|
|
160
|
+
req: SensitivePurgeRequest,
|
|
161
|
+
removeIds: ReadonlySet<string>,
|
|
162
|
+
targetArtifactDigests: string[],
|
|
163
|
+
purgeActions: Array<(typeof PURGE_ACTIONS)[number]>,
|
|
164
|
+
): PurgeTombstoneEvent {
|
|
165
|
+
return validateLabEvent(assignEventId({
|
|
166
|
+
schemaVersion: LAB_EVENT_SCHEMA_VERSION,
|
|
167
|
+
eventKind: "purge_tombstone" as const,
|
|
168
|
+
recordedAt: req.recordedAt ?? Date.now(),
|
|
169
|
+
producer: LAB_PRODUCER,
|
|
170
|
+
producerVersion: req.producerVersion ?? LAB_PRODUCER_VERSION,
|
|
171
|
+
targetEventIds: [...removeIds].sort(),
|
|
172
|
+
targetArtifactDigests,
|
|
173
|
+
reason: "sensitive_evidence" as const,
|
|
174
|
+
purgeActions,
|
|
175
|
+
})) as PurgeTombstoneEvent;
|
|
176
|
+
}
|
|
177
|
+
|
|
145
178
|
/**
|
|
146
179
|
* Exceptional sensitive-evidence purge:
|
|
147
180
|
* physically remove targeted JSONL lines and artifacts, append purge_tombstone,
|
|
@@ -153,89 +186,125 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto
|
|
|
153
186
|
const targetArtifactDigests = [...(req.targetArtifactDigests ?? [])].sort();
|
|
154
187
|
const purgeActions = [...(req.purgeActions ?? PURGE_ACTIONS)].sort();
|
|
155
188
|
const explicitSensitive = new Set(targetArtifactDigests);
|
|
156
|
-
|
|
157
|
-
const replay = replayLabLedger(paths.ledgerPath);
|
|
158
|
-
const index = buildInvalidationIndex(replay.events);
|
|
159
|
-
const removeIds = expandSensitiveArtifactEventTargets(
|
|
160
|
-
replay.events,
|
|
161
|
-
index,
|
|
162
|
-
new Set(targetEventIds),
|
|
163
|
-
explicitSensitive,
|
|
164
|
-
);
|
|
165
|
-
|
|
166
|
-
const tombstonePayload = {
|
|
167
|
-
schemaVersion: LAB_EVENT_SCHEMA_VERSION,
|
|
168
|
-
eventKind: "purge_tombstone" as const,
|
|
169
|
-
recordedAt: req.recordedAt ?? Date.now(),
|
|
170
|
-
producer: LAB_PRODUCER,
|
|
171
|
-
producerVersion: req.producerVersion ?? LAB_PRODUCER_VERSION,
|
|
172
|
-
targetEventIds: [...removeIds].sort(),
|
|
173
|
-
targetArtifactDigests,
|
|
174
|
-
reason: "sensitive_evidence" as const,
|
|
175
|
-
purgeActions,
|
|
176
|
-
};
|
|
177
|
-
const tombstone = validateLabEvent(assignEventId(tombstonePayload)) as PurgeTombstoneEvent;
|
|
178
|
-
|
|
179
|
-
const deletionPlan = purgeActions.includes("artifact")
|
|
180
|
-
? artifactDeletionPlan(replay.events, index, removeIds, targetArtifactDigests)
|
|
181
|
-
: { deletable: [], retainedExplicit: [] };
|
|
182
|
-
|
|
183
|
-
if (deletionPlan.retainedExplicit.length > 0) {
|
|
184
|
-
throw new PurgeError(
|
|
185
|
-
"sensitive_bytes_retained",
|
|
186
|
-
`explicit sensitive artifacts remain required: ${deletionPlan.retainedExplicit.join(",")}`,
|
|
187
|
-
);
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
let dir: TrustedArtifactDir | null = null;
|
|
191
189
|
const completed: string[] = [];
|
|
190
|
+
// Cell object: the mutation callback assigns through the property, which keeps
|
|
191
|
+
// the post-mutation read at the declared type (a closure-captured let would
|
|
192
|
+
// narrow to null and break the combined-error report below).
|
|
193
|
+
const deferredExport: { error: PurgeError | null } = { error: null };
|
|
194
|
+
let operationError: PurgeError | null = null;
|
|
195
|
+
let tombstone: PurgeTombstoneEvent | null = null;
|
|
196
|
+
|
|
192
197
|
try {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
198
|
+
tombstone = withLedgerMutation(paths.ledgerPath, (ledger) => {
|
|
199
|
+
// Replay and plan under the same lock as every append. Otherwise an event
|
|
200
|
+
// appended after this snapshot can be lost by the atomic rename or can
|
|
201
|
+
// start referencing an artifact after the deletion plan was calculated.
|
|
202
|
+
const replay = ledger.replay();
|
|
203
|
+
const index = buildInvalidationIndex(replay.events);
|
|
204
|
+
const removeIds = expandSensitiveArtifactEventTargets(
|
|
205
|
+
replay.events,
|
|
206
|
+
index,
|
|
207
|
+
new Set(targetEventIds),
|
|
208
|
+
explicitSensitive,
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
const deletionPlan = purgeActions.includes("artifact")
|
|
212
|
+
? artifactDeletionPlan(replay.events, index, removeIds, targetArtifactDigests)
|
|
213
|
+
: { deletable: [], retainedExplicit: [] };
|
|
214
|
+
|
|
215
|
+
if (deletionPlan.retainedExplicit.length > 0) {
|
|
216
|
+
throw new PurgeError(
|
|
217
|
+
"sensitive_bytes_retained",
|
|
218
|
+
`explicit sensitive artifacts remain required: ${deletionPlan.retainedExplicit.join(",")}`,
|
|
219
|
+
);
|
|
206
220
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
const kept: LabEvent[] = [];
|
|
212
|
-
for (const event of replay.events) {
|
|
213
|
-
if (removeIds.has(event.eventId)) continue;
|
|
214
|
-
kept.push(event);
|
|
221
|
+
|
|
222
|
+
if (purgeActions.includes("scratch")) {
|
|
223
|
+
purgeBoundedDirectory(paths.scratchDir);
|
|
224
|
+
completed.push("scratch");
|
|
215
225
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
226
|
+
if (purgeActions.includes("export")) {
|
|
227
|
+
try {
|
|
228
|
+
purgeLocalPublicEvidenceCopies(req.configDir);
|
|
229
|
+
completed.push("export");
|
|
230
|
+
} catch (err) {
|
|
231
|
+
// Export deletion is independent from artifact/ledger/sqlite deletion. Keep
|
|
232
|
+
// deleting every other requested sensitive copy, then report this failure.
|
|
233
|
+
deferredExport.error = normalizePurgeError(err, completed);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
let dir: TrustedArtifactDir | null = null;
|
|
238
|
+
try {
|
|
239
|
+
if (purgeActions.includes("artifact")) {
|
|
240
|
+
if (deletionPlan.deletable.length > 0) {
|
|
241
|
+
dir = openTrustedArtifactDir(paths.artifactsDir);
|
|
242
|
+
deleteArtifactsFailClosed(dir, deletionPlan.deletable);
|
|
243
|
+
}
|
|
244
|
+
completed.push("artifact");
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Never persist a tombstone claiming that export completed when the export
|
|
248
|
+
// purge failed. Other independent actions remain recordable and continue.
|
|
249
|
+
const tombstoneActions = deferredExport.error
|
|
250
|
+
? purgeActions.filter((action) => action !== "export")
|
|
251
|
+
: purgeActions;
|
|
252
|
+
const hasTombstoneTarget = removeIds.size > 0
|
|
253
|
+
|| targetArtifactDigests.length > 0
|
|
254
|
+
|| tombstoneActions.includes("scratch")
|
|
255
|
+
|| tombstoneActions.includes("export");
|
|
256
|
+
|
|
257
|
+
if (tombstoneActions.length > 0 && (hasTombstoneTarget || !deferredExport.error)) {
|
|
258
|
+
const mutationTombstone = buildPurgeTombstone(req, removeIds, targetArtifactDigests, tombstoneActions);
|
|
259
|
+
if (purgeActions.includes("ledger")) {
|
|
260
|
+
const kept: LabEvent[] = [];
|
|
261
|
+
for (const event of replay.events) {
|
|
262
|
+
if (removeIds.has(event.eventId)) continue;
|
|
263
|
+
kept.push(event);
|
|
264
|
+
}
|
|
265
|
+
kept.push(mutationTombstone);
|
|
266
|
+
atomicRewriteLedger(paths.ledgerPath, kept);
|
|
267
|
+
completed.push("ledger");
|
|
268
|
+
} else {
|
|
269
|
+
ledger.append(mutationTombstone);
|
|
270
|
+
}
|
|
271
|
+
return mutationTombstone;
|
|
272
|
+
}
|
|
273
|
+
return null;
|
|
274
|
+
} finally {
|
|
275
|
+
if (dir) closeTrustedArtifactDir(dir);
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
// SQLite is disposable and rebuildLabProjection replays the canonical
|
|
280
|
+
// ledger again, so it does not need to extend the mutation lock duration.
|
|
223
281
|
if (purgeActions.includes("sqlite")) {
|
|
224
282
|
rebuildLabProjection(req.configDir);
|
|
225
283
|
completed.push("sqlite");
|
|
226
284
|
}
|
|
227
|
-
|
|
228
|
-
return tombstone;
|
|
229
285
|
} catch (err) {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
286
|
+
operationError = normalizePurgeError(err, completed);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const deferredExportError = deferredExport.error;
|
|
290
|
+
if (operationError && deferredExportError) {
|
|
233
291
|
throw new PurgeError(
|
|
234
292
|
"purge_failed",
|
|
235
|
-
|
|
236
|
-
completed,
|
|
293
|
+
`export purge failed: ${deferredExportError.message}; subsequent purge failure (${operationError.code}): ${operationError.message}`,
|
|
294
|
+
[...new Set([...completed, ...deferredExportError.completedActions, ...operationError.completedActions])],
|
|
237
295
|
);
|
|
238
|
-
} finally {
|
|
239
|
-
if (dir) closeTrustedArtifactDir(dir);
|
|
240
296
|
}
|
|
297
|
+
if (operationError) throw operationError;
|
|
298
|
+
if (deferredExportError) {
|
|
299
|
+
throw new PurgeError(
|
|
300
|
+
deferredExportError.code,
|
|
301
|
+
deferredExportError.message,
|
|
302
|
+
[...new Set([...completed, ...deferredExportError.completedActions])],
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
if (!tombstone) {
|
|
306
|
+
throw new PurgeError("purge_failed", "purge completed without a durable tombstone", completed);
|
|
307
|
+
}
|
|
308
|
+
return tombstone;
|
|
241
309
|
}
|
|
310
|
+
|
package/src/lab/ledger/store.ts
CHANGED
|
@@ -25,6 +25,12 @@ export interface LedgerStore {
|
|
|
25
25
|
replay(): ReplayResult;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
export interface LedgerMutationContext {
|
|
29
|
+
replay(): ReplayResult;
|
|
30
|
+
append(event: LabEvent): void;
|
|
31
|
+
appendIfAbsent(event: LabEvent): boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
28
34
|
const LEDGER_LOCK_STALE_MS = 60_000;
|
|
29
35
|
const LEDGER_LOCK_WAIT_MS = 5_000;
|
|
30
36
|
|
|
@@ -36,10 +42,7 @@ interface LedgerLockMeta {
|
|
|
36
42
|
|
|
37
43
|
/** Block synchronously for the given duration (ledger lock retry only). */
|
|
38
44
|
function sleepSyncMs(ms: number): void {
|
|
39
|
-
|
|
40
|
-
while (Date.now() < end) {
|
|
41
|
-
/* spin */
|
|
42
|
-
}
|
|
45
|
+
Bun.sleepSync(ms);
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
/** Read pid, createdAt, and token metadata from a ledger lock file, if well-formed. */
|
|
@@ -83,8 +86,94 @@ function isLedgerLockStale(lockPath: string): boolean {
|
|
|
83
86
|
return false;
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
|
-
|
|
87
|
-
|
|
89
|
+
return !isLockHolderAlive(meta.pid);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Write lock ownership metadata to a newly created exclusive lock file. */
|
|
93
|
+
function writeLedgerLockMeta(fd: number, token: string): void {
|
|
94
|
+
const metadataBytes = Buffer.from(JSON.stringify({
|
|
95
|
+
pid: process.pid,
|
|
96
|
+
createdAt: Date.now(),
|
|
97
|
+
token,
|
|
98
|
+
}), "utf8");
|
|
99
|
+
let written = 0;
|
|
100
|
+
while (written < metadataBytes.byteLength) {
|
|
101
|
+
const n = writeSync(fd, metadataBytes, written, metadataBytes.byteLength - written);
|
|
102
|
+
if (n <= 0) {
|
|
103
|
+
throw new LabValidationError("short_write", "ledger lock metadata write incomplete");
|
|
104
|
+
}
|
|
105
|
+
written += n;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Discard a lock whose exclusive creator failed before publishing ownership metadata. */
|
|
110
|
+
function discardUninitialisedLedgerLock(lockPath: string, lockFd: number): void {
|
|
111
|
+
try {
|
|
112
|
+
closeSync(lockFd);
|
|
113
|
+
} catch {
|
|
114
|
+
/* ignore */
|
|
115
|
+
}
|
|
116
|
+
try {
|
|
117
|
+
unlinkSync(lockPath);
|
|
118
|
+
} catch {
|
|
119
|
+
/* best-effort */
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Release a lock file only when the token still matches the path owner. */
|
|
124
|
+
function releaseLedgerLock(lockPath: string, lockFd: number, token: string): void {
|
|
125
|
+
try {
|
|
126
|
+
closeSync(lockFd);
|
|
127
|
+
} catch {
|
|
128
|
+
/* ignore */
|
|
129
|
+
}
|
|
130
|
+
try {
|
|
131
|
+
const meta = readLedgerLockMeta(lockPath);
|
|
132
|
+
if (meta?.token === token) unlinkSync(lockPath);
|
|
133
|
+
} catch {
|
|
134
|
+
/* best-effort */
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Recover one stale lock while holding a separate recovery mutex.
|
|
140
|
+
*
|
|
141
|
+
* The recovery mutex prevents two waiters from both observing the same stale
|
|
142
|
+
* owner and then unlinking each other's replacement lock. If a process dies
|
|
143
|
+
* while holding the recovery mutex, acquisition fails closed instead of
|
|
144
|
+
* guessing ownership of that mutex.
|
|
145
|
+
*/
|
|
146
|
+
function recoverStaleLedgerLock(lockPath: string): boolean {
|
|
147
|
+
const recoveryPath = `${lockPath}.recovery`;
|
|
148
|
+
const token = randomBytes(16).toString("hex");
|
|
149
|
+
let recoveryFd: number;
|
|
150
|
+
try {
|
|
151
|
+
recoveryFd = openSync(
|
|
152
|
+
recoveryPath,
|
|
153
|
+
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY,
|
|
154
|
+
0o600,
|
|
155
|
+
);
|
|
156
|
+
} catch (error) {
|
|
157
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST") return false;
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
try {
|
|
162
|
+
writeLedgerLockMeta(recoveryFd, token);
|
|
163
|
+
} catch (error) {
|
|
164
|
+
discardUninitialisedLedgerLock(recoveryPath, recoveryFd);
|
|
165
|
+
throw error;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
try {
|
|
169
|
+
// Re-check after taking the recovery mutex. Another waiter may already
|
|
170
|
+
// have recovered the old lock and installed a live replacement.
|
|
171
|
+
if (!existsSync(lockPath) || !isLedgerLockStale(lockPath)) return false;
|
|
172
|
+
unlinkSync(lockPath);
|
|
173
|
+
return true;
|
|
174
|
+
} finally {
|
|
175
|
+
releaseLedgerLock(recoveryPath, recoveryFd, token);
|
|
176
|
+
}
|
|
88
177
|
}
|
|
89
178
|
|
|
90
179
|
/** Create a ledger lock file exclusively, recovering stale locks when needed. */
|
|
@@ -97,30 +186,19 @@ function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number;
|
|
|
97
186
|
} catch (error) {
|
|
98
187
|
if (existsSync(lockPath) && isLedgerLockStale(lockPath)) {
|
|
99
188
|
try {
|
|
100
|
-
|
|
101
|
-
} catch (
|
|
102
|
-
if (Date.now() >= deadline) throw
|
|
103
|
-
sleepSyncMs(10);
|
|
189
|
+
if (recoverStaleLedgerLock(lockPath)) continue;
|
|
190
|
+
} catch (recoveryError) {
|
|
191
|
+
if (Date.now() >= deadline) throw recoveryError;
|
|
104
192
|
}
|
|
105
|
-
continue;
|
|
106
193
|
}
|
|
107
194
|
if (Date.now() >= deadline) throw error;
|
|
108
195
|
sleepSyncMs(10);
|
|
109
196
|
continue;
|
|
110
197
|
}
|
|
111
198
|
try {
|
|
112
|
-
|
|
113
|
-
const written = writeSync(fd, metadataBytes);
|
|
114
|
-
if (written !== metadataBytes.byteLength) {
|
|
115
|
-
throw new LabValidationError("short_write", "ledger lock metadata write incomplete");
|
|
116
|
-
}
|
|
199
|
+
writeLedgerLockMeta(fd, token);
|
|
117
200
|
} catch (error) {
|
|
118
|
-
|
|
119
|
-
try {
|
|
120
|
-
unlinkSync(lockPath);
|
|
121
|
-
} catch {
|
|
122
|
-
/* best-effort */
|
|
123
|
-
}
|
|
201
|
+
discardUninitialisedLedgerLock(lockPath, fd);
|
|
124
202
|
throw error;
|
|
125
203
|
}
|
|
126
204
|
return { fd, token };
|
|
@@ -128,21 +206,6 @@ function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number;
|
|
|
128
206
|
throw new Error("ledger lock acquisition timed out");
|
|
129
207
|
}
|
|
130
208
|
|
|
131
|
-
/** Release a ledger lock only when the token still matches the lock file. */
|
|
132
|
-
function releaseLedgerLock(lockPath: string, lockFd: number, token: string): void {
|
|
133
|
-
try {
|
|
134
|
-
closeSync(lockFd);
|
|
135
|
-
} catch {
|
|
136
|
-
/* ignore */
|
|
137
|
-
}
|
|
138
|
-
try {
|
|
139
|
-
const meta = readLedgerLockMeta(lockPath);
|
|
140
|
-
if (meta?.token === token) unlinkSync(lockPath);
|
|
141
|
-
} catch {
|
|
142
|
-
/* best-effort */
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
209
|
/** Run a ledger mutation while holding the compatibility ledger lock file. */
|
|
147
210
|
function withLedgerLock<T>(ledgerPath: string, fn: () => T): T {
|
|
148
211
|
const lockPath = `${ledgerPath}.lock`;
|
|
@@ -156,11 +219,10 @@ function withLedgerLock<T>(ledgerPath: string, fn: () => T): T {
|
|
|
156
219
|
}
|
|
157
220
|
}
|
|
158
221
|
|
|
159
|
-
/** Durable append of one validated event as a single JSONL line + fsync. */
|
|
160
|
-
|
|
161
|
-
const validated = validateLabEvent(event);
|
|
222
|
+
/** Durable append of one already-validated event as a single JSONL line + fsync. */
|
|
223
|
+
function appendValidatedLabEvent(ledgerPath: string, event: LabEvent): void {
|
|
162
224
|
mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 });
|
|
163
|
-
const line = `${jcsStringify(
|
|
225
|
+
const line = `${jcsStringify(event)}\n`;
|
|
164
226
|
const bytes = new TextEncoder().encode(line);
|
|
165
227
|
const fd = openSync(ledgerPath, "a", 0o600);
|
|
166
228
|
try {
|
|
@@ -178,26 +240,78 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void {
|
|
|
178
240
|
}
|
|
179
241
|
}
|
|
180
242
|
|
|
243
|
+
function isThenable(value: unknown): value is PromiseLike<unknown> {
|
|
244
|
+
return (
|
|
245
|
+
(typeof value === "object" && value !== null) || typeof value === "function"
|
|
246
|
+
) && typeof (value as { then?: unknown }).then === "function";
|
|
247
|
+
}
|
|
248
|
+
|
|
181
249
|
/**
|
|
182
|
-
*
|
|
183
|
-
*
|
|
250
|
+
* Serialize a ledger read-modify-write transaction with all ordinary appends.
|
|
251
|
+
* The callback is intentionally synchronous. Mutation methods become invalid
|
|
252
|
+
* as soon as the callback returns, so an accidental async continuation cannot
|
|
253
|
+
* write after the lock has been released.
|
|
184
254
|
*/
|
|
185
|
-
export function
|
|
186
|
-
|
|
255
|
+
export function withLedgerMutation<T>(
|
|
256
|
+
ledgerPath: string,
|
|
257
|
+
fn: (mutation: LedgerMutationContext) => T,
|
|
258
|
+
): T {
|
|
187
259
|
return withLedgerLock(ledgerPath, () => {
|
|
188
|
-
|
|
189
|
-
const
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
260
|
+
let active = true;
|
|
261
|
+
const requireActive = () => {
|
|
262
|
+
if (!active) {
|
|
263
|
+
throw new LabValidationError(
|
|
264
|
+
"inactive_ledger_mutation",
|
|
265
|
+
"ledger mutation context used after its lock was released",
|
|
266
|
+
);
|
|
193
267
|
}
|
|
268
|
+
};
|
|
269
|
+
const replay = () => {
|
|
270
|
+
requireActive();
|
|
271
|
+
return replayLabLedger(ledgerPath);
|
|
272
|
+
};
|
|
273
|
+
const append = (event: LabEvent) => {
|
|
274
|
+
requireActive();
|
|
275
|
+
appendValidatedLabEvent(ledgerPath, validateLabEvent(event));
|
|
276
|
+
};
|
|
277
|
+
const appendIfAbsent = (event: LabEvent): boolean => {
|
|
278
|
+
requireActive();
|
|
279
|
+
const validated = validateLabEvent(event);
|
|
280
|
+
if (replay().events.some((row) => row.eventId === validated.eventId)) return false;
|
|
281
|
+
appendValidatedLabEvent(ledgerPath, validated);
|
|
282
|
+
return true;
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
try {
|
|
286
|
+
const result = fn({ replay, append, appendIfAbsent });
|
|
287
|
+
if (isThenable(result)) {
|
|
288
|
+
throw new LabValidationError(
|
|
289
|
+
"async_ledger_mutation",
|
|
290
|
+
"ledger mutation callback must be synchronous",
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
return result;
|
|
294
|
+
} finally {
|
|
295
|
+
active = false;
|
|
194
296
|
}
|
|
195
|
-
if (fresh.has(validated.eventId)) return false;
|
|
196
|
-
appendLabEvent(ledgerPath, validated);
|
|
197
|
-
return true;
|
|
198
297
|
});
|
|
199
298
|
}
|
|
200
299
|
|
|
300
|
+
/** Durable append of one validated event as a single JSONL line + fsync. */
|
|
301
|
+
export function appendLabEvent(ledgerPath: string, event: LabEvent): void {
|
|
302
|
+
withLedgerMutation(ledgerPath, (mutation) => {
|
|
303
|
+
mutation.append(event);
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Append only when eventId is absent. Uses the same mutation lock as every
|
|
309
|
+
* other ledger writer so the presence check and append are one transaction.
|
|
310
|
+
*/
|
|
311
|
+
export function appendLabEventIfAbsent(ledgerPath: string, event: LabEvent): boolean {
|
|
312
|
+
return withLedgerMutation(ledgerPath, (mutation) => mutation.appendIfAbsent(event));
|
|
313
|
+
}
|
|
314
|
+
|
|
201
315
|
function processLine(
|
|
202
316
|
line: string,
|
|
203
317
|
lineNumber: number,
|
|
@@ -415,4 +529,4 @@ export function openLedgerStore(configDir?: string): LedgerStore {
|
|
|
415
529
|
|
|
416
530
|
export function defaultLedgerPath(configDir?: string): string {
|
|
417
531
|
return labLedgerPath(configDir);
|
|
418
|
-
}
|
|
532
|
+
}
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
} from "../digest";
|
|
18
18
|
import type { ObservationEvent } from "../events/types";
|
|
19
19
|
import { assignEventId } from "../events/validate";
|
|
20
|
-
import {
|
|
20
|
+
import { withLedgerMutation } from "../ledger/store";
|
|
21
21
|
import { ensureLabDirs } from "../paths";
|
|
22
22
|
import type {
|
|
23
23
|
CaseAuthority,
|
|
@@ -285,12 +285,14 @@ export function persistConformanceResult(
|
|
|
285
285
|
const ownsStore = !opts.artifactStore;
|
|
286
286
|
const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir);
|
|
287
287
|
try {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
288
|
+
return withLedgerMutation(paths.ledgerPath, (ledger) => {
|
|
289
|
+
const { event } = observationFromConformanceResult(result, caseRecord, authority, {
|
|
290
|
+
...opts,
|
|
291
|
+
artifactStore: store,
|
|
292
|
+
});
|
|
293
|
+
ledger.append(event);
|
|
294
|
+
return { event, ledgerPath: paths.ledgerPath };
|
|
291
295
|
});
|
|
292
|
-
appendLabEvent(paths.ledgerPath, event);
|
|
293
|
-
return { event, ledgerPath: paths.ledgerPath };
|
|
294
296
|
} finally {
|
|
295
297
|
if (ownsStore) store.close();
|
|
296
298
|
}
|
|
@@ -5,7 +5,7 @@ import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, LAB_PRODUCER_VERSION, OBSERVATI
|
|
|
5
5
|
import { fixtureDigest, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest";
|
|
6
6
|
import type { FailureRecordV1, ObservationEvent } from "../events/types";
|
|
7
7
|
import { assignEventId } from "../events/validate";
|
|
8
|
-
import {
|
|
8
|
+
import { withLedgerMutation } from "../ledger/store";
|
|
9
9
|
import { ensureLabDirs } from "../paths";
|
|
10
10
|
import type { CaseAuthority, CaseRecord } from "../conformance/types";
|
|
11
11
|
import { trustedLiveResultRetryable } from "../live/executor";
|
|
@@ -106,6 +106,12 @@ export function observationFromLiveResult(result: LiveScenarioRunResult, caseRec
|
|
|
106
106
|
|
|
107
107
|
export function persistLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): PersistedLiveObservation {
|
|
108
108
|
const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir);
|
|
109
|
-
try {
|
|
109
|
+
try {
|
|
110
|
+
return withLedgerMutation(paths.ledgerPath, (ledger) => {
|
|
111
|
+
const { event } = observationFromLiveResult(result, caseRecord, authority, { ...opts, artifactStore: store });
|
|
112
|
+
ledger.append(event);
|
|
113
|
+
return { event, ledgerPath: paths.ledgerPath };
|
|
114
|
+
});
|
|
115
|
+
}
|
|
110
116
|
finally { if (ownsStore) store.close(); }
|
|
111
117
|
}
|