@prismatic-io/lux 0.0.1 → 0.0.2-preview.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/answerers/persona/index.d.ts +2 -1
- package/lib/answerers/persona/index.d.ts.map +1 -1
- package/lib/answerers/persona/index.js +19 -5
- package/lib/answerers/persona/index.js.map +1 -1
- package/lib/answerers/terminal/index.js +1 -1
- package/lib/answerers/terminal/index.js.map +1 -1
- package/lib/assertions/authoring.d.ts +14 -0
- package/lib/assertions/authoring.d.ts.map +1 -0
- package/lib/assertions/authoring.js +3 -0
- package/lib/assertions/authoring.js.map +1 -0
- package/lib/assertions/core/helpers.d.ts +2 -7
- package/lib/assertions/core/helpers.d.ts.map +1 -1
- package/lib/assertions/core/helpers.js +2 -6
- package/lib/assertions/core/helpers.js.map +1 -1
- package/lib/assertions/core/index.d.ts.map +1 -1
- package/lib/assertions/core/index.js +2 -1
- package/lib/assertions/core/index.js.map +1 -1
- package/lib/assertions/core/resource-checks.d.ts +19 -0
- package/lib/assertions/core/resource-checks.d.ts.map +1 -1
- package/lib/assertions/core/resource-checks.js +32 -0
- package/lib/assertions/core/resource-checks.js.map +1 -1
- package/lib/authoring.d.ts +1 -0
- package/lib/authoring.d.ts.map +1 -1
- package/lib/authoring.js +1 -0
- package/lib/authoring.js.map +1 -1
- package/lib/cli/program.d.ts.map +1 -1
- package/lib/cli/program.js +12 -67
- package/lib/cli/program.js.map +1 -1
- package/lib/cli/render/reporter.d.ts.map +1 -1
- package/lib/cli/render/reporter.js +16 -3
- package/lib/cli/render/reporter.js.map +1 -1
- package/lib/cli/run-options.d.ts +1 -0
- package/lib/cli/run-options.d.ts.map +1 -1
- package/lib/cli/run-options.js.map +1 -1
- package/lib/cli/view.d.ts +13 -0
- package/lib/cli/view.d.ts.map +1 -0
- package/lib/cli/view.js +126 -0
- package/lib/cli/view.js.map +1 -0
- package/lib/core/action-events.d.ts +29 -0
- package/lib/core/action-events.d.ts.map +1 -0
- package/lib/core/action-events.js +51 -0
- package/lib/core/action-events.js.map +1 -0
- package/lib/core/agent-activity.d.ts +49 -0
- package/lib/core/agent-activity.d.ts.map +1 -0
- package/lib/core/agent-activity.js +206 -0
- package/lib/core/agent-activity.js.map +1 -0
- package/lib/core/agent-events.d.ts +26 -0
- package/lib/core/agent-events.d.ts.map +1 -0
- package/lib/core/agent-events.js +13 -0
- package/lib/core/agent-events.js.map +1 -0
- package/lib/core/artifact-evidence.d.ts +8 -0
- package/lib/core/artifact-evidence.d.ts.map +1 -1
- package/lib/core/artifact-evidence.js +36 -4
- package/lib/core/artifact-evidence.js.map +1 -1
- package/lib/core/assertion.d.ts +19 -9
- package/lib/core/assertion.d.ts.map +1 -1
- package/lib/core/assertion.js +26 -0
- package/lib/core/assertion.js.map +1 -1
- package/lib/core/case-authoring-state.d.ts +12 -0
- package/lib/core/case-authoring-state.d.ts.map +1 -0
- package/lib/core/case-authoring-state.js +43 -0
- package/lib/core/case-authoring-state.js.map +1 -0
- package/lib/core/case.d.ts +29 -1
- package/lib/core/case.d.ts.map +1 -1
- package/lib/core/case.js +14 -6
- package/lib/core/case.js.map +1 -1
- package/lib/core/driver-metrics.d.ts +98 -0
- package/lib/core/driver-metrics.d.ts.map +1 -0
- package/lib/core/driver-metrics.js +36 -0
- package/lib/core/driver-metrics.js.map +1 -0
- package/lib/core/driver.d.ts +3 -0
- package/lib/core/driver.d.ts.map +1 -1
- package/lib/core/driver.js.map +1 -1
- package/lib/core/events.d.ts +36 -0
- package/lib/core/events.d.ts.map +1 -1
- package/lib/core/events.js +12 -2
- package/lib/core/events.js.map +1 -1
- package/lib/core/experiment.d.ts +19 -0
- package/lib/core/experiment.d.ts.map +1 -1
- package/lib/core/experiment.js +2 -0
- package/lib/core/experiment.js.map +1 -1
- package/lib/core/index.d.ts +4 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +4 -0
- package/lib/core/index.js.map +1 -1
- package/lib/core/lifecycle-fixtures.d.ts +3 -1
- package/lib/core/lifecycle-fixtures.d.ts.map +1 -1
- package/lib/core/lifecycle-fixtures.js +4 -2
- package/lib/core/lifecycle-fixtures.js.map +1 -1
- package/lib/core/lifecycle-hooks.d.ts +9 -0
- package/lib/core/lifecycle-hooks.d.ts.map +1 -1
- package/lib/core/lifecycle-hooks.js +8 -2
- package/lib/core/lifecycle-hooks.js.map +1 -1
- package/lib/core/run.d.ts +150 -0
- package/lib/core/run.d.ts.map +1 -1
- package/lib/core/run.js +9 -0
- package/lib/core/run.js.map +1 -1
- package/lib/core/wire.d.ts +12 -0
- package/lib/core/wire.d.ts.map +1 -1
- package/lib/core/wire.js +5 -1
- package/lib/core/wire.js.map +1 -1
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.js +73 -8
- package/lib/drivers/claude-code/parse-events.js.map +1 -1
- package/lib/drivers/codex/app-events.d.ts +9 -2
- package/lib/drivers/codex/app-events.d.ts.map +1 -1
- package/lib/drivers/codex/app-events.js +93 -12
- package/lib/drivers/codex/app-events.js.map +1 -1
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +10 -14
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/drivers/subprocess/index.d.ts.map +1 -1
- package/lib/drivers/subprocess/index.js +6 -0
- package/lib/drivers/subprocess/index.js.map +1 -1
- package/lib/index.d.ts +10 -6
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +10 -6
- package/lib/index.js.map +1 -1
- package/lib/orchestrator/annotation-store.d.ts.map +1 -1
- package/lib/orchestrator/annotation-store.js +6 -2
- package/lib/orchestrator/annotation-store.js.map +1 -1
- package/lib/orchestrator/case-extensions.d.ts +6 -0
- package/lib/orchestrator/case-extensions.d.ts.map +1 -0
- package/lib/orchestrator/case-extensions.js +28 -0
- package/lib/orchestrator/case-extensions.js.map +1 -0
- package/lib/orchestrator/compare.d.ts +13 -1
- package/lib/orchestrator/compare.d.ts.map +1 -1
- package/lib/orchestrator/compare.js +12 -0
- package/lib/orchestrator/compare.js.map +1 -1
- package/lib/orchestrator/config.d.ts.map +1 -1
- package/lib/orchestrator/config.js +0 -2
- package/lib/orchestrator/config.js.map +1 -1
- package/lib/orchestrator/doctor.d.ts.map +1 -1
- package/lib/orchestrator/doctor.js +5 -3
- package/lib/orchestrator/doctor.js.map +1 -1
- package/lib/orchestrator/experiment-contracts.d.ts +19 -0
- package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.js +17 -0
- package/lib/orchestrator/experiment-evaluation.js.map +1 -1
- package/lib/orchestrator/experiment-report.d.ts +19 -0
- package/lib/orchestrator/experiment-report.d.ts.map +1 -1
- package/lib/orchestrator/experiment-runs.d.ts +38 -0
- package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
- package/lib/orchestrator/grader-experiment.d.ts +12 -0
- package/lib/orchestrator/grader-experiment.d.ts.map +1 -1
- package/lib/orchestrator/grading.d.ts +2 -1
- package/lib/orchestrator/grading.d.ts.map +1 -1
- package/lib/orchestrator/grading.js +16 -4
- package/lib/orchestrator/grading.js.map +1 -1
- package/lib/orchestrator/index.d.ts +1 -1
- package/lib/orchestrator/index.d.ts.map +1 -1
- package/lib/orchestrator/index.js +1 -1
- package/lib/orchestrator/index.js.map +1 -1
- package/lib/orchestrator/list-runs.d.ts +7 -1
- package/lib/orchestrator/list-runs.d.ts.map +1 -1
- package/lib/orchestrator/list-runs.js +50 -6
- package/lib/orchestrator/list-runs.js.map +1 -1
- package/lib/orchestrator/loader.d.ts.map +1 -1
- package/lib/orchestrator/loader.js +2 -0
- package/lib/orchestrator/loader.js.map +1 -1
- package/lib/orchestrator/orchestrator.d.ts +3 -0
- package/lib/orchestrator/orchestrator.d.ts.map +1 -1
- package/lib/orchestrator/orchestrator.js +42 -14
- package/lib/orchestrator/orchestrator.js.map +1 -1
- package/lib/orchestrator/owned-lock.d.ts.map +1 -1
- package/lib/orchestrator/owned-lock.js +52 -38
- package/lib/orchestrator/owned-lock.js.map +1 -1
- package/lib/orchestrator/regrade.d.ts +6 -0
- package/lib/orchestrator/regrade.d.ts.map +1 -1
- package/lib/orchestrator/regrade.js +8 -4
- package/lib/orchestrator/regrade.js.map +1 -1
- package/lib/orchestrator/run-execution.d.ts +2 -1
- package/lib/orchestrator/run-execution.d.ts.map +1 -1
- package/lib/orchestrator/run-execution.js +20 -5
- package/lib/orchestrator/run-execution.js.map +1 -1
- package/lib/orchestrator/suite-compare.d.ts +34 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.d.ts +68 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.js +45 -7
- package/lib/orchestrator/suite-summary.js.map +1 -1
- package/lib/previewer/artifacts.d.ts +25 -0
- package/lib/previewer/artifacts.d.ts.map +1 -0
- package/lib/previewer/artifacts.js +180 -0
- package/lib/previewer/artifacts.js.map +1 -0
- package/lib/previewer/server.d.ts +105 -0
- package/lib/previewer/server.d.ts.map +1 -0
- package/lib/previewer/server.js +542 -0
- package/lib/previewer/server.js.map +1 -0
- package/package.json +2 -2
- package/skills/lux/SKILL.md +7 -3
- package/skills/lux/references/cli.md +25 -2
- package/skills/lux/references/eval-authoring.md +158 -4
- package/skills/lux-answerer/SKILL.md +1 -1
- package/src/answerers/persona/index.ts +20 -4
- package/src/answerers/terminal/index.ts +1 -1
- package/src/assertions/authoring.ts +81 -0
- package/src/assertions/core/helpers.ts +3 -6
- package/src/assertions/core/index.ts +2 -0
- package/src/assertions/core/resource-checks.ts +35 -0
- package/src/authoring.ts +1 -0
- package/src/cli/program.ts +14 -93
- package/src/cli/render/reporter.ts +18 -3
- package/src/cli/run-options.ts +1 -0
- package/src/cli/view.ts +168 -0
- package/src/core/action-events.ts +59 -0
- package/src/core/agent-activity.ts +236 -0
- package/src/core/agent-events.ts +17 -0
- package/src/core/artifact-evidence.ts +47 -4
- package/src/core/assertion.ts +41 -22
- package/src/core/case-authoring-state.ts +65 -0
- package/src/core/case.ts +57 -6
- package/src/core/driver-metrics.ts +51 -0
- package/src/core/driver.ts +3 -0
- package/src/core/events.ts +14 -2
- package/src/core/experiment.ts +2 -0
- package/src/core/index.ts +4 -0
- package/src/core/lifecycle-fixtures.ts +7 -2
- package/src/core/lifecycle-hooks.ts +16 -2
- package/src/core/run.ts +9 -0
- package/src/core/wire.ts +5 -1
- package/src/drivers/claude-code/parse-events.ts +98 -12
- package/src/drivers/codex/app-events.ts +116 -16
- package/src/drivers/codex/index.ts +16 -14
- package/src/drivers/subprocess/index.ts +6 -0
- package/src/index.ts +50 -6
- package/src/orchestrator/annotation-store.ts +5 -1
- package/src/orchestrator/case-extensions.ts +44 -0
- package/src/orchestrator/compare.ts +18 -0
- package/src/orchestrator/config.ts +0 -2
- package/src/orchestrator/doctor.ts +5 -3
- package/src/orchestrator/experiment-evaluation.ts +23 -0
- package/src/orchestrator/grading.ts +21 -2
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/list-runs.ts +66 -5
- package/src/orchestrator/loader.ts +2 -0
- package/src/orchestrator/orchestrator.ts +86 -40
- package/src/orchestrator/owned-lock.ts +54 -34
- package/src/orchestrator/regrade.ts +8 -7
- package/src/orchestrator/run-execution.ts +24 -3
- package/src/orchestrator/suite-summary.ts +101 -8
- package/src/previewer/artifacts.ts +221 -0
- package/src/previewer/server.ts +765 -0
- package/viewer/app.css +2075 -0
- package/viewer/app.js +606 -0
- package/viewer/artifact-view.js +48 -0
- package/viewer/charts.js +86 -0
- package/viewer/detail.js +1214 -0
- package/viewer/format.js +129 -0
- package/viewer/index.html +151 -0
- package/lib/answerers/scripted/index.d.ts +0 -8
- package/lib/answerers/scripted/index.d.ts.map +0 -1
- package/lib/answerers/scripted/index.js +0 -163
- package/lib/answerers/scripted/index.js.map +0 -1
- package/src/answerers/scripted/index.ts +0 -186
|
@@ -5,6 +5,7 @@ import { glob } from "tinyglobby";
|
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { RubricConfigSchema } from "../assertions/rubric/index.js";
|
|
7
7
|
import type { EvalCase, Registry } from "../core/index.js";
|
|
8
|
+
import { registryForEvalCase } from "./case-extensions.js";
|
|
8
9
|
import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
|
|
9
10
|
import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
|
|
10
11
|
import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
|
|
@@ -559,12 +560,13 @@ const inspectCase = async (
|
|
|
559
560
|
): Promise<CliTarget[]> => {
|
|
560
561
|
try {
|
|
561
562
|
const authored = await loadEvalCase(discovered.path);
|
|
563
|
+
const caseRegistry = registryForEvalCase(registry, authored);
|
|
562
564
|
const drivers = caseDriverReferences(authored.driver, config);
|
|
563
565
|
const answerer = authored.answerer ?? config.defaultAnswerer;
|
|
564
|
-
validateCaseDrivers(discovered.id, drivers,
|
|
565
|
-
validateCaseAnswerer(discovered.id, answerer,
|
|
566
|
+
validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
|
|
567
|
+
validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
|
|
566
568
|
return [
|
|
567
|
-
...(await validateCaseAssertions(discovered.id, authored, config,
|
|
569
|
+
...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
|
|
568
570
|
...executionTargetsForCase(discovered.id, drivers, answerer),
|
|
569
571
|
];
|
|
570
572
|
} catch (error) {
|
|
@@ -101,6 +101,23 @@ const evaluationMetrics = (
|
|
|
101
101
|
),
|
|
102
102
|
});
|
|
103
103
|
|
|
104
|
+
const evaluationMetricDefinitions = (summary: SuiteSummary) =>
|
|
105
|
+
Object.fromEntries(
|
|
106
|
+
Object.entries(summary.metrics).flatMap(([name, metric]) => {
|
|
107
|
+
if (!metric.unit && !metric.label && !metric.better) return [];
|
|
108
|
+
return [
|
|
109
|
+
[
|
|
110
|
+
name,
|
|
111
|
+
{
|
|
112
|
+
unit: metric.unit ?? "number",
|
|
113
|
+
better: metric.better ?? "neutral",
|
|
114
|
+
...(metric.label ? { label: metric.label } : {}),
|
|
115
|
+
},
|
|
116
|
+
],
|
|
117
|
+
];
|
|
118
|
+
}),
|
|
119
|
+
);
|
|
120
|
+
|
|
104
121
|
const summaryUsage = (summary: SuiteSummary) =>
|
|
105
122
|
summary.samples.reduce(
|
|
106
123
|
(usage, sample) =>
|
|
@@ -277,6 +294,7 @@ const agentEvaluationFromSuite = (input: {
|
|
|
277
294
|
summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
|
|
278
295
|
),
|
|
279
296
|
metrics: evaluationMetrics(summary, candidate),
|
|
297
|
+
metricDefinitions: evaluationMetricDefinitions(summary),
|
|
280
298
|
samples: evaluationSamples(summary),
|
|
281
299
|
metricCalls: summary.samples.length,
|
|
282
300
|
judgeCalls: results.reduce(
|
|
@@ -352,6 +370,10 @@ const aggregateAgentUnits = (
|
|
|
352
370
|
.map(([, values]) => average(values)),
|
|
353
371
|
]),
|
|
354
372
|
);
|
|
373
|
+
const metricDefinitions = Object.assign(
|
|
374
|
+
{},
|
|
375
|
+
...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
|
|
376
|
+
);
|
|
355
377
|
return {
|
|
356
378
|
candidateId: candidate.id,
|
|
357
379
|
profileId,
|
|
@@ -367,6 +389,7 @@ const aggregateAgentUnits = (
|
|
|
367
389
|
name === "subject.bytes" ? candidate.sourceBytes : average(values),
|
|
368
390
|
]),
|
|
369
391
|
),
|
|
392
|
+
metricDefinitions,
|
|
370
393
|
samples,
|
|
371
394
|
metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
|
|
372
395
|
judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),
|
|
@@ -76,6 +76,7 @@ export const graderConfigHash = (input: {
|
|
|
76
76
|
assertions: readonly AssertionEvaluatorIdentity[];
|
|
77
77
|
validatedAssertionsHash: string;
|
|
78
78
|
graderContext?: unknown;
|
|
79
|
+
skipJudge?: boolean;
|
|
79
80
|
}): string =>
|
|
80
81
|
valueHash({
|
|
81
82
|
luxVersion: input.luxVersion,
|
|
@@ -83,6 +84,7 @@ export const graderConfigHash = (input: {
|
|
|
83
84
|
assertions: input.assertions,
|
|
84
85
|
validatedAssertionsHash: input.validatedAssertionsHash,
|
|
85
86
|
graderContext: input.graderContext ?? null,
|
|
87
|
+
skipJudge: input.skipJudge ?? false,
|
|
86
88
|
});
|
|
87
89
|
|
|
88
90
|
export const validateAssertionPlan = async (
|
|
@@ -133,6 +135,7 @@ export const gradeRun = async (
|
|
|
133
135
|
complete: (attemptIds: readonly string[]) => void | Promise<void>;
|
|
134
136
|
},
|
|
135
137
|
validatedConfigs?: readonly unknown[],
|
|
138
|
+
skipJudge = false,
|
|
136
139
|
): Promise<GradingReport> => {
|
|
137
140
|
const guardedObserver = guardRunObserver(observer);
|
|
138
141
|
const assertions = run.case.assertions ?? [];
|
|
@@ -141,12 +144,27 @@ export const gradeRun = async (
|
|
|
141
144
|
}
|
|
142
145
|
|
|
143
146
|
const entries: GradingAssertionEntry[] = [];
|
|
147
|
+
const skippedAssertions: Array<{
|
|
148
|
+
id: string;
|
|
149
|
+
type: string;
|
|
150
|
+
name: string;
|
|
151
|
+
reason: "judge-disabled";
|
|
152
|
+
}> = [];
|
|
144
153
|
let weightedScoreSum = 0;
|
|
145
154
|
let weightSum = 0;
|
|
146
155
|
let allPassed = true;
|
|
147
156
|
let gradingIncomplete = false;
|
|
148
157
|
|
|
149
158
|
for (const [index, a] of assertions.entries()) {
|
|
159
|
+
if (skipJudge && a.type === "rubric") {
|
|
160
|
+
skippedAssertions.push({
|
|
161
|
+
id: a.id ?? `${a.type}:${index + 1}`,
|
|
162
|
+
type: a.type,
|
|
163
|
+
name: describeAssertion(a),
|
|
164
|
+
reason: "judge-disabled",
|
|
165
|
+
});
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
150
168
|
// Grading is the last cancellation point — an LLM judge can run for
|
|
151
169
|
// minutes per assertion. An aborted grade can't claim everything passed.
|
|
152
170
|
if (abortSignal?.aborted) {
|
|
@@ -247,12 +265,13 @@ export const gradeRun = async (
|
|
|
247
265
|
}
|
|
248
266
|
|
|
249
267
|
let caseScore: number | null = null;
|
|
250
|
-
if (!gradingIncomplete
|
|
268
|
+
if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
|
|
251
269
|
return {
|
|
252
270
|
schemaVersion: 1,
|
|
253
|
-
casePassed: gradingIncomplete ? null : allPassed,
|
|
271
|
+
casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
|
|
254
272
|
caseScore,
|
|
255
273
|
assertions: entries,
|
|
274
|
+
...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
|
|
256
275
|
...(gradingIncomplete
|
|
257
276
|
? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
|
|
258
277
|
: {}),
|
|
@@ -2,6 +2,11 @@ import { existsSync } from "node:fs";
|
|
|
2
2
|
import { constants, lstat, open, readdir } from "node:fs/promises";
|
|
3
3
|
import { basename, join, relative, sep } from "node:path";
|
|
4
4
|
import {
|
|
5
|
+
ActionProgressPayloadSchema,
|
|
6
|
+
AgentCompletedPayloadSchema,
|
|
7
|
+
AgentSpawnPayloadSchema,
|
|
8
|
+
type Artifact,
|
|
9
|
+
ArtifactSchema,
|
|
5
10
|
type GradingReport,
|
|
6
11
|
GradingReportSchema,
|
|
7
12
|
harnessUsage,
|
|
@@ -31,6 +36,11 @@ export type ListedRun = {
|
|
|
31
36
|
startedAt: string | null;
|
|
32
37
|
endedAt: string | null;
|
|
33
38
|
durationMs: number | null;
|
|
39
|
+
timeToFirstActionMs: number | null;
|
|
40
|
+
timeToFirstMutationMs: number | null;
|
|
41
|
+
timeToFirstDurableMutationMs: number | null;
|
|
42
|
+
driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
|
|
43
|
+
subagentCount: number;
|
|
34
44
|
casePassed: boolean | null;
|
|
35
45
|
caseScore: number | null;
|
|
36
46
|
assertionPassed: number;
|
|
@@ -63,6 +73,12 @@ const summarizeRun = async (
|
|
|
63
73
|
startedAt,
|
|
64
74
|
endedAt,
|
|
65
75
|
durationMs,
|
|
76
|
+
timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
|
|
77
|
+
timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
|
|
78
|
+
timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
|
|
79
|
+
driverMetrics: meta?.driverMetrics ?? {},
|
|
80
|
+
subagentCount:
|
|
81
|
+
meta?.agentActivity?.filter((activity) => activity.agent.role === "subagent").length ?? 0,
|
|
66
82
|
casePassed: grading?.casePassed ?? null,
|
|
67
83
|
caseScore: grading?.caseScore ?? null,
|
|
68
84
|
assertionPassed: passing,
|
|
@@ -127,7 +143,7 @@ export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
|
|
|
127
143
|
export const formatRunsTable = (runs: ListedRun[]): string => {
|
|
128
144
|
if (runs.length === 0) return "(no runs found)";
|
|
129
145
|
const rows: string[][] = [
|
|
130
|
-
["status", "case", "driver", "ts", "exit", "score", "duration"],
|
|
146
|
+
["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
|
|
131
147
|
...runs.map((r) => {
|
|
132
148
|
let status = "ungraded";
|
|
133
149
|
if (r.casePassed === true) status = "PASS";
|
|
@@ -137,7 +153,8 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
|
|
|
137
153
|
? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
|
|
138
154
|
: "—";
|
|
139
155
|
const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
|
|
140
|
-
|
|
156
|
+
const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
|
|
157
|
+
return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
|
|
141
158
|
}),
|
|
142
159
|
];
|
|
143
160
|
const header = rows[0] ?? [];
|
|
@@ -156,6 +173,9 @@ export type RunDetail = {
|
|
|
156
173
|
artifacts: string[];
|
|
157
174
|
};
|
|
158
175
|
|
|
176
|
+
export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
|
|
177
|
+
(await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
|
|
178
|
+
|
|
159
179
|
const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
|
|
160
180
|
if (!existsSync(path)) return [];
|
|
161
181
|
if ((await lstat(path)).isSymbolicLink()) {
|
|
@@ -291,6 +311,27 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
|
|
|
291
311
|
`phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
|
|
292
312
|
);
|
|
293
313
|
}
|
|
314
|
+
const actions = detail.metadata.actionTiming;
|
|
315
|
+
if (actions) {
|
|
316
|
+
const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
|
|
317
|
+
lines.push(
|
|
318
|
+
`first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
const activities = detail.metadata.agentActivity ?? [];
|
|
322
|
+
const subagents = activities.filter((activity) => activity.agent.role === "subagent");
|
|
323
|
+
if (subagents.length > 0) {
|
|
324
|
+
lines.push(`subagents: ${subagents.length}`);
|
|
325
|
+
for (const activity of subagents) {
|
|
326
|
+
const label = activity.agent.name
|
|
327
|
+
? `${activity.agent.name} (${activity.agent.id})`
|
|
328
|
+
: activity.agent.id;
|
|
329
|
+
const agentActions = activity.actions;
|
|
330
|
+
lines.push(
|
|
331
|
+
` ${label}: ${activity.status}, ${(activity.durationMs / 1000).toFixed(1)}s, events=${activity.eventCount}, tokens=${totalTokens(activity.usage)}, tools=${activity.toolCalls.total} (${activity.toolCalls.succeeded} ok/${activity.toolCalls.failed} failed/${activity.toolCalls.unresolved} unresolved), actions=${agentActions.total}, mutations=${agentActions.mutations}`,
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
294
335
|
}
|
|
295
336
|
};
|
|
296
337
|
|
|
@@ -361,17 +402,37 @@ const formatEventLine = (event: RunEvent): string => {
|
|
|
361
402
|
}`;
|
|
362
403
|
}
|
|
363
404
|
return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
|
|
364
|
-
case "progress":
|
|
405
|
+
case "progress": {
|
|
406
|
+
const agent = event.agent?.role === "subagent" ? ` [subagent ${event.agent.id}]` : "";
|
|
407
|
+
if (event.kind === "agent-spawn") {
|
|
408
|
+
const payload = AgentSpawnPayloadSchema.safeParse(event.payload).data;
|
|
409
|
+
return payload
|
|
410
|
+
? `agent-spawn: ${payload.agent.name ?? payload.agent.id} (${payload.agent.id})`
|
|
411
|
+
: "agent-spawn: malformed";
|
|
412
|
+
}
|
|
413
|
+
if (event.kind === "agent-completed") {
|
|
414
|
+
const payload = AgentCompletedPayloadSchema.safeParse(event.payload).data;
|
|
415
|
+
return payload
|
|
416
|
+
? `agent-completed: ${payload.agentId} (${payload.status})`
|
|
417
|
+
: "agent-completed: malformed";
|
|
418
|
+
}
|
|
419
|
+
if (event.kind === "action") {
|
|
420
|
+
const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
|
|
421
|
+
return payload
|
|
422
|
+
? `action${agent}: ${payload.name} (${payload.category})`
|
|
423
|
+
: "action: malformed";
|
|
424
|
+
}
|
|
365
425
|
if (event.kind === "tool-call") {
|
|
366
426
|
const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
|
|
367
|
-
return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
|
|
427
|
+
return payload ? `tool-call${agent}: ${payload.name}` : "tool-call: malformed";
|
|
368
428
|
}
|
|
369
429
|
if (event.kind === "tool-result") {
|
|
370
430
|
const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
|
|
371
431
|
if (!payload) return "tool-result: malformed";
|
|
372
|
-
return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
|
|
432
|
+
return `tool-result${agent}: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
|
|
373
433
|
}
|
|
374
434
|
return `progress(${event.kind})`;
|
|
435
|
+
}
|
|
375
436
|
case "lifecycle":
|
|
376
437
|
return `lifecycle: ${event.stage}`;
|
|
377
438
|
case "ready":
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
2
|
import { pathToFileURL } from "node:url";
|
|
3
|
+
import { transferEvalCaseExtensions } from "../core/case-authoring-state.js";
|
|
3
4
|
import { type EvalCase, EvalCaseSchema, formatZodIssues } from "../core/index.js";
|
|
4
5
|
|
|
5
6
|
export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
|
|
@@ -15,5 +16,6 @@ export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
|
|
|
15
16
|
if (!result.success) {
|
|
16
17
|
throw new Error(`Invalid eval case at ${absolute}: ${formatZodIssues(result.error)}`);
|
|
17
18
|
}
|
|
19
|
+
transferEvalCaseExtensions(mod.default as EvalCase, result.data);
|
|
18
20
|
return result.data;
|
|
19
21
|
};
|
|
@@ -18,6 +18,12 @@ import {
|
|
|
18
18
|
type SuiteLifecycleHookContext,
|
|
19
19
|
valueHash,
|
|
20
20
|
} from "../core/index.js";
|
|
21
|
+
import {
|
|
22
|
+
fixturesForEvalCase,
|
|
23
|
+
mergeLifecycleFixtures,
|
|
24
|
+
registryForEvalCase,
|
|
25
|
+
valuesForFixtures,
|
|
26
|
+
} from "./case-extensions.js";
|
|
21
27
|
import { evaluatorImplementationHash } from "./evaluator-identity.js";
|
|
22
28
|
import {
|
|
23
29
|
assertionEvaluatorIdentities,
|
|
@@ -122,6 +128,8 @@ export type RunRequest = {
|
|
|
122
128
|
/** Root that relative `fixtures` dir paths resolve against. */
|
|
123
129
|
fixturesRoot?: string | undefined;
|
|
124
130
|
answererOverride?: string | undefined;
|
|
131
|
+
/** Skip built-in rubric judge assertions without changing subject execution. */
|
|
132
|
+
skipJudge?: boolean | undefined;
|
|
125
133
|
startedAt?: Date | undefined;
|
|
126
134
|
/** The `runSuite` invocation this case attempt belongs to. */
|
|
127
135
|
suiteId?: string | undefined;
|
|
@@ -476,11 +484,6 @@ export class Orchestrator {
|
|
|
476
484
|
}
|
|
477
485
|
|
|
478
486
|
async run(request: RunRequest): Promise<RunResult> {
|
|
479
|
-
if (this.lifecycleFixtures.length > 0 && !request.suiteId) {
|
|
480
|
-
throw new Error(
|
|
481
|
-
"Orchestrator.run() cannot execute configured lifecycle fixtures; use runSuite() for a single case",
|
|
482
|
-
);
|
|
483
|
-
}
|
|
484
487
|
if (!request.case && !request.casePath) {
|
|
485
488
|
throw new Error("RunRequest requires either 'case' or 'casePath'");
|
|
486
489
|
}
|
|
@@ -489,11 +492,24 @@ export class Orchestrator {
|
|
|
489
492
|
}
|
|
490
493
|
const casePath = request.casePath ? resolve(request.casePath) : INLINE_CASE_PATH;
|
|
491
494
|
const authoredCase = request.case ?? (await loadEvalCase(casePath));
|
|
495
|
+
const lifecycleFixtures = mergeLifecycleFixtures(
|
|
496
|
+
this.lifecycleFixtures,
|
|
497
|
+
fixturesForEvalCase(authoredCase),
|
|
498
|
+
);
|
|
499
|
+
if (
|
|
500
|
+
lifecycleFixtures.some(({ setupSuite, setupCase }) => setupSuite || setupCase) &&
|
|
501
|
+
!request.suiteId
|
|
502
|
+
) {
|
|
503
|
+
throw new Error(
|
|
504
|
+
"Orchestrator.run() cannot execute lifecycle fixtures; use runSuite() for a single case",
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
const registry = registryForEvalCase(this.registry, authoredCase);
|
|
492
508
|
const evalCase = bindSubjectRoot(authoredCase, request.subjectRoot);
|
|
493
509
|
const caseId = resolveCaseId(casePath, evalCase, request.caseId);
|
|
494
510
|
const validatedAssertionConfigs = await validateAssertionPlan(
|
|
495
511
|
evalCase.assertions ?? [],
|
|
496
|
-
|
|
512
|
+
registry,
|
|
497
513
|
);
|
|
498
514
|
|
|
499
515
|
const driverRef = resolveDriverReference(evalCase, this.defaultDriver);
|
|
@@ -506,11 +522,11 @@ export class Orchestrator {
|
|
|
506
522
|
);
|
|
507
523
|
const answererName = answererNameOf(answererRef);
|
|
508
524
|
|
|
509
|
-
const driverPlugin =
|
|
525
|
+
const driverPlugin = registry.getDriver(driverName);
|
|
510
526
|
if (!driverPlugin) {
|
|
511
527
|
throw new Error(`no driver registered for '${driverName}'`);
|
|
512
528
|
}
|
|
513
|
-
const answererPlugin =
|
|
529
|
+
const answererPlugin = registry.getAnswerer(answererName);
|
|
514
530
|
if (!answererPlugin) {
|
|
515
531
|
throw new Error(`no answerer registered for '${answererName}'`);
|
|
516
532
|
}
|
|
@@ -544,7 +560,7 @@ export class Orchestrator {
|
|
|
544
560
|
try {
|
|
545
561
|
const assertionImplementations = assertionEvaluatorIdentities(
|
|
546
562
|
authoredCase.assertions ?? [],
|
|
547
|
-
|
|
563
|
+
registry,
|
|
548
564
|
);
|
|
549
565
|
const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
|
|
550
566
|
const implementationHash = await this.evaluatorImplementationIdentity();
|
|
@@ -554,34 +570,40 @@ export class Orchestrator {
|
|
|
554
570
|
assertions: assertionImplementations,
|
|
555
571
|
validatedAssertionsHash,
|
|
556
572
|
graderContext: this.harness,
|
|
573
|
+
skipJudge: request.skipJudge ?? false,
|
|
557
574
|
});
|
|
558
|
-
return await this.runWithPlugins(
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
graderConfigHash: graderHash,
|
|
572
|
-
evaluatorConfigHash: valueHash({
|
|
573
|
-
luxVersion: LUX_VERSION,
|
|
574
|
-
implementationHash,
|
|
575
|
-
harness: this.harness,
|
|
576
|
-
driver: { name: driverPlugin.name, create: driverPlugin.create },
|
|
577
|
-
answerer: { name: answererPlugin.name, create: answererPlugin.create },
|
|
578
|
-
assertions: assertionImplementations,
|
|
579
|
-
lifecycleFixtures: this.lifecycleFixtures,
|
|
575
|
+
return await this.runWithPlugins(
|
|
576
|
+
{
|
|
577
|
+
request,
|
|
578
|
+
casePath,
|
|
579
|
+
authoredCase,
|
|
580
|
+
evalCase,
|
|
581
|
+
validatedAssertionConfigs,
|
|
582
|
+
caseId,
|
|
583
|
+
driverName,
|
|
584
|
+
answererName,
|
|
585
|
+
answererConfig: answererCfg,
|
|
586
|
+
driverConfigHash: valueHash(driverCfg),
|
|
587
|
+
answererConfigHash: valueHash(answererCfg),
|
|
580
588
|
validatedAssertionsHash,
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
589
|
+
graderConfigHash: graderHash,
|
|
590
|
+
evaluatorConfigHash: valueHash({
|
|
591
|
+
luxVersion: LUX_VERSION,
|
|
592
|
+
implementationHash,
|
|
593
|
+
harness: this.harness,
|
|
594
|
+
driver: { name: driverPlugin.name, create: driverPlugin.create },
|
|
595
|
+
answerer: { name: answererPlugin.name, create: answererPlugin.create },
|
|
596
|
+
assertions: assertionImplementations,
|
|
597
|
+
lifecycleFixtures,
|
|
598
|
+
validatedAssertionsHash,
|
|
599
|
+
skipJudge: request.skipJudge ?? false,
|
|
600
|
+
}),
|
|
601
|
+
fixtureValues: valuesForFixtures(lifecycleFixtures),
|
|
602
|
+
driver,
|
|
603
|
+
answerer,
|
|
604
|
+
},
|
|
605
|
+
registry,
|
|
606
|
+
);
|
|
585
607
|
} finally {
|
|
586
608
|
await closeQuietly(() => driver.close(), `driver '${driverName}'`);
|
|
587
609
|
await closeQuietly(() => answerer.close(), `answerer '${answererName}'`);
|
|
@@ -599,6 +621,7 @@ export class Orchestrator {
|
|
|
599
621
|
runsRoot: string;
|
|
600
622
|
fixturesRoot?: string | undefined;
|
|
601
623
|
answererOverride?: string | undefined;
|
|
624
|
+
skipJudge?: boolean | undefined;
|
|
602
625
|
observer?: SuiteObserver | undefined;
|
|
603
626
|
/** Aborts the cases in flight and stops the pool pulling new ones. */
|
|
604
627
|
abortSignal?: AbortSignal | undefined;
|
|
@@ -628,6 +651,20 @@ export class Orchestrator {
|
|
|
628
651
|
const observer = opts.observer;
|
|
629
652
|
const requestedConcurrency = validateSuiteConcurrency(opts.concurrency);
|
|
630
653
|
const concurrency = Math.min(requestedConcurrency, Math.max(items.length, 1));
|
|
654
|
+
const authoredCases = await Promise.allSettled(items.map((item) => loadEvalCase(item.path)));
|
|
655
|
+
const localFixtures = authoredCases.map((result) =>
|
|
656
|
+
result.status === "fulfilled" ? fixturesForEvalCase(result.value) : [],
|
|
657
|
+
);
|
|
658
|
+
const lifecycleFixtures = mergeLifecycleFixtures(this.lifecycleFixtures, ...localFixtures);
|
|
659
|
+
const globalFixtureNames = new Set(this.lifecycleFixtures.map((fixture) => fixture.name));
|
|
660
|
+
const itemIndexesByFixture = new Map<string, number[]>();
|
|
661
|
+
for (const [index, fixtures] of localFixtures.entries()) {
|
|
662
|
+
for (const fixture of fixtures) {
|
|
663
|
+
const indexes = itemIndexesByFixture.get(fixture.name) ?? [];
|
|
664
|
+
indexes.push(index);
|
|
665
|
+
itemIndexesByFixture.set(fixture.name, indexes);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
631
668
|
const suite = await openSuiteRun({
|
|
632
669
|
runsRoot: opts.runsRoot,
|
|
633
670
|
concurrency,
|
|
@@ -639,7 +676,7 @@ export class Orchestrator {
|
|
|
639
676
|
...(opts.variantId ? { variantId: opts.variantId } : {}),
|
|
640
677
|
...(opts.candidateId ? { candidateId: opts.candidateId } : {}),
|
|
641
678
|
...(opts.subjectHash ? { subjectHash: opts.subjectHash } : {}),
|
|
642
|
-
fixtures:
|
|
679
|
+
fixtures: lifecycleFixtures.map(({ name, pluginVersion }) => ({
|
|
643
680
|
name,
|
|
644
681
|
pluginVersion,
|
|
645
682
|
})),
|
|
@@ -677,9 +714,16 @@ export class Orchestrator {
|
|
|
677
714
|
if (!opts.abortSignal?.aborted) {
|
|
678
715
|
try {
|
|
679
716
|
cleanupSuiteHooks = await setupLifecycleHooks(
|
|
680
|
-
|
|
717
|
+
lifecycleFixtures,
|
|
681
718
|
"suite",
|
|
682
|
-
(hook) =>
|
|
719
|
+
(hook) => {
|
|
720
|
+
if (globalFixtureNames.has(hook.name)) return hook.setupSuite?.(hookContext);
|
|
721
|
+
const indexes = itemIndexesByFixture.get(hook.name) ?? [];
|
|
722
|
+
return hook.setupSuite?.({
|
|
723
|
+
...hookContext,
|
|
724
|
+
items: indexes.flatMap((index) => hookContext.items[index] ?? []),
|
|
725
|
+
});
|
|
726
|
+
},
|
|
683
727
|
opts.abortSignal,
|
|
684
728
|
);
|
|
685
729
|
} catch (error) {
|
|
@@ -718,7 +762,7 @@ export class Orchestrator {
|
|
|
718
762
|
let cleanupCaseHooks: () => void | Promise<void>;
|
|
719
763
|
try {
|
|
720
764
|
cleanupCaseHooks = await setupLifecycleHooks(
|
|
721
|
-
this.lifecycleFixtures,
|
|
765
|
+
mergeLifecycleFixtures(this.lifecycleFixtures, localFixtures[index] ?? []),
|
|
722
766
|
"case",
|
|
723
767
|
(hook) =>
|
|
724
768
|
hook.setupCase?.({
|
|
@@ -749,6 +793,7 @@ export class Orchestrator {
|
|
|
749
793
|
runsRoot: opts.runsRoot,
|
|
750
794
|
fixturesRoot: opts.fixturesRoot,
|
|
751
795
|
answererOverride: opts.answererOverride,
|
|
796
|
+
skipJudge: opts.skipJudge,
|
|
752
797
|
observer: handle ?? observer,
|
|
753
798
|
abortSignal: opts.abortSignal,
|
|
754
799
|
suiteId: suite.manifest.suiteId,
|
|
@@ -906,11 +951,12 @@ export class Orchestrator {
|
|
|
906
951
|
|
|
907
952
|
private runWithPlugins(
|
|
908
953
|
args: Omit<RunWithPluginsArgs, "luxVersion" | "registry">,
|
|
954
|
+
registry: Registry = this.registry,
|
|
909
955
|
): Promise<RunResult> {
|
|
910
956
|
return executeRunWithPlugins({
|
|
911
957
|
...args,
|
|
912
958
|
luxVersion: LUX_VERSION,
|
|
913
|
-
registry
|
|
959
|
+
registry,
|
|
914
960
|
});
|
|
915
961
|
}
|
|
916
962
|
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { link, lstat, open,
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { link, lstat, open, readFile, rm, unlink } from "node:fs/promises";
|
|
3
3
|
import { hostname } from "node:os";
|
|
4
|
-
import { basename, dirname, join } from "node:path";
|
|
5
|
-
import { setTimeout as delay } from "node:timers/promises";
|
|
6
4
|
|
|
7
5
|
export type LockRecoveryClaim = {
|
|
8
6
|
path: string;
|
|
@@ -60,36 +58,54 @@ const linkedClaim = (lockPath: string, claimPath: string): LockRecoveryClaim =>
|
|
|
60
58
|
release: () => rm(claimPath, { force: true }),
|
|
61
59
|
});
|
|
62
60
|
|
|
63
|
-
|
|
61
|
+
type RecoveryMutexOwner = {
|
|
62
|
+
pid: number;
|
|
63
|
+
hostname: string;
|
|
64
|
+
lockToken: string;
|
|
65
|
+
};
|
|
64
66
|
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
if (claimHost !== recoveryHost || !rawPid || !id || extra !== undefined) return false;
|
|
67
|
+
const recoveryOwnerIsDead = (owner: RecoveryMutexOwner): boolean => {
|
|
68
|
+
if (owner.hostname !== hostname()) return false;
|
|
68
69
|
try {
|
|
69
|
-
process.kill(
|
|
70
|
+
process.kill(owner.pid, 0);
|
|
70
71
|
return false;
|
|
71
72
|
} catch (error) {
|
|
72
73
|
return (error as NodeJS.ErrnoException).code === "ESRCH";
|
|
73
74
|
}
|
|
74
75
|
};
|
|
75
76
|
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
77
|
+
const acquireRecoveryMutex = async (
|
|
78
|
+
lockPath: string,
|
|
79
|
+
): Promise<{ path: string; lockToken: string } | null> => {
|
|
80
|
+
const path = `${lockPath}.recovery`;
|
|
81
|
+
while (true) {
|
|
82
|
+
try {
|
|
83
|
+
const owner = await publishOwnedLock(path, {
|
|
84
|
+
pid: process.pid,
|
|
85
|
+
hostname: hostname(),
|
|
86
|
+
});
|
|
87
|
+
return { path, lockToken: owner.lockToken };
|
|
88
|
+
} catch (error) {
|
|
89
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
89
90
|
}
|
|
90
|
-
|
|
91
|
+
let owner: RecoveryMutexOwner;
|
|
92
|
+
try {
|
|
93
|
+
const value = JSON.parse(await readFile(path, "utf8")) as Partial<RecoveryMutexOwner>;
|
|
94
|
+
if (
|
|
95
|
+
!Number.isSafeInteger(value.pid) ||
|
|
96
|
+
typeof value.hostname !== "string" ||
|
|
97
|
+
typeof value.lockToken !== "string"
|
|
98
|
+
) {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
owner = value as RecoveryMutexOwner;
|
|
102
|
+
} catch (error) {
|
|
103
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
if (!recoveryOwnerIsDead(owner)) return null;
|
|
107
|
+
await releaseOwnedLock(path, owner.lockToken);
|
|
91
108
|
}
|
|
92
|
-
return false;
|
|
93
109
|
};
|
|
94
110
|
|
|
95
111
|
/**
|
|
@@ -97,24 +113,28 @@ const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): P
|
|
|
97
113
|
* Foreign claims stay conservative because foreign lock owners do too.
|
|
98
114
|
*/
|
|
99
115
|
export const claimLockForRecovery = async (lockPath: string): Promise<LockRecoveryClaim | null> => {
|
|
100
|
-
const
|
|
116
|
+
const mutex = await acquireRecoveryMutex(lockPath);
|
|
117
|
+
if (!mutex) return null;
|
|
118
|
+
const claimPath = `${lockPath}.recovering-${mutex.lockToken}`;
|
|
101
119
|
try {
|
|
102
120
|
await link(lockPath, claimPath);
|
|
103
121
|
} catch (error) {
|
|
104
122
|
const code = (error as NodeJS.ErrnoException).code;
|
|
123
|
+
await releaseOwnedLock(mutex.path, mutex.lockToken);
|
|
105
124
|
if (code === "ENOENT") return null;
|
|
106
125
|
throw error;
|
|
107
126
|
}
|
|
108
127
|
const claim = linkedClaim(lockPath, claimPath);
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
128
|
+
return {
|
|
129
|
+
...claim,
|
|
130
|
+
release: async () => {
|
|
131
|
+
try {
|
|
132
|
+
await claim.release();
|
|
133
|
+
} finally {
|
|
134
|
+
await releaseOwnedLock(mutex.path, mutex.lockToken);
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
};
|
|
118
138
|
};
|
|
119
139
|
|
|
120
140
|
/** Remove only the canonical lock that still carries this acquisition token. */
|