harness-everything 0.30.9 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +34 -0
- package/README.md +8 -9
- package/VERIFICATION.md +1 -1
- package/benchmarks/results/live-host/claude-2026-10-08-issue-297/README.md +27 -0
- package/benchmarks/results/live-host/codex-2026-10-13/README.md +55 -0
- package/benchmarks/results/live-host/codex-2026-10-13/observation-records.json +88 -0
- package/benchmarks/results/pr-303-review-2026-10-13/README.md +81 -0
- package/benchmarks/results/pr-303-review-2026-10-13/ci/run.json +1330 -0
- package/benchmarks/results/pr-303-review-2026-10-13/diagnostics/mechanism-35-clean-env.log +1 -0
- package/benchmarks/results/pr-303-review-2026-10-13/diagnostics/mechanism-35-inherited-flag.log +34 -0
- package/benchmarks/results/pr-303-review-2026-10-13/diagnostics/verify-gate-captured.json +46 -0
- package/benchmarks/results/pr-303-review-2026-10-13/diagnostics/verify-gate-diagnosis.md +13 -0
- package/benchmarks/results/pr-303-review-2026-10-13/diagnostics/verify-gate.json +46 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/summary.json +57 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-collision.log +8 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-consistency.log +624 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-docs-capabilities.log +131 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-platform-compatibility.log +9 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-plugin-openai.log +6 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-plugin-submission.log +5 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-references.log +6 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-release.log +73 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-repo-contract.log +5 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-routing-invariants.log +785 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/contracts/test-routing-skills.log +44 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/final-fast/summary.json +12 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/final-fast/test-consistency.log +624 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/final-fast/test-release.log +73 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/heavy/summary.json +79 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/heavy/test-mechanism.log +1737 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/heavy/test-mutations.log +16 -0
- package/benchmarks/results/pr-303-review-2026-10-13/gates/heavy/test.log +3297 -0
- package/benchmarks/results/pr-303-review-2026-10-13/manifest.json +319 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/binding-resolution-green.log +36 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/determinism.json +174 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/quality-evidence.json +314 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/quality-report.json +638 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/red.log +240 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/untyped-fable-probe.json +22 -0
- package/benchmarks/results/pr-303-review-2026-10-13/regression/workflow-obligations-green.log +254 -0
- package/benchmarks/results/pr-303-review-2026-10-13/verification/mechanism-35-clean-env-fixed.log +1 -0
- package/benchmarks/results/pr-303-review-2026-10-13/verification/mechanism-35-inherited-flag-fixed.log +1 -0
- package/benchmarks/results/pr-303-review-2026-10-13/verification/nested-test.meta.json +8 -0
- package/benchmarks/results/pr-303-review-2026-10-13/verification/nested-test.stderr.log +90 -0
- package/benchmarks/results/pr-303-review-2026-10-13/verification/nested-test.stdout.log +3186 -0
- package/benchmarks/results/pr-303-review-2026-10-13/verification/verify-gate-final.json +46 -0
- package/bin/cli.js +1 -1
- package/ci/doc-capability-consistency.test.js +11 -1
- package/ci/invariant-routing.test.js +7 -4
- package/ci/mechanism-2ag-router-worktree-contract.test.js +2 -2
- package/ci/mechanism-2ai-workspace-fingerprint-batch.test.js +2 -1
- package/ci/mechanism-2al-codex-hooks-compat.test.js +19 -0
- package/ci/mechanism-2am-observations-session-import.test.js +221 -0
- package/ci/mechanism-2an-workflow-obligations.test.js +352 -3
- package/ci/mechanism-2ao-plugin-sync.test.js +129 -4
- package/ci/mechanism-2aq-fable-stage-bindings.test.js +168 -0
- package/ci/mechanism-2ar-binding-resolution.test.js +211 -0
- package/ci/mechanism-2w-platform-compatibility.test.js +12 -0
- package/ci/mechanism-34-observations.test.js +64 -0
- package/ci/mechanism-35-verification-contract.test.js +1 -1
- package/ci/mutation-check.js +2 -0
- package/ci/mutation-loader.js +4 -0
- package/ci/openai-plugin-package.test.js +7 -5
- package/ci/prompt-contract-regressions.test.js +7 -6
- package/ci/router-workflow-plan.test.js +109 -14
- package/ci/skill-routing-check.js +44 -15
- package/ci/workflow-plan-consumers.test.js +41 -1
- package/docs/architecture.md +10 -10
- package/docs/enforcement-locks.md +1 -1
- package/docs/mechanism-first-skill-mesh.md +12 -11
- package/docs/openai-plugin.md +1 -1
- package/docs/philosophy.md +7 -6
- package/docs/platform-capabilities.md +14 -12
- package/docs/platform-compatibility.json +5 -5
- package/docs/routing.md +16 -17
- package/docs/system-one-observations.md +36 -9
- package/docs/system-one-routing.md +10 -7
- package/docs/system-one-suggestion-gates.md +5 -5
- package/docs/workflow-runtime.md +6 -2
- package/docs/workflows/harness-everything.md +14 -14
- package/evals/harness-everything/tasks/positive-tier1.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier2.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier3.yaml +1 -1
- package/fable-mode/CONTRACT-FORMAT.md +4 -1
- package/fable-mode/SKILL.md +6 -8
- package/fable-mode/agents/fable-orchestrator.md +10 -3
- package/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/fable-mode/fable-haiku/SKILL.md +1 -1
- package/fable-mode/fable-opus/SKILL.md +1 -1
- package/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/fable-mode/scripts/workflow-plan-consumer.js +101 -2
- package/harness-everything/SKILL.md +8 -7
- package/harness-everything/references/router-workflow-plan.md +11 -6
- package/harness-everything/references/triage-and-tiers.md +21 -23
- package/harness-everything/schemas/router-workflow-plan.schema.json +2 -0
- package/harness-everything/scripts/kernel-router-core.js +52 -55
- package/harness-everything/scripts/kernel-router.js +48 -5
- package/harness-everything/scripts/router-contract-core.js +7 -7
- package/harness-everything/scripts/router-contract.js +1 -3
- package/harness-everything/scripts/routing-keywords.json +417 -121
- package/harness-everything/scripts/tier-router.js +64 -113
- package/hooks/scripts/contract-test.js +29 -3
- package/hooks/scripts/lib/binding-resolver.js +76 -0
- package/hooks/scripts/lib/fable-contracts.js +1 -1
- package/hooks/scripts/lib/observations.js +88 -12
- package/hooks/scripts/lib/workflow-obligations.js +292 -7
- package/hooks/scripts/lib/workflow-runtime.js +8 -1
- package/hooks/scripts/workflow-disposition.js +116 -2
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/contract-test.js +29 -3
- package/plugins/harness-everything/hooks/scripts/lib/binding-resolver.js +76 -0
- package/plugins/harness-everything/hooks/scripts/lib/fable-contracts.js +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/observations.js +88 -12
- package/plugins/harness-everything/hooks/scripts/lib/workflow-obligations.js +292 -7
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +8 -1
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +116 -2
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/scripts/lib/advisory-text.js +2 -2
- package/plugins/harness-everything/scripts/lib/generated-skills.js +78 -0
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +4 -1
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +6 -8
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +10 -3
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/scripts/workflow-plan-consumer.js +101 -2
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +8 -7
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +11 -6
- package/plugins/harness-everything/skills/harness-everything/references/triage-and-tiers.md +21 -23
- package/plugins/harness-everything/skills/harness-everything/schemas/router-workflow-plan.schema.json +2 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router-core.js +52 -55
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +48 -5
- package/plugins/harness-everything/skills/harness-everything/scripts/router-contract-core.js +7 -7
- package/plugins/harness-everything/skills/harness-everything/scripts/router-contract.js +1 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/routing-keywords.json +417 -121
- package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +64 -113
- package/sbom.cdx.json +7 -7
- package/scripts/codex-hooks-compat.js +39 -14
- package/scripts/lib/advisory-text.js +2 -2
- package/scripts/lib/generated-skills.js +78 -0
- package/scripts/plugin-sync.js +100 -29
- package/scripts/system-one-observations-export.js +29 -2
- package/scripts/system-one-observations-import-codex-sessions.js +307 -0
- package/scripts/system-one-observations-review.js +11 -1
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"status": "UNCHECKED_DISCOVERED",
|
|
3
|
+
"reason": "1 verification candidate(s) detected but not executed",
|
|
4
|
+
"repoRoot": "<workspace>",
|
|
5
|
+
"roots": [
|
|
6
|
+
{
|
|
7
|
+
"path": "<workspace>",
|
|
8
|
+
"status": "UNCHECKED_DISCOVERED",
|
|
9
|
+
"reason": "1 verification candidate(s) detected but not executed",
|
|
10
|
+
"checks": [
|
|
11
|
+
{
|
|
12
|
+
"id": "package-test",
|
|
13
|
+
"run": "npm run test",
|
|
14
|
+
"cwd": "<workspace>",
|
|
15
|
+
"source": "package-auto",
|
|
16
|
+
"exitCode": 0,
|
|
17
|
+
"durationMs": 245096
|
|
18
|
+
}
|
|
19
|
+
],
|
|
20
|
+
"candidates": [
|
|
21
|
+
{
|
|
22
|
+
"source": "git-hook",
|
|
23
|
+
"run": "git commit (pre-commit hook configured)",
|
|
24
|
+
"cwd": "<workspace>"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
],
|
|
29
|
+
"checks": [
|
|
30
|
+
{
|
|
31
|
+
"id": "package-test",
|
|
32
|
+
"run": "npm run test",
|
|
33
|
+
"cwd": "<workspace>",
|
|
34
|
+
"source": "package-auto",
|
|
35
|
+
"exitCode": 0,
|
|
36
|
+
"durationMs": 245096
|
|
37
|
+
}
|
|
38
|
+
],
|
|
39
|
+
"candidates": [
|
|
40
|
+
{
|
|
41
|
+
"source": "git-hook",
|
|
42
|
+
"run": "git commit (pre-commit hook configured)",
|
|
43
|
+
"cwd": "<workspace>"
|
|
44
|
+
}
|
|
45
|
+
]
|
|
46
|
+
}
|
package/bin/cli.js
CHANGED
|
@@ -103,7 +103,7 @@ Commands:
|
|
|
103
103
|
error instead of silently doing nothing.
|
|
104
104
|
self-regression Run syntax and routing checks before committing changes (alias: test)
|
|
105
105
|
next "<prompt>" Print the Harness Kernel routing recommendation for a prompt:
|
|
106
|
-
tier/rationale + mandatory invariants +
|
|
106
|
+
tier/rationale + mandatory invariants + normalized knowledge signals.
|
|
107
107
|
Hookless platforms can call this explicitly at the start of software
|
|
108
108
|
work; hosts with UserPromptSubmit hooks receive it automatically.
|
|
109
109
|
verify-install Compare installed Harness manifests and skill trees with
|
|
@@ -190,6 +190,7 @@ for (const rel of OPENCODE_SURFACES) {
|
|
|
190
190
|
|
|
191
191
|
const matrix = JSON.parse(read('docs/platform-compatibility.json'));
|
|
192
192
|
const opencode = matrix.platforms.find(platform => platform.id === 'opencode');
|
|
193
|
+
const codex = matrix.platforms.find(platform => platform.id === 'codex');
|
|
193
194
|
for (const dimension of ['hooksLifecycle', 'liveHostVerification']) {
|
|
194
195
|
if (opencode?.capabilities[dimension]?.status !== 'Partial') {
|
|
195
196
|
fail(`OpenCode ${dimension}: must remain Partial`);
|
|
@@ -198,7 +199,16 @@ for (const dimension of ['hooksLifecycle', 'liveHostVerification']) {
|
|
|
198
199
|
if (opencode?.capabilities.liveHostVerification.evidence !== OPENCODE_EVIDENCE) {
|
|
199
200
|
fail('OpenCode liveHostVerification: missing exact retained artifact path');
|
|
200
201
|
}
|
|
201
|
-
|
|
202
|
+
if (codex?.capabilities.hooksLifecycle.status !== 'Partial') {
|
|
203
|
+
fail('Codex hooksLifecycle: expected partial live verification for the compatibility fallback');
|
|
204
|
+
}
|
|
205
|
+
if (codex?.capabilities.liveHostVerification.status !== 'Partial') {
|
|
206
|
+
fail('Codex liveHostVerification: expected partial status with preserved evidence');
|
|
207
|
+
}
|
|
208
|
+
if (codex?.capabilities.liveHostVerification.evidence !== 'benchmarks/results/live-host/codex-2026-10-13/README.md') {
|
|
209
|
+
fail('Codex liveHostVerification: missing exact retained artifact path');
|
|
210
|
+
}
|
|
211
|
+
for (const platform of matrix.platforms.filter(platform => !['opencode', 'codex'].includes(platform.id))) {
|
|
202
212
|
if (platform.capabilities.liveHostVerification.status !== 'Unknown') {
|
|
203
213
|
fail(`${platform.id}: liveHostVerification must remain Unknown without its own evidence`);
|
|
204
214
|
}
|
|
@@ -11,13 +11,15 @@ check(tier2.status===0,'kernel router exits successfully');
|
|
|
11
11
|
check(/RECOMMENDED TIER:\s*Tier 2/i.test(tier2.stdout),'classifier result is preserved');
|
|
12
12
|
check(tier2.stdout.includes('"strategy":"iterative-single"'),'Tier 2 selects iterative-single workflow');
|
|
13
13
|
check(tier2.stdout.includes('loop-awareness:'),'iterative work exposes loop-awareness instead of a hard budget');
|
|
14
|
-
check(tier2.stdout.includes('
|
|
14
|
+
check(tier2.stdout.includes('ACTIVE-STEP KNOWLEDGE BINDINGS (MUST)'),'active-step binding resolution is explicitly mandatory');
|
|
15
15
|
check(tier2.stdout.includes('ORCHESTRATION POLICY: Selected-topology required obligations and applicable skill core contracts are semantic MUSTs'),'orchestration policy preserves semantic MUST obligations');
|
|
16
16
|
check(tier2.stdout.includes('WORKFLOW SEMANTIC CONTRACT (NON-BLOCKING OBSERVATION)'),'kernel separates semantic contract from non-blocking observation');
|
|
17
17
|
check(tier2.stdout.includes('Tier 3 / Fable broad mutation MUST resolve isolation'),'worktree isolation disposition is a semantic MUST');
|
|
18
18
|
check(!tier2.stdout.includes('SELECTED WORKFLOW IS GUIDANCE'),'old planning-guidance label is absent');
|
|
19
19
|
check(!tier2.stdout.includes('unavailable isolation means BLOCKED'),'old worktree hard-block wording is absent');
|
|
20
|
-
check(tier2.stdout.includes('
|
|
20
|
+
check(tier2.stdout.includes('future-step bindings remain undisclosed'),'future step knowledge stays undisclosed');
|
|
21
|
+
check(tier2.stdout.includes('unknown/unavailable binding stays visible and unresolved'),'required binding gaps remain visible without adding a lock');
|
|
22
|
+
check(!tier2.stdout.includes('evaluate-suggestions-before-skip'),'global read-every-suggestion invariant is removed');
|
|
21
23
|
check(tier2.stdout.includes('visible-status-updates'),'workflow plan carries mandatory user-visible status invariant');
|
|
22
24
|
check(tier2.stdout.includes('USER-VISIBLE HARNESS STATUS CONTRACT (MUST)'),'kernel emits the mandatory unified status contract');
|
|
23
25
|
for(const field of ['### 🚦 Harness Status','- **Current:**','- **Read / Evidence:**','- **Next:**','- **Risk / Blocked:**']){
|
|
@@ -31,7 +33,8 @@ check(tier2.stdout.includes('semantic communication MUST, not a hard execution l
|
|
|
31
33
|
const tier3=spawnSync(process.execPath,[kernel,'audit the entire repository architecture and coordinate multiple modules'],{cwd:ROOT,encoding:'utf8'});
|
|
32
34
|
check(tier3.status===0,'Tier 3 routing exits successfully');
|
|
33
35
|
check(/RECOMMENDED TIER:\s*Tier 3/i.test(tier3.stdout),'macro task remains Tier 3');
|
|
34
|
-
check(tier3.stdout.includes('
|
|
36
|
+
check(tier3.stdout.includes('isolated-worktree-before-mutation'),'Tier 3 keeps the worktree isolation invariant');
|
|
37
|
+
check(!tier3.stdout.includes('using-git-worktrees/SKILL.md'),'Tier 3 does not push a global worktree skill path');
|
|
35
38
|
check(tier3.stdout.includes('Tier 3 / Fable broad mutation MUST resolve isolation'),'Tier 3 requires an isolation disposition without a hard lock');
|
|
36
39
|
|
|
37
40
|
const hs=read('harness-everything/SKILL.md');
|
|
@@ -60,7 +63,7 @@ const philosophy=read('docs/philosophy.md');
|
|
|
60
63
|
check(/Contract Strength: MUST \/ SHOULD \/ MAY/i.test(philosophy),'philosophy defines canonical MUST/SHOULD/MAY strength');
|
|
61
64
|
for(const phrase of [
|
|
62
65
|
'Route before software/project execution',
|
|
63
|
-
'
|
|
66
|
+
'Resolve active required skill/reference bindings before their step',
|
|
64
67
|
'Resolve selected-topology invariants/stages/checks/synthesis/verification',
|
|
65
68
|
'Discover relevant environment/host facts',
|
|
66
69
|
'Resolve Tier-3/Fable isolation before broad mutation',
|
|
@@ -16,7 +16,7 @@ const tier3 = router.buildRouterContract({
|
|
|
16
16
|
|
|
17
17
|
assert.strictEqual(tier3.strategy, 'fable-staged');
|
|
18
18
|
assert.ok(tier3.requiredInvariants.includes('isolated-worktree-before-mutation'), 'Tier 3 plan must carry mutation isolation invariant');
|
|
19
|
-
assert.
|
|
19
|
+
assert.strictEqual(tier3.suggestedSkills.length, 0, 'Tier 3 must not push a global worktree skill list');
|
|
20
20
|
assert.strictEqual(tier3.workspace.required, false, 'Git worktree isolation must remain distinct from durable multi-agent workspace state');
|
|
21
21
|
assert.strictEqual(tier3.mutationIsolation.onUnavailable, 'degraded', 'unavailable isolation records an explicit degraded fallback instead of a cognitive hard block');
|
|
22
22
|
|
|
@@ -29,7 +29,7 @@ const tier2 = router.buildRouterContract({
|
|
|
29
29
|
|
|
30
30
|
assert.strictEqual(tier2.strategy, 'iterative-single');
|
|
31
31
|
assert.ok(!tier2.requiredInvariants.includes('isolated-worktree-before-mutation'), 'Tier 2 must not be globally forced into major-workflow isolation');
|
|
32
|
-
assert.
|
|
32
|
+
assert.strictEqual(tier2.suggestedSkills.length, 0, 'Tier 2 also has no topology-wide skill suggestions');
|
|
33
33
|
|
|
34
34
|
const explicitFable = router.buildWorkflowPlan({
|
|
35
35
|
routingStatus: 'ok',
|
|
@@ -189,7 +189,8 @@ try {
|
|
|
189
189
|
});
|
|
190
190
|
check(routed.status === 0, '#169 large-workspace Tier-2 fixture routes normally');
|
|
191
191
|
const requirements = JSON.stringify([
|
|
192
|
-
{ id: 'req-fingerprint', summary: 'Exercise large-workspace shell admission', acceptance: 'the shell call remains available above the fingerprint observation cap' },
|
|
192
|
+
{ id: 'req-fingerprint', stepType: 'behavior-change', summary: 'Exercise large-workspace shell admission', acceptance: 'the shell call remains available above the fingerprint observation cap' },
|
|
193
|
+
{ id: 'req-verify', stepType: 'verification', summary: 'Verify shell admission', acceptance: 'objective test evidence is recorded' },
|
|
193
194
|
]);
|
|
194
195
|
const planned = node('hooks/scripts/workflow-disposition.js', undefined, [
|
|
195
196
|
'plan', '--session-id', sessionId,
|
|
@@ -73,6 +73,8 @@ try {
|
|
|
73
73
|
const state = compat.status({ codexHome });
|
|
74
74
|
check(state.installed && state.healthy && state.exactEntriesPresent === state.expectedEntries,
|
|
75
75
|
'status verifies exact installed entry hashes and runtime tree');
|
|
76
|
+
check(state.observationCapture.status === 'configured-unverified' && state.observationCapture.trustRequired,
|
|
77
|
+
'status distinguishes installed hooks from Codex-trusted live capture');
|
|
76
78
|
|
|
77
79
|
const config = read(hooksFile);
|
|
78
80
|
check(config.description === original.description &&
|
|
@@ -158,6 +160,18 @@ try {
|
|
|
158
160
|
check(!fs.existsSync(path.join(emptyHome, 'hooks.json')),
|
|
159
161
|
'uninstall removes hooks.json only when Harness created it and no user entries remain');
|
|
160
162
|
|
|
163
|
+
const marketplaceSource = path.join(root, 'marketplace-plugin');
|
|
164
|
+
const marketplaceHome = path.join(root, 'marketplace-codex');
|
|
165
|
+
fs.cpSync(compat.SOURCE_PLUGIN_ROOT, marketplaceSource, { recursive: true });
|
|
166
|
+
fs.writeFileSync(path.join(marketplaceSource, 'hooks', 'marketplace-source-marker.txt'), 'newer marketplace source\n');
|
|
167
|
+
const marketplaceInstall = compat.install({ codexHome: marketplaceHome, sourcePluginRoot: marketplaceSource });
|
|
168
|
+
const marketplaceManifest = read(path.join(marketplaceHome, 'harness-everything', 'compat-hooks', 'manifest.json'));
|
|
169
|
+
check(marketplaceInstall.sourcePluginSha256 === compat.treeHash(marketplaceSource) &&
|
|
170
|
+
marketplaceManifest.sourcePluginSha256 === compat.treeHash(marketplaceSource) &&
|
|
171
|
+
fs.readFileSync(path.join(marketplaceManifest.runtimeDir, 'hooks', 'marketplace-source-marker.txt'), 'utf8') === 'newer marketplace source\n',
|
|
172
|
+
'compatibility runtime is copied from the verified installed marketplace source');
|
|
173
|
+
compat.uninstall({ codexHome: marketplaceHome });
|
|
174
|
+
|
|
161
175
|
const duplicateHome = path.join(root, 'duplicate-codex');
|
|
162
176
|
fs.mkdirSync(duplicateHome, { recursive: true });
|
|
163
177
|
compat.install({ codexHome: duplicateHome });
|
|
@@ -273,6 +287,11 @@ try {
|
|
|
273
287
|
check(malformedRefused && fs.readFileSync(malformed, 'utf8') === malformedBefore,
|
|
274
288
|
'malformed user hooks.json fails closed without replacement');
|
|
275
289
|
|
|
290
|
+
const inactive = compat.status({ codexHome: path.join(root, 'not-installed') });
|
|
291
|
+
check(inactive.observationCapture.status === 'inactive' &&
|
|
292
|
+
inactive.observationCapture.reason === 'compatibility-hooks-not-installed',
|
|
293
|
+
'status clearly reports inactive observation capture when hooks are absent');
|
|
294
|
+
|
|
276
295
|
const source = read(path.join(compat.SOURCE_PLUGIN_ROOT, 'hooks', 'hooks.json'));
|
|
277
296
|
const sourceCount = Object.values(source.hooks).reduce((sum, entries) => sum + entries.length, 0);
|
|
278
297
|
check(sourceCount === installed.entries,
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const assert = require('assert');
|
|
5
|
+
const fs = require('fs');
|
|
6
|
+
const os = require('os');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const { spawnSync } = require('child_process');
|
|
9
|
+
|
|
10
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
11
|
+
const IMPORTER = path.join(ROOT, 'scripts', 'system-one-observations-import-codex-sessions.js');
|
|
12
|
+
const EXPORTER = path.join(ROOT, 'scripts', 'system-one-observations-export.js');
|
|
13
|
+
const REVIEW = path.join(ROOT, 'scripts', 'system-one-observations-review.js');
|
|
14
|
+
const obs = require('../hooks/scripts/lib/observations');
|
|
15
|
+
const LABEL = '<!-- harness-label {"v":1,"validity":"actionable","contextDependent":false,"tier":"tier2","intents":["fix"],"workflow":null,"skills":[]} -->';
|
|
16
|
+
const TIER1_LABEL = '<!-- harness-label {"v":1,"validity":"actionable","contextDependent":false,"tier":"tier1","intents":["fix"],"workflow":null,"skills":[]} -->';
|
|
17
|
+
|
|
18
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-session-label-import-'));
|
|
19
|
+
const sessions = path.join(root, 'sessions');
|
|
20
|
+
const store = path.join(root, 'observations');
|
|
21
|
+
fs.mkdirSync(path.join(sessions, '2026', '10'), { recursive: true });
|
|
22
|
+
const rollout = path.join(sessions, '2026', '10', 'rollout-fixture.jsonl');
|
|
23
|
+
|
|
24
|
+
const rows = [
|
|
25
|
+
{ type: 'session_meta', payload: { session_id: 'fixture-session' } },
|
|
26
|
+
{ type: 'event_msg', timestamp: '2026-09-26T23:59:59.000Z', payload: { type: 'task_started', turn_id: 'old', started_at: '2026-09-26T23:59:59.000Z' } },
|
|
27
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'pre-revision prompt' }] } },
|
|
28
|
+
{ type: 'event_msg', timestamp: '2026-09-26T23:59:59.500Z', payload: { type: 'task_complete', turn_id: 'old', last_agent_message: LABEL } },
|
|
29
|
+
{ type: 'event_msg', timestamp: '2026-09-27T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'included', started_at: '2026-09-27T00:00:00.000Z' } },
|
|
30
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [
|
|
31
|
+
{ type: 'input_text', text: 'Please fix the parser. Contact user@example.com.' },
|
|
32
|
+
{ type: 'input_text', text: 'injected host context must stay out of the prompt row' }
|
|
33
|
+
] } },
|
|
34
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'later context message' }] } },
|
|
35
|
+
{ type: 'event_msg', timestamp: '2026-09-27T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'included', last_agent_message: 'Done.\n' + LABEL } },
|
|
36
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'assistant-fallback', started_at: '2026-09-30T00:00:00.000Z' } },
|
|
37
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'fallback prompt from the rollout message' }] } },
|
|
38
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Completed answer.\n' + LABEL }] } },
|
|
39
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'assistant-fallback' } },
|
|
40
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'explicit-unlabeled-final', started_at: '2026-09-30T00:00:00.000Z' } },
|
|
41
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'an earlier assistant message had a label' }] } },
|
|
42
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', phase: 'commentary', content: [{ type: 'output_text', text: 'Earlier answer.\n' + TIER1_LABEL }] } },
|
|
43
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'explicit-unlabeled-final', last_agent_message: 'This is the explicit final reply without a label.' } },
|
|
44
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'unobserved-write', started_at: '2026-09-30T00:00:00.000Z' } },
|
|
45
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'write a small parser change' }] } },
|
|
46
|
+
{ type: 'response_item', payload: { type: 'function_call', name: 'apply_patch', arguments: '*** Begin Patch\n*** Add File: parser.js\n+module.exports = true;\n*** End Patch' } },
|
|
47
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'unobserved-write', last_agent_message: TIER1_LABEL } },
|
|
48
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'not-final-assistant', started_at: '2026-09-30T00:00:00.000Z' } },
|
|
49
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'do not use an earlier assistant message label' }] } },
|
|
50
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: LABEL }] } },
|
|
51
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'A later assistant message replaced this one.' }] } },
|
|
52
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'not-final-assistant', last_agent_message: 'No label in the final assistant message.' } },
|
|
53
|
+
{ type: 'event_msg', timestamp: '2026-09-28T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'missing-prompt', started_at: '2026-09-28T00:00:00.000Z' } },
|
|
54
|
+
{ type: 'event_msg', timestamp: '2026-09-28T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'missing-prompt', last_agent_message: LABEL } },
|
|
55
|
+
{ type: 'event_msg', timestamp: '2026-09-29T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'invalid-label', started_at: '2026-09-29T00:00:00.000Z' } },
|
|
56
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'malformed label prompt' }] } },
|
|
57
|
+
{ type: 'event_msg', timestamp: '2026-09-29T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'invalid-label', last_agent_message: '<!-- harness-label {bad json} -->' } },
|
|
58
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'ambiguous-summary', started_at: '2026-09-30T00:00:00.000Z' } },
|
|
59
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'resolve conflicting final representations' }] } },
|
|
60
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', phase: 'final_answer', content: [{ type: 'output_text', text: 'Actual final answer.\n' + LABEL }] } },
|
|
61
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'ambiguous-summary', last_agent_message: 'The completion summary omitted the final response text.' } },
|
|
62
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:02:00.000Z', payload: { type: 'task_started', turn_id: 'after-ambiguous', started_at: '2026-09-30T00:02:00.000Z' } },
|
|
63
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'continue after ambiguous final representations' }] } },
|
|
64
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:03:00.000Z', payload: { type: 'task_complete', turn_id: 'after-ambiguous', last_agent_message: 'Follow-up answer.\n' + LABEL } },
|
|
65
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:04:00.000Z', payload: { type: 'task_started', turn_id: 'equivalent-body', started_at: '2026-09-30T00:04:00.000Z' } },
|
|
66
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'recover an equivalent assistant body' }] } },
|
|
67
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', phase: 'final_answer', content: [{ type: 'output_text', text: 'Equivalent final answer.\n' + LABEL }] } },
|
|
68
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:05:00.000Z', payload: { type: 'task_complete', turn_id: 'equivalent-body', last_agent_message: 'Equivalent final answer.' } },
|
|
69
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:06:00.000Z', payload: { type: 'task_started', turn_id: 'nonstring-completion', started_at: '2026-09-30T00:06:00.000Z' } },
|
|
70
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'fallback from non-string completion' }] } },
|
|
71
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Non-string fallback answer.\n' + LABEL }] } },
|
|
72
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:07:00.000Z', payload: { type: 'task_complete', turn_id: 'nonstring-completion', last_agent_message: null } },
|
|
73
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:08:00.000Z', payload: { type: 'task_started', turn_id: 'invalid-with-assistant-label', started_at: '2026-09-30T00:08:00.000Z' } },
|
|
74
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'reject malformed completion despite assistant label' }] } },
|
|
75
|
+
{ type: 'response_item', payload: { type: 'message', role: 'assistant', phase: 'final_answer', content: [{ type: 'output_text', text: 'A different assistant body.\n' + LABEL }] } },
|
|
76
|
+
{ type: 'event_msg', timestamp: '2026-09-30T00:09:00.000Z', payload: { type: 'task_complete', turn_id: 'invalid-with-assistant-label', last_agent_message: '<!-- harness-label {bad json} -->' } },
|
|
77
|
+
];
|
|
78
|
+
fs.writeFileSync(rollout, rows.map(row => JSON.stringify(row)).join('\n') + '\n');
|
|
79
|
+
|
|
80
|
+
function runImport(targetStore = store) {
|
|
81
|
+
return spawnSync(process.execPath, [IMPORTER, '--sessions', sessions, '--store', targetStore, '--since', '2026-09-27'], {
|
|
82
|
+
cwd: ROOT,
|
|
83
|
+
encoding: 'utf8',
|
|
84
|
+
timeout: 10000,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const hookEnv = { HARNESS_OBSERVATIONS: 'on', HARNESS_OBSERVATIONS_DIR: store };
|
|
90
|
+
obs.handle({ hook_event_name: 'UserPromptSubmit', session_id: 'fixture-session', turn_id: 'included',
|
|
91
|
+
prompt: 'Please fix the parser. Contact user@example.com.', cwd: root }, hookEnv);
|
|
92
|
+
obs.handle({ hook_event_name: 'PostToolUse', session_id: 'fixture-session', turn_id: 'included', cwd: root,
|
|
93
|
+
tool_name: 'apply_patch', tool_input: { patch: '*** Begin Patch\n*** Add File: parser.js\n+module.exports = true;\n*** End Patch' } }, hookEnv);
|
|
94
|
+
obs.handle({ hook_event_name: 'Stop', session_id: 'fixture-session', turn_id: 'included', last_agent_message: TIER1_LABEL }, hookEnv);
|
|
95
|
+
|
|
96
|
+
const first = runImport();
|
|
97
|
+
assert.strictEqual(first.status, 0, 'session import failed:\n' + first.stdout + '\n' + first.stderr);
|
|
98
|
+
const report = JSON.parse(first.stdout);
|
|
99
|
+
assert.strictEqual(report.imported, 5);
|
|
100
|
+
assert.strictEqual(report.skipped.beforeCutoff, 1);
|
|
101
|
+
assert.strictEqual(report.skipped.missingPrompt, 1);
|
|
102
|
+
assert.strictEqual(report.skipped.missingLabel, 2, 'an explicit unlabeled completion remains a missing-label skip when the labeled assistant item was commentary');
|
|
103
|
+
assert.strictEqual(report.skipped.ambiguousFinalMessage, 1, 'conflicting final representations with a labeled final assistant item are reported as ambiguous');
|
|
104
|
+
assert.strictEqual(report.skipped.invalidLabel, 2, 'an invalid completion label remains invalid even when an assistant item has a valid label');
|
|
105
|
+
assert.strictEqual(report.skipped.duplicate, 1, 'backfill skips a turn already captured by live Codex hooks');
|
|
106
|
+
assert.strictEqual(report.skipped.ambiguousLiveMatch, 0);
|
|
107
|
+
|
|
108
|
+
const index = JSON.parse(fs.readFileSync(path.join(store, 'observations-index.json'), 'utf8'));
|
|
109
|
+
assert.strictEqual(index.records.length, 6);
|
|
110
|
+
const liveRecord = index.records.find(entry => entry.writer && entry.writer.turnId === 'included');
|
|
111
|
+
assert(liveRecord, 'live records retain Codex turn IDs for cross-source dedupe');
|
|
112
|
+
assert.strictEqual(liveRecord.behavior.filesWritten, 1, 'the live fixture contains an observed file write');
|
|
113
|
+
assert.deepStrictEqual(obs.contradictions(liveRecord.selfReport, liveRecord.behavior), ['tier1-with-writes']);
|
|
114
|
+
assert.strictEqual(index.records.filter(entry => entry.writer && entry.writer.sessionId === 'fixture-session' && entry.writer.turnId === 'included').length, 1,
|
|
115
|
+
'live capture and session backfill leave one record for a turn');
|
|
116
|
+
|
|
117
|
+
const fallbackRecord = index.records.find(entry => entry.captureMethod === 'session-backfill' && entry.turn === 3);
|
|
118
|
+
assert(fallbackRecord, 'the response_item assistant label is recovered when task_complete omits it');
|
|
119
|
+
const fallbackText = JSON.parse(fs.readFileSync(path.join(store, 'text', fallbackRecord.contentSha256 + '.json'), 'utf8'));
|
|
120
|
+
assert.strictEqual(fallbackText.prompt, 'fallback prompt from the rollout message');
|
|
121
|
+
const backfilledTier1 = index.records.find(entry => entry.captureMethod === 'session-backfill' && entry.selfReport.tier === 'tier1');
|
|
122
|
+
assert(backfilledTier1, 'the tier1 label from the rollout is imported');
|
|
123
|
+
assert.strictEqual(backfilledTier1.behavior, null, 'session backfill records behavior as unobserved rather than zero');
|
|
124
|
+
assert(!index.records.some(entry => entry.writer && entry.writer.turnId === 'ambiguous-summary'),
|
|
125
|
+
'a differing completion string must not inherit the assistant message label');
|
|
126
|
+
const afterAmbiguous = index.records.find(entry => entry.writer && entry.writer.turnId === 'after-ambiguous');
|
|
127
|
+
assert(afterAmbiguous, 'the turn after an ambiguous final can still be imported');
|
|
128
|
+
const afterAmbiguousText = JSON.parse(fs.readFileSync(path.join(store, 'text', afterAmbiguous.contentSha256 + '.json'), 'utf8'));
|
|
129
|
+
assert.strictEqual(afterAmbiguousText.previous, '', 'an unresolved final disagreement must not seed later previous-message context');
|
|
130
|
+
assert(index.records.some(entry => entry.writer && entry.writer.turnId === 'equivalent-body'),
|
|
131
|
+
'a completion string matching the assistant body after label removal safely recovers its label');
|
|
132
|
+
assert(index.records.some(entry => entry.writer && entry.writer.turnId === 'nonstring-completion'),
|
|
133
|
+
'a non-string task_complete field falls back to the final assistant label');
|
|
134
|
+
assert(!index.records.some(entry => entry.writer && entry.writer.turnId === 'invalid-with-assistant-label'),
|
|
135
|
+
'a valid assistant label never replaces an invalid completion label');
|
|
136
|
+
|
|
137
|
+
const exportDir = path.join(root, 'export');
|
|
138
|
+
const exported = spawnSync(process.execPath, [EXPORTER, '--store', store, '--out', exportDir, '--self-report-only'], { cwd: ROOT, encoding: 'utf8', timeout: 10000 });
|
|
139
|
+
assert.strictEqual(exported.status, 0, 'export after backfill failed:\n' + exported.stdout + '\n' + exported.stderr);
|
|
140
|
+
const labels = fs.readFileSync(path.join(exportDir, 'labels-observed.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
|
|
141
|
+
const exportReport = JSON.parse(fs.readFileSync(path.join(exportDir, 'export-report.json'), 'utf8'));
|
|
142
|
+
assert.strictEqual(labels.filter(label => label.behaviorEvidence === 'unobserved').length, 5);
|
|
143
|
+
assert.deepStrictEqual(exportReport.behaviorEvidence, { observed: 1, unobserved: 5 });
|
|
144
|
+
assert.strictEqual(exportReport.contradictions['tier1-with-writes'], 1, 'only the observed live write contributes a contradiction');
|
|
145
|
+
|
|
146
|
+
const reviewHtml = path.join(root, 'review.html');
|
|
147
|
+
const reviewed = spawnSync(process.execPath, [REVIEW, '--store', store, '--out', reviewHtml], { cwd: ROOT, encoding: 'utf8', timeout: 10000 });
|
|
148
|
+
assert.strictEqual(reviewed.status, 0, 'observation review page failed:\n' + reviewed.stdout + '\n' + reviewed.stderr);
|
|
149
|
+
assert.match(fs.readFileSync(reviewHtml, 'utf8'), /Behavior evidence is unobserved/);
|
|
150
|
+
|
|
151
|
+
const legacyStore = path.join(root, 'legacy-observations');
|
|
152
|
+
fs.mkdirSync(legacyStore, { recursive: true });
|
|
153
|
+
fs.writeFileSync(path.join(legacyStore, 'observations-index.json'), JSON.stringify({ schemaVersion: 1, records: [
|
|
154
|
+
{
|
|
155
|
+
id: 'legacy-live-record', source: 'observation', host: 'codex', turn: 2,
|
|
156
|
+
writer: { sessionId: 'fixture-session' }, contentSha256: 'legacy-content-hash', behavior: { filesWritten: 1 }
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
id: 'old-backfill-record', source: 'observation', captureMethod: 'session-backfill', host: 'codex', turn: 1,
|
|
160
|
+
writer: { sessionId: 'older-session', turnId: 'old' }, behavior: { filesWritten: 0, toolCalls: 0 }
|
|
161
|
+
}
|
|
162
|
+
] }));
|
|
163
|
+
const legacy = runImport(legacyStore);
|
|
164
|
+
assert.strictEqual(legacy.status, 0, legacy.stderr);
|
|
165
|
+
const legacyReport = JSON.parse(legacy.stdout);
|
|
166
|
+
assert.strictEqual(legacyReport.skipped.ambiguousLiveMatch, 1,
|
|
167
|
+
'legacy live records without a turn ID are explicitly skipped when the same session/turn is ambiguous');
|
|
168
|
+
assert.strictEqual(legacyReport.migratedUnobservedBehavior, 1, 'prior backfill records are migrated away from synthetic zero behavior');
|
|
169
|
+
const legacyIndex = JSON.parse(fs.readFileSync(path.join(legacyStore, 'observations-index.json'), 'utf8'));
|
|
170
|
+
assert.strictEqual(legacyIndex.records.find(entry => entry.id === 'old-backfill-record').behavior, null);
|
|
171
|
+
|
|
172
|
+
const second = runImport();
|
|
173
|
+
assert.strictEqual(second.status, 0, second.stderr);
|
|
174
|
+
assert.strictEqual(JSON.parse(second.stdout).imported, 0, 'repeated imports must not duplicate records');
|
|
175
|
+
assert.strictEqual(JSON.parse(second.stdout).skipped.duplicate, 6);
|
|
176
|
+
assert.strictEqual(JSON.parse(fs.readFileSync(path.join(store, 'observations-index.json'), 'utf8')).records.length, 6);
|
|
177
|
+
|
|
178
|
+
const lockSessions = path.join(root, 'lock-timeout-sessions');
|
|
179
|
+
fs.mkdirSync(lockSessions, { recursive: true });
|
|
180
|
+
const lockRollout = path.join(lockSessions, 'lock-timeout.jsonl');
|
|
181
|
+
fs.writeFileSync(lockRollout, [
|
|
182
|
+
{ type: 'event_msg', timestamp: '2026-10-01T00:00:00.000Z', payload: { type: 'task_started', turn_id: 'lock-timeout', started_at: '2026-10-01T00:00:00.000Z' } },
|
|
183
|
+
{ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'a unique turn that cannot be persisted while locked' }] } },
|
|
184
|
+
{ type: 'event_msg', timestamp: '2026-10-01T00:01:00.000Z', payload: { type: 'task_complete', turn_id: 'lock-timeout', last_agent_message: LABEL } },
|
|
185
|
+
].map(row => JSON.stringify(row)).join('\n') + '\n');
|
|
186
|
+
const lockStore = path.join(root, 'lock-timeout-store');
|
|
187
|
+
fs.mkdirSync(lockStore, { recursive: true });
|
|
188
|
+
fs.writeFileSync(path.join(lockStore, '.index.lock'), 'held for importer timeout fixture');
|
|
189
|
+
const locked = spawnSync(process.execPath, [IMPORTER, '--sessions', lockSessions, '--store', lockStore, '--since', '2026-09-27'], {
|
|
190
|
+
cwd: ROOT,
|
|
191
|
+
encoding: 'utf8',
|
|
192
|
+
timeout: 10000,
|
|
193
|
+
});
|
|
194
|
+
assert.strictEqual(locked.status, 1, 'an importer lock timeout must return a failing CLI exit code');
|
|
195
|
+
const lockReport = JSON.parse(locked.stdout);
|
|
196
|
+
assert.strictEqual(lockReport.failed.lockTimeout, 1, 'the report distinguishes a lock timeout from a duplicate');
|
|
197
|
+
assert.strictEqual(fs.existsSync(path.join(lockStore, 'observations-index.json')), false,
|
|
198
|
+
'a unique turn must not be reported as persisted when the index lock times out');
|
|
199
|
+
|
|
200
|
+
const migrationStore = path.join(root, 'migration-timeout-store');
|
|
201
|
+
fs.mkdirSync(migrationStore, { recursive: true });
|
|
202
|
+
fs.writeFileSync(path.join(migrationStore, 'observations-index.json'), JSON.stringify({ schemaVersion: 1, records: [
|
|
203
|
+
{ id: 'old-backfill', captureMethod: 'session-backfill', behavior: { filesWritten: 0 } }
|
|
204
|
+
] }));
|
|
205
|
+
fs.writeFileSync(path.join(migrationStore, '.index.lock'), 'held during migration fixture');
|
|
206
|
+
const migrationLocked = spawnSync(process.execPath, [IMPORTER, '--sessions', lockSessions, '--store', migrationStore, '--since', '2026-09-27'], {
|
|
207
|
+
cwd: ROOT,
|
|
208
|
+
encoding: 'utf8',
|
|
209
|
+
timeout: 10000,
|
|
210
|
+
});
|
|
211
|
+
assert.strictEqual(migrationLocked.status, 1, 'a migration lock timeout must also fail the importer CLI');
|
|
212
|
+
assert.strictEqual(JSON.parse(migrationLocked.stdout).failed.lockTimeout, 1,
|
|
213
|
+
'the migration path reports its lock timeout through the same failure count');
|
|
214
|
+
const migrationIndex = JSON.parse(fs.readFileSync(path.join(migrationStore, 'observations-index.json'), 'utf8'));
|
|
215
|
+
assert.deepStrictEqual(migrationIndex.records[0].behavior, { filesWritten: 0 },
|
|
216
|
+
'a failed migration does not rewrite the index or proceed with imports');
|
|
217
|
+
} finally {
|
|
218
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
console.log('PASS: post-revision Codex session labels import as redacted, prompt-paired, idempotent observations.');
|