@openwop/openwop-conformance 2.42.8 → 2.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +3 -3
- package/coverage.md +1 -1
- package/dist/cli.js +3 -4
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures.md +2 -1
- package/package.json +2 -2
- package/requirements.json +374 -33
- package/scenario-majors.json +17 -5
- package/schemas/CORPUS-STAMP.json +37 -36
- package/src/cli.ts +3 -4
- package/src/lib/auditIntegrity.ts +45 -0
- package/src/lib/mock-ai-budgets.ts +26 -0
- package/src/lib/mock-node-lock.ts +66 -0
- package/src/scenarios/audit-anomaly-shape.test.ts +100 -0
- package/src/scenarios/audit-checkpoint-signature.test.ts +14 -15
- package/src/scenarios/audit-log-integrity.test.ts +102 -46
- package/src/scenarios/byok-roundtrip.test.ts +21 -7
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +17 -8
- package/src/scenarios/envelope-retry-attempted.test.ts +4 -0
- package/src/scenarios/envelope-truncated.test.ts +16 -16
- package/src/scenarios/v2-approval-reject-disposition.test.ts +162 -0
- package/src/scenarios/v2-registry-lifecycle.test.ts +232 -0
- package/src/scenarios/v2-webhook-delivery-shape.test.ts +5 -0
|
@@ -1,11 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Track 13: audit-log integrity profile (auth-profiles.md v1.1)
|
|
2
|
+
* Track 13: audit-log integrity profile (auth-profiles.md v1.1), and at major 2
|
|
3
|
+
* the `auditLogIntegrity` family (RFC 0224, spec/v2/core/security-defaults.md
|
|
4
|
+
* §Audit-log integrity).
|
|
3
5
|
*
|
|
4
|
-
* Verifies that hosts claiming
|
|
5
|
-
* 1.
|
|
6
|
-
*
|
|
6
|
+
* Verifies that hosts claiming audit-log integrity:
|
|
7
|
+
* 1. Advertise the facets a verifier needs — at major 1
|
|
8
|
+
* `capabilities.auth.auditLogIntegrity.hashChain: true` plus the key; at
|
|
9
|
+
* major 2 a record that validates against the generated family schema,
|
|
10
|
+
* whose `checkpointPublicKey` is an Ed25519 SPKI key.
|
|
11
|
+
* 2. Expose `GET /v1/audit/verify` (major 2: `GET /audit/verify`) which
|
|
12
|
+
* returns `{chainValid, checkpoints, anomalies}`.
|
|
7
13
|
* 3. Report `chainValid: true` for an unmodified range.
|
|
8
14
|
* 4. Surface at least one signed checkpoint with a non-empty `signature`.
|
|
15
|
+
* 5. List checkpoints ascending, no more than `checkpointIntervalEntries`
|
|
16
|
+
* apart (RFC 0224 §A.3). A host that omits a checkpoint from the verify
|
|
17
|
+
* body leaves a gap the advertised cadence forbids.
|
|
18
|
+
*
|
|
19
|
+
* Major 1 gates through behaviorGate (strict mode fails an unadvertised
|
|
20
|
+
* profile). Major 2 records `inapplicable` before any assertion when the
|
|
21
|
+
* family is absent — presence of the record is the claim.
|
|
9
22
|
*
|
|
10
23
|
* Tamper detection (mutating an entry then asserting `chainValid: false`)
|
|
11
24
|
* requires admin access to the host's audit store and is NOT exercised
|
|
@@ -13,14 +26,18 @@
|
|
|
13
26
|
* test for tamper detection — see auth-profiles.md.
|
|
14
27
|
*
|
|
15
28
|
* @see spec/v1/auth-profiles.md §"Audit-log integrity"
|
|
29
|
+
* @see spec/v2/core/security-defaults.md §"Audit-log integrity"
|
|
16
30
|
*/
|
|
17
31
|
|
|
18
32
|
import { describe, it, expect } from 'vitest';
|
|
19
33
|
import { driver } from '../lib/driver.js';
|
|
20
34
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
21
|
-
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
22
35
|
import { req } from '../lib/requirement-ids.js';
|
|
23
36
|
import { softSkip } from '../lib/soft-skip.js';
|
|
37
|
+
import { targetMajor } from '../lib/seams.js';
|
|
38
|
+
import { v2RefValidator, v2Validator } from '../lib/v2.js';
|
|
39
|
+
import { checkpointPublicKey } from '../lib/audit-checkpoint.js';
|
|
40
|
+
import { AUDIT_PROFILE, auditClaimName, auditIntegrityAdvert, auditVerifyPath, type AuditIntegrityCaps } from '../lib/auditIntegrity.js';
|
|
24
41
|
import Ajv2020 from 'ajv/dist/2020.js';
|
|
25
42
|
import { readFileSync } from 'node:fs';
|
|
26
43
|
import { join } from 'node:path';
|
|
@@ -31,67 +48,77 @@ import { SCHEMAS_DIR } from '../lib/paths.js';
|
|
|
31
48
|
* is enforced. (Until 2026-09-27 the suite stripped them because the reference
|
|
32
49
|
* postgres/sqlite hosts emitted a per-checkpoint `verified` bit the schema does
|
|
33
50
|
* not name; the hosts now keep that bit host-internal. The aggregate verdict is
|
|
34
|
-
* the schema's OPTIONAL `checkpointsValid`.)
|
|
51
|
+
* the schema's OPTIONAL `checkpointsValid`.) Major 2 validates against the v2 copy.
|
|
35
52
|
*/
|
|
36
53
|
const VERIFY_RESULT_SCHEMA = JSON.parse(
|
|
37
54
|
readFileSync(join(SCHEMAS_DIR, 'audit-verify-result.schema.json'), 'utf8'),
|
|
38
55
|
) as Record<string, unknown>;
|
|
39
56
|
|
|
40
|
-
|
|
41
|
-
hashChain?: boolean;
|
|
42
|
-
checkpointSignatureAlgorithm?: string;
|
|
43
|
-
checkpointPublicKey?: string;
|
|
44
|
-
checkpointIntervalEntries?: number;
|
|
45
|
-
checkpointIntervalSeconds?: number;
|
|
46
|
-
}
|
|
57
|
+
const CADENCE = 'openwop.requirement.0224.checkpoint-cadence';
|
|
47
58
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
59
|
+
/**
|
|
60
|
+
* The gate, per major. Major 1: behaviorGate on the profile (unchanged). Major 2:
|
|
61
|
+
* the family record or null. A null is recorded `inapplicable` by the caller
|
|
62
|
+
* before any assertion (`unclaimed()`).
|
|
63
|
+
*/
|
|
64
|
+
async function gate(): Promise<AuditIntegrityCaps | null> {
|
|
65
|
+
const caps = await auditIntegrityAdvert();
|
|
66
|
+
if (targetMajor() === 2) return caps;
|
|
67
|
+
return behaviorGate(AUDIT_PROFILE, caps !== null) ? caps : null;
|
|
51
68
|
}
|
|
52
69
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
70
|
+
const unclaimed = (): undefined => softSkip('inapplicable', `the host does not claim ${auditClaimName()} at major ${targetMajor()}`);
|
|
71
|
+
|
|
72
|
+
function validateVerifyBody(body: unknown): { ok: boolean; errors: string } {
|
|
73
|
+
if (targetMajor() === 2) return v2Validator('audit-verify-result')(body);
|
|
74
|
+
const validate = new Ajv2020({ strict: false, allErrors: true }).compile(VERIFY_RESULT_SCHEMA);
|
|
75
|
+
const ok = validate(body) as boolean;
|
|
76
|
+
return { ok, errors: JSON.stringify(validate.errors ?? []) };
|
|
57
77
|
}
|
|
58
78
|
|
|
59
79
|
describe('audit-log-integrity: profile shape', () => {
|
|
60
80
|
it('host that claims the profile advertises required capability fields', async () => {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const disco = await driver.get('/.well-known/openwop');
|
|
66
|
-
const integrity =
|
|
67
|
-
capabilityFamily<AuthCaps>(disco.json, 'auth')
|
|
68
|
-
?.auditLogIntegrity ?? {};
|
|
81
|
+
const integrity = await gate();
|
|
82
|
+
if (integrity === null) return unclaimed();
|
|
69
83
|
|
|
70
|
-
|
|
71
|
-
'
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
84
|
+
if (targetMajor() === 2) {
|
|
85
|
+
const shape = v2RefValidator('capabilities.schema.json#/properties/auditLogIntegrity')(integrity);
|
|
86
|
+
expect(shape.ok, req('openwop.it.audit-log-integrity.host-that-claims-the-profile-advertises-required-capability-fields',
|
|
87
|
+
'security-defaults.md §"Audit-log integrity"',
|
|
88
|
+
`the auditLogIntegrity record MUST validate against the generated family schema: ${shape.errors}`,
|
|
89
|
+
)).toBe(true);
|
|
90
|
+
let keyOk = true;
|
|
91
|
+
try { checkpointPublicKey(String(integrity.checkpointPublicKey ?? '')); } catch { keyOk = false; }
|
|
92
|
+
expect(keyOk, req('openwop.it.audit-log-integrity.host-that-claims-the-profile-advertises-required-capability-fields',
|
|
93
|
+
'security-defaults.md §"Audit-log integrity"',
|
|
94
|
+
'auditLogIntegrity.checkpointPublicKey MUST be an Ed25519 SubjectPublicKeyInfo, base64',
|
|
95
|
+
)).toBe(true);
|
|
96
|
+
} else {
|
|
97
|
+
expect(integrity.hashChain, req('openwop.it.audit-log-integrity.host-that-claims-the-profile-advertises-required-capability-fields',
|
|
98
|
+
'auth-profiles.md §"Audit-log integrity"',
|
|
99
|
+
"openwop-audit-log-integrity profile MUST advertise auditLogIntegrity.hashChain: true",
|
|
100
|
+
)).toBe(true);
|
|
101
|
+
expect(integrity.checkpointSignatureAlgorithm, req('openwop.it.audit-log-integrity.host-that-claims-the-profile-advertises-required-capability-fields',
|
|
102
|
+
'auth-profiles.md §"Audit-log integrity" §"Key management"',
|
|
103
|
+
'checkpointSignatureAlgorithm MUST be present (canonical: ed25519)',
|
|
104
|
+
)).toBeDefined();
|
|
105
|
+
expect(typeof integrity.checkpointPublicKey).toBe('string');
|
|
106
|
+
}
|
|
79
107
|
});
|
|
80
108
|
});
|
|
81
109
|
|
|
82
110
|
describe('audit-log-integrity: verify endpoint returns chainValid', () => {
|
|
83
111
|
it('GET /v1/audit/verify on a recent range reports chainValid: true', async () => {
|
|
84
|
-
if (
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
112
|
+
if ((await gate()) === null) return unclaimed();
|
|
87
113
|
|
|
88
|
-
const
|
|
114
|
+
const path = auditVerifyPath(0, 100);
|
|
115
|
+
const verify = await driver.get(path);
|
|
89
116
|
if (verify.status === 404) {
|
|
90
117
|
// Host claims the profile but doesn't expose the endpoint — that's
|
|
91
118
|
// a profile-claim violation. Fail explicitly.
|
|
92
119
|
expect(verify.status, req('openwop.it.audit-log-integrity.get-v1-audit-verify-on-a-recent-range-reports-chainvalid-true',
|
|
93
120
|
'auth-profiles.md §"Audit-log integrity" §"Verification endpoint"',
|
|
94
|
-
|
|
121
|
+
`claiming ${auditClaimName()} REQUIRES exposing GET ${path.split('?')[0]}`,
|
|
95
122
|
)).not.toBe(404);
|
|
96
123
|
return softSkip('blocked', 'precondition not met — `verify.status === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
97
124
|
}
|
|
@@ -111,11 +138,10 @@ describe('audit-log-integrity: verify endpoint returns chainValid', () => {
|
|
|
111
138
|
// length > 0)`. The body is now validated against
|
|
112
139
|
// audit-verify-result.schema.json (checkpoints[] + anomalies[] required;
|
|
113
140
|
// each checkpoint needs checkpoint/atSequence/merkleRoot/signature).
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
expect(shapeOk, req('openwop.it.audit-log-integrity.get-v1-audit-verify-on-a-recent-range-reports-chainvalid-true',
|
|
141
|
+
const shape = validateVerifyBody(body);
|
|
142
|
+
expect(shape.ok, req('openwop.it.audit-log-integrity.get-v1-audit-verify-on-a-recent-range-reports-chainvalid-true',
|
|
117
143
|
'audit-verify-result.schema.json',
|
|
118
|
-
`GET
|
|
144
|
+
`GET ${path.split('?')[0]} MUST return the AuditVerifyResult shape: ${shape.errors}`,
|
|
119
145
|
)).toBe(true);
|
|
120
146
|
|
|
121
147
|
expect(body.chainValid, req('openwop.it.audit-log-integrity.get-v1-audit-verify-on-a-recent-range-reports-chainvalid-true',
|
|
@@ -132,3 +158,33 @@ describe('audit-log-integrity: verify endpoint returns chainValid', () => {
|
|
|
132
158
|
}
|
|
133
159
|
});
|
|
134
160
|
});
|
|
161
|
+
|
|
162
|
+
describe('audit-log-integrity: checkpoints at the advertised cadence', () => {
|
|
163
|
+
it('verify lists every checkpoint ascending, no more than checkpointIntervalEntries apart', async () => {
|
|
164
|
+
const integrity = await gate();
|
|
165
|
+
if (integrity === null) return unclaimed();
|
|
166
|
+
const every = integrity.checkpointIntervalEntries;
|
|
167
|
+
if (typeof every !== 'number' || !Number.isInteger(every) || every < 1) {
|
|
168
|
+
// Required at major 2 (the shape leg owns that failure); optional at major 1.
|
|
169
|
+
return softSkip(targetMajor() === 2 ? 'blocked' : 'inapplicable', 'checkpointIntervalEntries is not advertised as a positive integer, so no cadence is claimed');
|
|
170
|
+
}
|
|
171
|
+
const verify = await driver.get(auditVerifyPath(0, 1_000_000));
|
|
172
|
+
if (verify.status !== 200) {
|
|
173
|
+
return softSkip('blocked', `the verify endpoint answered ${verify.status}; the verify leg owns that contract`);
|
|
174
|
+
}
|
|
175
|
+
const seqs = (((verify.json as { checkpoints?: Array<{ atSequence?: unknown }> }).checkpoints) ?? []).map((c) => c.atSequence);
|
|
176
|
+
if (seqs.length === 0) {
|
|
177
|
+
return softSkip('inapplicable', 'the host has minted no checkpoint yet, so there is no cadence to observe');
|
|
178
|
+
}
|
|
179
|
+
let prev = 0;
|
|
180
|
+
for (const at of seqs) {
|
|
181
|
+
expect(typeof at === 'number' && Number.isInteger(at), req(CADENCE, 'security-defaults.md §"Audit-log integrity"', 'every checkpoint MUST carry an integer atSequence')).toBe(true);
|
|
182
|
+
const n = at as number;
|
|
183
|
+
expect(n >= prev, req(CADENCE, 'security-defaults.md §"Audit-log integrity"', `checkpoints MUST be listed ascending by atSequence: ${n} follows ${prev}`)).toBe(true);
|
|
184
|
+
expect(n - prev, req(CADENCE, 'security-defaults.md §"Audit-log integrity"',
|
|
185
|
+
`a checkpoint MUST anchor at most checkpointIntervalEntries (${every}) entries: checkpoint at ${n} follows ${prev === 0 ? 'the start of the log' : `the checkpoint at ${prev}`} — a checkpoint is missing from the verify body, or was never minted`,
|
|
186
|
+
)).toBeLessThanOrEqual(every);
|
|
187
|
+
prev = n;
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
});
|
|
@@ -6,12 +6,23 @@
|
|
|
6
6
|
* asserts credentialRefs DO resolve and DO get used (positive test) —
|
|
7
7
|
* with redaction-safe verification via SHA-256 hashing.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* This is a FLOOR scenario (`openwop.floor.byok-roundtrip`), so a host that
|
|
10
|
+
* cannot run it has NOT witnessed the floor. When the canary fixture is not
|
|
11
|
+
* advertised, or the start-run call answers 404/422, every test records
|
|
12
|
+
* `blocked` with the reason — never a trivial pass. A production posture that
|
|
13
|
+
* keeps the `conformance.secret.echo` seam off therefore leaves its claimed
|
|
14
|
+
* `openwop-secrets` profile uncertified, and only that profile: the cut's
|
|
15
|
+
* other claims still certify. Hosts that opt into `OPENWOP_CONFORMANCE_FIXTURES=1`
|
|
12
16
|
* AND pre-provision a secret under `openwop-conformance-canary-secret`
|
|
13
17
|
* expose the surface and the scenarios run end-to-end.
|
|
14
18
|
*
|
|
19
|
+
* Why not `describe.skipIf`: a file whose tests are all skipped WITHOUT a
|
|
20
|
+
* recorded reason is an unclassified return (RFC 0148 §A), and an
|
|
21
|
+
* unclassified FLOOR row in a claimed profile rejects the whole
|
|
22
|
+
* certification, not just that profile. Measured on openwop-app's
|
|
23
|
+
* 2026-09-28 post-deploy major-1 cut: 2,116 rows, `certified: none`, the
|
|
24
|
+
* only unclassified row being this file.
|
|
25
|
+
*
|
|
15
26
|
* The scenarios assert shape + non-empty + redaction, not exact value
|
|
16
27
|
* equality — any host-defined canary value works as long as it's
|
|
17
28
|
* non-empty and the host's `secrets.resolve` returns it intact.
|
|
@@ -34,17 +45,18 @@ const SHA256_HEX_RE = /^[0-9a-f]{64}$/;
|
|
|
34
45
|
|
|
35
46
|
const BYOK_WORKFLOW_ID = 'openwop-smoke-byok-roundtrip';
|
|
36
47
|
const SKIP_NO_FIXTURE = !isFixtureAdvertised(BYOK_WORKFLOW_ID);
|
|
48
|
+
const NO_FIXTURE_REASON = `precondition not met — the \`${BYOK_WORKFLOW_ID}\` fixture is not advertised, so this host cannot witness the BYOK roundtrip floor (the \`conformance.secret.echo\` seam and the \`openwop-conformance-canary-secret\` canary are off)`;
|
|
37
49
|
|
|
38
|
-
describe
|
|
50
|
+
describe('byok: end-to-end credentialRef resolution roundtrip (openwop-byok profile)', () => {
|
|
39
51
|
it('the canary fixture run MUST resolve a host-provisioned secret and emit SHA-256 hex', async () => {
|
|
52
|
+
if (SKIP_NO_FIXTURE) return softSkip('blocked', NO_FIXTURE_REASON);
|
|
40
53
|
const create = await driver.post('/v1/runs', {
|
|
41
54
|
workflowId: 'openwop-smoke-byok-roundtrip',
|
|
42
55
|
});
|
|
43
56
|
|
|
44
|
-
// Fixture absent OR canary not provisioned
|
|
45
|
-
// Scenario passes trivially.
|
|
57
|
+
// Fixture absent OR canary not provisioned: the floor is not witnessed.
|
|
46
58
|
if (create.status === 404 || create.status === 422) {
|
|
47
|
-
return softSkip('
|
|
59
|
+
return softSkip('blocked', 'precondition not met — `create.status === 404 || create.status === 422` returned early (fixture advertised but the run could not start: canary not provisioned or seam unavailable)');
|
|
48
60
|
}
|
|
49
61
|
|
|
50
62
|
expect(create.status, req('openwop.it.byok-roundtrip.the-canary-fixture-run-must-resolve-a-host-provisioned-secret-and-emit-sha-256-h',
|
|
@@ -102,6 +114,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('byok: end-to-end credentialRef resolution roun
|
|
|
102
114
|
});
|
|
103
115
|
|
|
104
116
|
it('BYOK fixture run MUST emit a node.completed event for the resolve step', async () => {
|
|
117
|
+
if (SKIP_NO_FIXTURE) return softSkip('blocked', NO_FIXTURE_REASON);
|
|
105
118
|
const create = await driver.post('/v1/runs', {
|
|
106
119
|
workflowId: 'openwop-smoke-byok-roundtrip',
|
|
107
120
|
});
|
|
@@ -128,6 +141,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('byok: end-to-end credentialRef resolution roun
|
|
|
128
141
|
});
|
|
129
142
|
|
|
130
143
|
it('BYOK fixture run event log MUST NOT echo the resolved secret value (redaction)', async () => {
|
|
144
|
+
if (SKIP_NO_FIXTURE) return softSkip('blocked', NO_FIXTURE_REASON);
|
|
131
145
|
const create = await driver.post('/v1/runs', {
|
|
132
146
|
workflowId: 'openwop-smoke-byok-roundtrip',
|
|
133
147
|
});
|
|
@@ -34,7 +34,11 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
34
34
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
35
35
|
import { req } from '../lib/requirement-ids.js';
|
|
36
36
|
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
37
|
-
import { readDispatchBudgets } from '../lib/mock-ai-budgets.js';
|
|
37
|
+
import { readDispatchBudgets, truncationBudgetVerdict } from '../lib/mock-ai-budgets.js';
|
|
38
|
+
import { holdMockNodes } from '../lib/mock-node-lock.js';
|
|
39
|
+
|
|
40
|
+
// The mock-AI program seam is keyed by node id; files sharing a fixture's node run one at a time.
|
|
41
|
+
holdMockNodes('truncated-structured-call', 'retry-attempted-structured-call');
|
|
38
42
|
|
|
39
43
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
40
44
|
/**
|
|
@@ -159,16 +163,21 @@ describe.skipIf(HTTP_SKIP)('envelope-completion-distinguishes-truncation: trunca
|
|
|
159
163
|
]);
|
|
160
164
|
if (seed.status === 404) return softSkip('blocked', 'precondition not met — `seed.status === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
161
165
|
|
|
162
|
-
await startRunAndRead(TRUNCATED_FIXTURE);
|
|
163
|
-
|
|
164
|
-
|
|
166
|
+
const run = await startRunAndRead(TRUNCATED_FIXTURE);
|
|
167
|
+
if (run === null) return softSkip('blocked', 'precondition not met — the run create or event-log read failed');
|
|
168
|
+
// RFC 0033 §B: MAY retry, and the retry budget SHOULD exceed the first
|
|
169
|
+
// attempt's. Compared through the dispatch-budgets seam; an unmet SHOULD is
|
|
170
|
+
// recorded, not failed (suite 2.42.9; this leg asserted `> 50` as a MUST).
|
|
171
|
+
const v = await truncationBudgetVerdict(NODE_OF[TRUNCATED_FIXTURE] as string);
|
|
172
|
+
if (v.kind === 'skip') return softSkip(v.skip, v.reason);
|
|
173
|
+
if (v.kind === 'unmet') return softSkip('inapplicable', `RFC 0033 §B SHOULD not met — the retry carried maxTokens ${v.retry} after the first attempt carried ${v.first}; recorded, not failed`);
|
|
165
174
|
expect(
|
|
166
|
-
|
|
167
|
-
req('openwop.it.envelope-completion-distinguishes-truncation.truncation-retry-budget-strictly-greater-than-initial-rfc-0033-b-truncationbudge',
|
|
175
|
+
v.retry,
|
|
176
|
+
req('openwop.it.envelope-completion-distinguishes-truncation.truncation-retry-budget-strictly-greater-than-initial-rfc-0033-b-truncationbudge',
|
|
168
177
|
'RFCS/0033-envelope-completion-contract.md §B',
|
|
169
|
-
|
|
178
|
+
`the truncation retry's budget SHOULD exceed the first attempt's (first ${v.first}, retry ${v.retry})`,
|
|
170
179
|
),
|
|
171
|
-
).toBeGreaterThan(
|
|
180
|
+
).toBeGreaterThan(v.first);
|
|
172
181
|
});
|
|
173
182
|
});
|
|
174
183
|
|
|
@@ -117,6 +117,10 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
117
117
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
118
118
|
import { req } from '../lib/requirement-ids.js';
|
|
119
119
|
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
120
|
+
import { holdMockNodes } from '../lib/mock-node-lock.js';
|
|
121
|
+
|
|
122
|
+
// The mock-AI program seam is keyed by node id; files sharing a fixture's node run one at a time.
|
|
123
|
+
holdMockNodes('retry-attempted-structured-call');
|
|
120
124
|
|
|
121
125
|
const FIXTURE = 'conformance-envelope-retry-attempted';
|
|
122
126
|
const NODE_ID = 'retry-attempted-structured-call';
|
|
@@ -22,6 +22,11 @@ import { pollUntilTerminal } from '../lib/polling.js';
|
|
|
22
22
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
23
23
|
import { req } from '../lib/requirement-ids.js';
|
|
24
24
|
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
25
|
+
import { holdMockNodes } from '../lib/mock-node-lock.js';
|
|
26
|
+
import { truncationBudgetVerdict } from '../lib/mock-ai-budgets.js';
|
|
27
|
+
|
|
28
|
+
// The mock-AI program seam is keyed by node id; files sharing a fixture's node run one at a time.
|
|
29
|
+
holdMockNodes('truncated-structured-call');
|
|
25
30
|
|
|
26
31
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
27
32
|
const FIXTURE = 'conformance-envelope-truncated';
|
|
@@ -50,12 +55,6 @@ async function startRunAndRead(): Promise<{ events: RunEvent[]; terminal: unknow
|
|
|
50
55
|
return { events, terminal };
|
|
51
56
|
}
|
|
52
57
|
|
|
53
|
-
async function lastBudget(): Promise<number | null> {
|
|
54
|
-
const res = await driver.get(`/v1/host/sample/test/mock-ai/last-dispatch-budget?nodeId=${encodeURIComponent(NODE_ID)}`);
|
|
55
|
-
if (res.status !== 200) return null;
|
|
56
|
-
return (res.json as { maxTokens?: number | null }).maxTokens ?? null;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
58
|
describe.skipIf(HTTP_SKIP)('envelope-truncated: runtime behavior (RFC 0032 §B.4 + RFC 0033 §B)', () => {
|
|
60
59
|
it('when mock returns stopReason: max_tokens on attempt 1, exactly one envelope.truncated event fires with stopReason: max_tokens', async () => {
|
|
61
60
|
if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!isFixtureAdvertised(FIXTURE)` returned early');
|
|
@@ -112,19 +111,20 @@ describe.skipIf(HTTP_SKIP)('envelope-truncated: runtime behavior (RFC 0032 §B.4
|
|
|
112
111
|
|
|
113
112
|
const result = await startRunAndRead();
|
|
114
113
|
if (result === null) return softSkip('blocked', 'precondition not met — `result === null` returned early (seam, prior step, or fixture unavailable)');
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
const
|
|
120
|
-
if (
|
|
114
|
+
// RFC 0033 §B: the retry budget SHOULD exceed the first attempt's (a MAY retry,
|
|
115
|
+
// a SHOULD increase). Suite 2.42.9 compares the budgets the provider actually
|
|
116
|
+
// received, and records an unmet SHOULD rather than failing it; until then this
|
|
117
|
+
// leg asserted `> 50` (the fixture value) as a MUST.
|
|
118
|
+
const v = await truncationBudgetVerdict(NODE_ID);
|
|
119
|
+
if (v.kind === 'skip') return softSkip(v.skip, v.reason);
|
|
120
|
+
if (v.kind === 'unmet') return softSkip('inapplicable', `RFC 0033 §B SHOULD not met — the retry carried maxTokens ${v.retry} after the first attempt carried ${v.first}; recorded, not failed`);
|
|
121
121
|
expect(
|
|
122
|
-
|
|
123
|
-
req('openwop.it.envelope-truncated.retry-attempt-receives-a-maxtokens-value-strictly-greater-than-the-previous-atte',
|
|
122
|
+
v.retry,
|
|
123
|
+
req('openwop.it.envelope-truncated.retry-attempt-receives-a-maxtokens-value-strictly-greater-than-the-previous-atte',
|
|
124
124
|
'RFCS/0033-envelope-completion-contract.md §B',
|
|
125
|
-
|
|
125
|
+
`the truncation retry's budget SHOULD exceed the first attempt's (first ${v.first}, retry ${v.retry})`,
|
|
126
126
|
),
|
|
127
|
-
).toBeGreaterThan(
|
|
127
|
+
).toBeGreaterThan(v.first);
|
|
128
128
|
});
|
|
129
129
|
|
|
130
130
|
it('run terminates `completed` after the second attempt succeeds', async () => {
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0223 — a rejected approval gate fails closed, and the failure is
|
|
3
|
+
* routable (`spec/v2/core/interrupt.md` §Rejection; suite 2.43.0, target
|
|
4
|
+
* major 2; gated on the `interrupt` family).
|
|
5
|
+
*
|
|
6
|
+
* Legs:
|
|
7
|
+
* 1. `conformance-approval` (one gate, no edges) resolved `reject` ⇒ the run
|
|
8
|
+
* terminates `failed`, and both the snapshot `error.code` and
|
|
9
|
+
* `run.failed.error.code` are `approval_rejected`;
|
|
10
|
+
* 2. the same run's `interrupt.resolved` records `action: "reject"` and
|
|
11
|
+
* `decision: "rejected"` (the `action` half is interrupt.md §Events, RFC
|
|
12
|
+
* 0183; RFC 0223 names both for a reject);
|
|
13
|
+
* 3. the gate's `node.failed` carries `approval_rejected` with
|
|
14
|
+
* `retryable: false`, and `run.failed.failedNodeId` names the gate;
|
|
15
|
+
* 4. `conformance-interrupt-quorum` (requiredApprovals 3, `majority`): one
|
|
16
|
+
* reject leaves the run `waiting-approval`; a second decides the gate and
|
|
17
|
+
* the run terminates `failed` with `approval_rejected`.
|
|
18
|
+
*
|
|
19
|
+
* The legs are separate rows so a host that misses one field fails that row
|
|
20
|
+
* only. The timeout disposition is not tested: no registered fixture carries a
|
|
21
|
+
* gate with a short `timeoutMs`, and the suite cannot make a host's timer fire.
|
|
22
|
+
* The routed case (an `all_complete` / `any_failed` edge) is not tested either:
|
|
23
|
+
* `conformance-approval` has no edges, and widening a fixture every host serves
|
|
24
|
+
* would force a re-registration (fixtures.md §conformance-approval-refine).
|
|
25
|
+
*
|
|
26
|
+
* The quorum leg presents distinct votes on one bearer through the resume
|
|
27
|
+
* value's `voter` member, as `interrupt-quorum-resolution` does at major 1. A
|
|
28
|
+
* host that counts both votes as one principal cannot be driven to a majority
|
|
29
|
+
* this way; that records `blocked`, not a failure.
|
|
30
|
+
*
|
|
31
|
+
* @see spec/v2/core/interrupt.md §Rejection
|
|
32
|
+
* @see spec/v2/errors.json approval_rejected
|
|
33
|
+
* @see RFCS/0223-approval-reject-disposition.md
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import { describe, it, expect } from 'vitest';
|
|
37
|
+
import { driver, type OpenWOPResponse } from '../lib/driver.js';
|
|
38
|
+
import { gateFamily } from '../lib/v2.js';
|
|
39
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
40
|
+
import { readErrorCode } from '../lib/error-envelope.js';
|
|
41
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
42
|
+
import { req } from '../lib/requirement-ids.js';
|
|
43
|
+
|
|
44
|
+
const DOC = 'spec/v2/core/interrupt.md §Rejection (RFC 0223)';
|
|
45
|
+
const R = (slug: string): string => `openwop.requirement.0223.${slug}`;
|
|
46
|
+
const FIXTURE = 'conformance-approval';
|
|
47
|
+
const QUORUM = 'conformance-interrupt-quorum';
|
|
48
|
+
const NODE_ID = 'gate';
|
|
49
|
+
const CODE = 'approval_rejected';
|
|
50
|
+
const TERMINAL = new Set(['completed', 'failed', 'cancelled']);
|
|
51
|
+
|
|
52
|
+
interface Ev { type?: string; nodeId?: string; payload?: Record<string, unknown> }
|
|
53
|
+
|
|
54
|
+
const enc = (id: string): string => encodeURIComponent(id);
|
|
55
|
+
async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> { try { return await fn(); } catch { return null; } }
|
|
56
|
+
|
|
57
|
+
async function snapshot(runId: string): Promise<Record<string, unknown> | null> {
|
|
58
|
+
const res = await http(() => driver.get(`/runs/${enc(runId)}`));
|
|
59
|
+
return res?.status === 200 && res.json && typeof res.json === 'object' ? (res.json as Record<string, unknown>) : null;
|
|
60
|
+
}
|
|
61
|
+
async function waitStatus(runId: string, wanted: ReadonlySet<string>, timeoutMs: number): Promise<Record<string, unknown> | null> {
|
|
62
|
+
const deadline = Date.now() + timeoutMs;
|
|
63
|
+
for (;;) {
|
|
64
|
+
const s = await snapshot(runId);
|
|
65
|
+
if (s !== null && wanted.has(String(s['status']))) return s;
|
|
66
|
+
if (Date.now() > deadline) return s;
|
|
67
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
async function events(runId: string): Promise<Ev[]> {
|
|
71
|
+
const res = await http(() => driver.get(`/runs/${enc(runId)}/events/poll?timeout=1`));
|
|
72
|
+
return ((res?.json as { events?: Ev[] } | null)?.events ?? []);
|
|
73
|
+
}
|
|
74
|
+
const errorCode = (o: unknown): string | undefined => (o as { code?: unknown } | null | undefined)?.code as string | undefined;
|
|
75
|
+
const resolve = (runId: string, resumeValue: unknown): Promise<OpenWOPResponse | null> => http(() => driver.post(`/runs/${enc(runId)}/interrupts/${enc(NODE_ID)}`, { resumeValue }));
|
|
76
|
+
|
|
77
|
+
/** Create a run of `fixture` and wait for it to suspend on its gate. */
|
|
78
|
+
async function suspended(fixture: string): Promise<{ runId: string } | { reason: string }> {
|
|
79
|
+
const res = await http(() => driver.post('/runs', { workflowId: fixture }));
|
|
80
|
+
if (res === null) return { reason: 'POST /runs unreachable (fetch failed)' };
|
|
81
|
+
const runId = (res.json as { runId?: unknown } | null)?.runId;
|
|
82
|
+
if (res.status !== 201 || typeof runId !== 'string') return { reason: `POST /runs (${fixture}) answered ${res.status} ${readErrorCode(res.json) ?? ''}`.trim() };
|
|
83
|
+
const s = await waitStatus(runId, new Set(['waiting-approval', ...TERMINAL]), 10_000);
|
|
84
|
+
if (s?.['status'] !== 'waiting-approval') return { reason: `the ${fixture} run did not suspend on its approval gate (status ${String(s?.['status'])})` };
|
|
85
|
+
return { runId };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** One rejected `conformance-approval` run per file: legs 1–3 read the same run. */
|
|
89
|
+
let rejected: Promise<{ runId: string; terminal: Record<string, unknown> | null; evs: Ev[] } | { reason: string }> | undefined;
|
|
90
|
+
function rejectedRun(): Promise<{ runId: string; terminal: Record<string, unknown> | null; evs: Ev[] } | { reason: string }> {
|
|
91
|
+
rejected ??= (async () => {
|
|
92
|
+
const s = await suspended(FIXTURE);
|
|
93
|
+
if ('reason' in s) return s;
|
|
94
|
+
const res = await resolve(s.runId, { action: 'reject' });
|
|
95
|
+
if (res === null || res.status < 200 || res.status >= 300) return { reason: `the reject resolve answered ${res?.status ?? 'nothing'} ${readErrorCode(res?.json) ?? ''} — interrupt resolution owns that contract` };
|
|
96
|
+
const terminal = await waitStatus(s.runId, TERMINAL, 10_000);
|
|
97
|
+
return { runId: s.runId, terminal, evs: await events(s.runId) };
|
|
98
|
+
})();
|
|
99
|
+
return rejected;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
describe('RFC 0223 — v2-approval-reject-disposition (gated on interrupt + conformance-approval)', () => {
|
|
103
|
+
it('a rejected gate with no routing edge fails the run with approval_rejected', async () => {
|
|
104
|
+
if (!(await gateFamily('interrupt'))) return softSkip('inapplicable', 'interrupt family not advertised (gate recorded under openwop.family.interrupt)');
|
|
105
|
+
const r = await rejectedRun();
|
|
106
|
+
if ('reason' in r) return softSkip('blocked', r.reason);
|
|
107
|
+
const id = R('reject-fails-run');
|
|
108
|
+
expect(r.terminal?.['status'], req(id, DOC, `with no edge admitting a failed source the run MUST terminate failed (got ${String(r.terminal?.['status'])})`)).toBe('failed');
|
|
109
|
+
expect(errorCode(r.terminal?.['error']), req(id, `${DOC}; runs.md §Snapshot`, 'the terminal snapshot error.code MUST be approval_rejected')).toBe(CODE);
|
|
110
|
+
const failed = r.evs.find((e) => e.type === 'run.failed');
|
|
111
|
+
expect(errorCode(failed?.payload?.['error']), req(id, DOC, 'run.failed.error.code MUST be approval_rejected')).toBe(CODE);
|
|
112
|
+
}, 45_000);
|
|
113
|
+
|
|
114
|
+
it('interrupt.resolved records action reject and decision rejected', async () => {
|
|
115
|
+
if (!(await gateFamily('interrupt'))) return softSkip('inapplicable', 'interrupt family not advertised (gate recorded under openwop.family.interrupt)');
|
|
116
|
+
const r = await rejectedRun();
|
|
117
|
+
if ('reason' in r) return softSkip('blocked', r.reason);
|
|
118
|
+
const id = R('reject-recorded');
|
|
119
|
+
const resolved = r.evs.find((e) => e.type === 'interrupt.resolved');
|
|
120
|
+
expect(resolved, req(id, `${DOC}; interrupt.md §Events`, 'the reject MUST be recorded as interrupt.resolved')).toBeDefined();
|
|
121
|
+
expect(
|
|
122
|
+
{ action: resolved?.payload?.['action'], decision: resolved?.payload?.['decision'] },
|
|
123
|
+
req(id, `${DOC}; interrupt.md §Events (RFC 0183)`, 'interrupt.resolved MUST carry action "reject" and decision "rejected"'),
|
|
124
|
+
).toEqual({ action: 'reject', decision: 'rejected' });
|
|
125
|
+
}, 45_000);
|
|
126
|
+
|
|
127
|
+
it('the gate node fails with approval_rejected, not retryable, and run.failed names it', async () => {
|
|
128
|
+
if (!(await gateFamily('interrupt'))) return softSkip('inapplicable', 'interrupt family not advertised (gate recorded under openwop.family.interrupt)');
|
|
129
|
+
const r = await rejectedRun();
|
|
130
|
+
if ('reason' in r) return softSkip('blocked', r.reason);
|
|
131
|
+
const id = R('reject-fails-node');
|
|
132
|
+
const nodeFailed = r.evs.find((e) => e.type === 'node.failed' && (e.payload?.['nodeId'] ?? e.nodeId) === NODE_ID);
|
|
133
|
+
expect(nodeFailed, req(id, DOC, 'the gate that does not turn the rejection into an output MUST emit node.failed')).toBeDefined();
|
|
134
|
+
const err = nodeFailed?.payload?.['error'] as { code?: unknown; retryable?: unknown } | undefined;
|
|
135
|
+
expect({ code: err?.code, retryable: err?.retryable }, req(id, DOC, 'node.failed.error MUST carry code approval_rejected and retryable false')).toEqual({ code: CODE, retryable: false });
|
|
136
|
+
const runFailed = r.evs.find((e) => e.type === 'run.failed');
|
|
137
|
+
expect(runFailed?.payload?.['failedNodeId'], req(id, DOC, 'run.failed.failedNodeId MUST name the gate')).toBe(NODE_ID);
|
|
138
|
+
}, 45_000);
|
|
139
|
+
|
|
140
|
+
it('a majority reject decides the quorum gate on the deciding vote and fails the run with approval_rejected', async () => {
|
|
141
|
+
if (!(await gateFamily('interrupt'))) return softSkip('inapplicable', 'interrupt family not advertised (gate recorded under openwop.family.interrupt)');
|
|
142
|
+
if (!isFixtureAdvertised(QUORUM)) return softSkip('inapplicable', `fixture ${QUORUM} is not advertised — the host does not claim the quorum fixture`);
|
|
143
|
+
const s = await suspended(QUORUM);
|
|
144
|
+
if ('reason' in s) return softSkip('blocked', s.reason);
|
|
145
|
+
const id = R('quorum-reject-fails-run');
|
|
146
|
+
const first = await resolve(s.runId, { action: 'reject', voter: 'approver-1' });
|
|
147
|
+
if (first === null || first.status < 200 || first.status >= 300) return softSkip('blocked', `the first reject vote answered ${first?.status ?? 'nothing'} ${readErrorCode(first?.json) ?? ''}`);
|
|
148
|
+
const between = (await snapshot(s.runId))?.['status'];
|
|
149
|
+
const resolvedEarly = (await events(s.runId)).some((e) => e.type === 'interrupt.resolved');
|
|
150
|
+
const second = await resolve(s.runId, { action: 'reject', voter: 'approver-2' });
|
|
151
|
+
if (second === null || second.status < 200 || second.status >= 300) return softSkip('blocked', `the second reject vote answered ${second?.status ?? 'nothing'} ${readErrorCode(second?.json) ?? ''}`);
|
|
152
|
+
const terminal = await waitStatus(s.runId, TERMINAL, 10_000);
|
|
153
|
+
if (between === 'waiting-approval' && terminal?.['status'] === 'waiting-approval') {
|
|
154
|
+
await http(() => driver.post(`/runs/${enc(s.runId)}/cancel`, {}));
|
|
155
|
+
return softSkip('blocked', 'the host counted both votes as one approver (one bearer; the resume value\'s voter was not honoured), so a majority could not be caused');
|
|
156
|
+
}
|
|
157
|
+
expect(between, req(id, DOC, 'one reject of three under majority does not decide the gate: the run MUST stay waiting-approval')).toBe('waiting-approval');
|
|
158
|
+
expect(resolvedEarly, req(id, DOC, 'a vote that does not decide the gate MUST NOT emit interrupt.resolved')).toBe(false);
|
|
159
|
+
expect(terminal?.['status'], req(id, DOC, `two rejects of three under majority MUST resolve the gate rejected and fail the run (got ${String(terminal?.['status'])})`)).toBe('failed');
|
|
160
|
+
expect(errorCode(terminal?.['error']), req(id, DOC, 'the terminal snapshot error.code MUST be approval_rejected')).toBe(CODE);
|
|
161
|
+
}, 60_000);
|
|
162
|
+
});
|