@openwop/openwop-conformance 2.42.6 → 2.42.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +3 -3
- package/coverage.md +1 -1
- package/dist/lib/soft-skip.js +13 -0
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures/conformance-envelope-retry-attempted.json +2 -1
- package/package.json +2 -2
- package/requirements.json +581 -175
- package/scenario-majors.json +22 -3
- package/schemas/CORPUS-STAMP.json +70 -67
- package/src/lib/audit-checkpoint.ts +72 -0
- package/src/lib/ext-claims.ts +96 -0
- package/src/lib/mock-ai-budgets.ts +40 -0
- package/src/lib/soft-skip.ts +14 -0
- package/src/scenarios/a2a-peer-authority.test.ts +6 -13
- package/src/scenarios/a2a-task-roundtrip.test.ts +6 -34
- package/src/scenarios/a2a-version-negotiation.test.ts +6 -12
- package/src/scenarios/a2ui-surface-delta-transport.test.ts +23 -12
- package/src/scenarios/agent-channel-dispatch.test.ts +13 -4
- package/src/scenarios/agent-live-invocation-bracket.test.ts +1 -1
- package/src/scenarios/agent-manifest-runtime.test.ts +3 -0
- package/src/scenarios/aiEnvelope.capBreached.test.ts +2 -2
- package/src/scenarios/aiEnvelope.contractRefusal.test.ts +2 -2
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +2 -2
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +2 -2
- package/src/scenarios/aiproviders-selfhosted-honesty.test.ts +2 -2
- package/src/scenarios/append-ordering.test.ts +26 -21
- package/src/scenarios/artifact-auth.test.ts +3 -1
- package/src/scenarios/artifact-type-registration-source.test.ts +2 -2
- package/src/scenarios/audit-checkpoint-signature.test.ts +63 -0
- package/src/scenarios/audit-checkpoint-vectors.test.ts +93 -0
- package/src/scenarios/audit-log-integrity.test.ts +7 -16
- package/src/scenarios/auth-mtls.test.ts +7 -6
- package/src/scenarios/auth-subject-link-key-class.test.ts +3 -3
- package/src/scenarios/auth-subject-link.test.ts +19 -18
- package/src/scenarios/blob-presign-expiry.test.ts +2 -2
- package/src/scenarios/byok-roundtrip.test.ts +4 -1
- package/src/scenarios/compensation-behavior.test.ts +10 -5
- package/src/scenarios/conformance-advertised-seam-required.test.ts +6 -0
- package/src/scenarios/connection-pack-apihosts.test.ts +2 -2
- package/src/scenarios/connection-provider-resolution.test.ts +2 -2
- package/src/scenarios/context-budget-transcript-bound.test.ts +7 -3
- package/src/scenarios/context-summarization-replay.test.ts +2 -0
- package/src/scenarios/cost-attribution.test.ts +3 -1
- package/src/scenarios/debugBundle.test.ts +12 -4
- package/src/scenarios/discovery.test.ts +6 -2
- package/src/scenarios/dispatch-cross-worker-handoff.test.ts +2 -2
- package/src/scenarios/dispatch-input-mapping.test.ts +2 -2
- package/src/scenarios/dispatch-output-mapping.test.ts +15 -2
- package/src/scenarios/distillation-token-budget.test.ts +2 -2
- package/src/scenarios/durability-poison-exhaustion.test.ts +2 -2
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +51 -15
- package/src/scenarios/envelope-nl-to-format-engaged.test.ts +2 -2
- package/src/scenarios/envelope-recovery-applied.test.ts +2 -2
- package/src/scenarios/envelope-refusal-shape.test.ts +2 -2
- package/src/scenarios/envelope-retry-attempted.test.ts +2 -2
- package/src/scenarios/envelope-retry-exhausted.test.ts +2 -2
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +1 -1
- package/src/scenarios/envelope-truncation-cap-exhaustion.test.ts +2 -2
- package/src/scenarios/eventOrdering.test.ts +17 -10
- package/src/scenarios/form-content-instantiation.test.ts +2 -2
- package/src/scenarios/heartbeat-idempotent-no-spam.test.ts +2 -2
- package/src/scenarios/highConcurrency.test.ts +9 -9
- package/src/scenarios/idempotency-concurrent-claim.test.ts +5 -5
- package/src/scenarios/idempotency-key-determinism.test.ts +8 -8
- package/src/scenarios/idempotency.test.ts +6 -6
- package/src/scenarios/idempotencyRetry.test.ts +16 -11
- package/src/scenarios/interrupt-token-matrix.test.ts +24 -11
- package/src/scenarios/mcp-cache-tenant-scope.test.ts +3 -3
- package/src/scenarios/mcp-current-auth-boundary.test.ts +9 -6
- package/src/scenarios/mcp-extension-opacity.test.ts +6 -5
- package/src/scenarios/mcp-mrtr-roundtrip.test.ts +9 -8
- package/src/scenarios/mcp-version-negotiation.test.ts +5 -10
- package/src/scenarios/media-url-inline-cap.test.ts +3 -2
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +2 -0
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +2 -2
- package/src/scenarios/memory-consolidation-idempotent.test.ts +2 -2
- package/src/scenarios/memory-injection-budget.test.ts +2 -2
- package/src/scenarios/multi-party-conversation-behavioral.test.ts +2 -2
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +9 -9
- package/src/scenarios/multi-region-idempotency.test.ts +2 -2
- package/src/scenarios/pack-registry-isolation.test.ts +13 -12
- package/src/scenarios/pack-registry.test.ts +3 -2
- package/src/scenarios/pause-resume.test.ts +30 -11
- package/src/scenarios/prompt-read-workspace-membership-enforced.test.ts +43 -34
- package/src/scenarios/provider-usage.test.ts +2 -2
- package/src/scenarios/redaction.test.ts +7 -5
- package/src/scenarios/redactionAdversarial.test.ts +5 -4
- package/src/scenarios/registry-public.test.ts +5 -1
- package/src/scenarios/replay-divergence-at-refusal.test.ts +10 -7
- package/src/scenarios/replay-fanout-suppression.test.ts +7 -4
- package/src/scenarios/replay-fork-arbitrary.test.ts +2 -2
- package/src/scenarios/replay-side-effect-suppression.test.ts +2 -2
- package/src/scenarios/replayDeterminism.test.ts +9 -4
- package/src/scenarios/runner-ledger.test.ts +4 -0
- package/src/scenarios/runtime-capabilities.test.ts +2 -0
- package/src/scenarios/speech-synthesis-roundtrip.test.ts +1 -1
- package/src/scenarios/streamReconnect.test.ts +2 -2
- package/src/scenarios/subrun-approval-gate.test.ts +2 -2
- package/src/scenarios/subworkflow-input-mapping.test.ts +2 -2
- package/src/scenarios/tool-catalog-projection.test.ts +10 -7
- package/src/scenarios/tool-session-lifecycle.test.ts +1 -1
- package/src/scenarios/trigger-ingestion.test.ts +2 -2
- package/src/scenarios/v2-a2ui-v09-surface.test.ts +15 -0
- package/src/scenarios/v2-client-version-header.test.ts +124 -0
- package/src/scenarios/v2-effect-identity-business-key.test.ts +12 -0
- package/src/scenarios/v2-ext-family-claims.test.ts +62 -0
- package/src/scenarios/v2-ext-rest-transport.test.ts +151 -0
- package/src/scenarios/v2-id-grammar.test.ts +17 -5
- package/src/scenarios/v2-idempotency-in-flight.test.ts +3 -0
- package/src/scenarios/v2-legacy-profiles-absent.test.ts +4 -6
- package/src/scenarios/v2-mcp-tasks.test.ts +1 -1
- package/src/scenarios/v2-min-client-version.test.ts +20 -14
- package/src/scenarios/v2-preferred-version-default.test.ts +19 -1
- package/src/scenarios/v2-refresh-sla.test.ts +6 -5
- package/src/scenarios/v2-sse-last-event-id-cursor.test.ts +3 -3
- package/src/scenarios/v2-sse-last-event-id.test.ts +2 -0
- package/src/scenarios/v2-version-header-honored.test.ts +3 -0
- package/src/scenarios/v2-webhook-durable-delivery.test.ts +2 -0
- package/src/scenarios/v2-webhook-generated-secret.test.ts +76 -0
- package/src/scenarios/v2-webhook-secret-rotation.test.ts +8 -1
- package/src/scenarios/verifier-gating.test.ts +2 -2
- package/src/scenarios/version-fold.test.ts +2 -2
- package/src/scenarios/wasm-pack-invoke-suspended.test.ts +7 -6
- package/src/scenarios/webhook-signed-delivery.test.ts +2 -0
- package/src/scenarios/workflow-chain-host-expansion.test.ts +10 -1
- package/src/scenarios/workflow-variable-format.test.ts +2 -2
- package/src/scenarios/workload-identity-chain-bounds.test.ts +6 -3
- package/vectors/audit-checkpoint-v1.json +260 -0
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
|
|
26
26
|
import { randomBytes } from 'node:crypto';
|
|
27
27
|
import { describe, it, expect } from 'vitest';
|
|
28
|
-
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
28
|
+
import { softSkip, seamAbsent, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
29
29
|
import { driver } from '../lib/driver.js';
|
|
30
30
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
31
31
|
import { req } from '../lib/requirement-ids.js';
|
|
@@ -139,7 +139,7 @@ describe('auth-subject-link: cross-lane deactivation (RFC 0159 §A.3 — opt-in)
|
|
|
139
139
|
// 2. A valid SAML assertion whose persistent NameID equals the externalId
|
|
140
140
|
// links to the same subject and authenticates.
|
|
141
141
|
const before = await driver.post('/v1/host/sample/auth/saml/validate', { idpUrl, variant: 'valid', nameId: externalId });
|
|
142
|
-
if (before.status === 404) return
|
|
142
|
+
if (before.status === 404) return blockedDespiteAssertions('SAML validate seam answered 404 after SCIM provisioning succeeded — the deactivation fail-close is unobserved');
|
|
143
143
|
expect(
|
|
144
144
|
(before.json as { authenticated?: boolean } | undefined)?.authenticated,
|
|
145
145
|
req('openwop.it.auth-subject-link.a-scim-deactivation-fail-closes-the-linked-saml-identity', 'auth-profiles.md §Subject linking', 'a valid linked SAML assertion authenticates before deactivation'),
|
|
@@ -189,23 +189,24 @@ describe('auth-subject-link: link-key hygiene (RFC 0159 §A.2 — opt-in)', () =
|
|
|
189
189
|
link.status,
|
|
190
190
|
req('openwop.it.auth-subject-link.a-mutable-pii-link-key-email-never-produces-a-cross-lane-pass', 'auth-profiles.md §Subject linking', 'RFC 0159 §A.2: a link on a mutable/PII key (email) MUST be rejected'),
|
|
191
191
|
).toBeGreaterThanOrEqual(400);
|
|
192
|
-
|
|
192
|
+
// A 4xx on the mutable-key link IS the §A.2 conformant answer: the requirement was
|
|
193
|
+
// observed. (This branch used to end in a blocked note, a partial-witness pass.)
|
|
194
|
+
} else {
|
|
195
|
+
// If the host accepted the request, it MUST NOT have formed a cross-lane
|
|
196
|
+
// link on email: a deactivation via the email "link" MUST NOT deny (or
|
|
197
|
+
// otherwise act on) an unrelated SAML subject, and MUST NOT let a mutable
|
|
198
|
+
// key authorize one. Probe: deactivate by email, then a SAML assertion for
|
|
199
|
+
// a DIFFERENT opaque subject MUST be unaffected by the email operation.
|
|
200
|
+
await driver.post('/v1/host/sample/auth/scim/provision', { scimUrl, op: 'deactivate-user', email: 'r.smith@example.test' });
|
|
201
|
+
const other = await driver.post('/v1/host/sample/auth/saml/validate', { idpUrl, variant: 'valid', nameId: 'idp-op-DIFFERENT' });
|
|
202
|
+
expect(
|
|
203
|
+
(other.json as { linkedDenied?: boolean } | undefined)?.linkedDenied === true,
|
|
204
|
+
req('openwop.it.auth-subject-link.a-mutable-pii-link-key-email-never-produces-a-cross-lane-pass',
|
|
205
|
+
'auth-profiles.md §Subject linking',
|
|
206
|
+
'RFC 0159 §A.2: a mutable-key (email) operation MUST NOT drive a cross-lane deny on any opaque subject',
|
|
207
|
+
),
|
|
208
|
+
).toBe(false);
|
|
193
209
|
}
|
|
194
|
-
|
|
195
|
-
// If the host accepted the request, it MUST NOT have formed a cross-lane
|
|
196
|
-
// link on email: a deactivation via the email "link" MUST NOT deny (or
|
|
197
|
-
// otherwise act on) an unrelated SAML subject, and MUST NOT let a mutable
|
|
198
|
-
// key authorize one. Probe: deactivate by email, then a SAML assertion for
|
|
199
|
-
// a DIFFERENT opaque subject MUST be unaffected by the email operation.
|
|
200
|
-
await driver.post('/v1/host/sample/auth/scim/provision', { scimUrl, op: 'deactivate-user', email: 'r.smith@example.test' });
|
|
201
|
-
const other = await driver.post('/v1/host/sample/auth/saml/validate', { idpUrl, variant: 'valid', nameId: 'idp-op-DIFFERENT' });
|
|
202
|
-
expect(
|
|
203
|
-
(other.json as { linkedDenied?: boolean } | undefined)?.linkedDenied === true,
|
|
204
|
-
req('openwop.it.auth-subject-link.a-mutable-pii-link-key-email-never-produces-a-cross-lane-pass',
|
|
205
|
-
'auth-profiles.md §Subject linking',
|
|
206
|
-
'RFC 0159 §A.2: a mutable-key (email) operation MUST NOT drive a cross-lane deny on any opaque subject',
|
|
207
|
-
),
|
|
208
|
-
).toBe(false);
|
|
209
210
|
});
|
|
210
211
|
});
|
|
211
212
|
|
|
@@ -17,7 +17,7 @@ import { describe, it, expect } from 'vitest';
|
|
|
17
17
|
import { driver } from '../lib/driver.js';
|
|
18
18
|
import { discoveryFamilies } from '../lib/discovery-capabilities.js';
|
|
19
19
|
import { req } from '../lib/requirement-ids.js';
|
|
20
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
20
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
21
21
|
|
|
22
22
|
interface DiscoveryDoc {
|
|
23
23
|
capabilities?: Record<string, unknown>;
|
|
@@ -84,7 +84,7 @@ describe('blob-presign-expiry: behavioral (RFC 0019 §B point 1)', () => {
|
|
|
84
84
|
|
|
85
85
|
// Fetch within the window — MUST return 200 + the bytes
|
|
86
86
|
const within = await driver.get(body.url!);
|
|
87
|
-
if (within.status === 404) return
|
|
87
|
+
if (within.status === 404) return blockedDespiteAssertions('the presigned URL answered 404 inside its TTL — neither the in-window resolve nor the post-expiry 403 is observable');
|
|
88
88
|
expect(
|
|
89
89
|
within.status,
|
|
90
90
|
req('openwop.it.blob-presign-expiry.presigned-url-must-resolve-to-the-blob-inside-its-ttl-window-and-return-403-afte', 'RFC 0019 §B point 1', 'presigned URL MUST resolve to 200 within its TTL window'),
|
|
@@ -81,7 +81,10 @@ describe.skipIf(SKIP_NO_FIXTURE)('byok: end-to-end credentialRef resolution roun
|
|
|
81
81
|
// Host doesn't expose node outputs in variables/outputs map —
|
|
82
82
|
// some hosts only expose them on the events stream. Skip the
|
|
83
83
|
// shape check; the run-completed assertion above is sufficient.
|
|
84
|
-
|
|
84
|
+
// partial-witness-ok: the run reached `completed`, which is the secret-resolution witness
|
|
85
|
+
// (a failed resolve fails the run); surfacing node outputs on variables/outputs is a MAY,
|
|
86
|
+
// so the SHA-256 shape check is an optional extra.
|
|
87
|
+
return softSkip('inapplicable', 'the host does not surface node outputs on variables/outputs (a MAY) — the SHA-256 shape check is not run; the completed run was asserted');
|
|
85
88
|
}
|
|
86
89
|
|
|
87
90
|
if ('secretSha256' in candidate && typeof candidate.secretSha256 === 'string') {
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
*/
|
|
41
41
|
|
|
42
42
|
import { describe, it, expect } from 'vitest';
|
|
43
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
43
|
+
import { blockedDespiteAssertions, softSkip } from '../lib/soft-skip.js';
|
|
44
44
|
import { driver } from '../lib/driver.js';
|
|
45
45
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
46
46
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
@@ -101,7 +101,7 @@ describe('RFC 0151 §C — compensation lifecycle (capability-gated behavior)',
|
|
|
101
101
|
'cannot be certified (RFC 0148 §A: unobservable resolves to `blocked`, not to a pass)',
|
|
102
102
|
),
|
|
103
103
|
).not.toBe(404);
|
|
104
|
-
|
|
104
|
+
throw new Error('unreachable: the assertion above fails first');
|
|
105
105
|
}
|
|
106
106
|
const events = (seam.json as { events?: { type: string }[] }).events ?? [];
|
|
107
107
|
const requestedAt = events.findIndex((e) => e.type === 'compensation.requested');
|
|
@@ -194,7 +194,10 @@ describe('RFC 0151 §C — compensation lifecycle (capability-gated behavior)',
|
|
|
194
194
|
`— got ${String(status)}`,
|
|
195
195
|
),
|
|
196
196
|
).toBe(true);
|
|
197
|
-
|
|
197
|
+
// partial-witness-ok: an unwind that started and did not complete folds to
|
|
198
|
+
// one of running | partial | failed, and the assertion above observed that;
|
|
199
|
+
// which of the three is host state the seam's event list cannot decide.
|
|
200
|
+
return softSkip('blocked', 'the unwind started and did not complete: the rollup was asserted to be running | partial | failed, but which one is not decidable from the seam events');
|
|
198
201
|
}
|
|
199
202
|
expect(
|
|
200
203
|
status,
|
|
@@ -267,8 +270,10 @@ describe('RFC 0151 §C — compensation lifecycle (capability-gated behavior)',
|
|
|
267
270
|
// that legitimately has a plan.
|
|
268
271
|
const types = (body.events ?? []).map((e) => e.type);
|
|
269
272
|
if (types.includes('compensation.requested')) {
|
|
270
|
-
|
|
271
|
-
|
|
273
|
+
// host-sample-test-seams.md §21: under `fail: false` the host MUST record no
|
|
274
|
+
// compensation.requested, and a host that fails the node anyway leaves this
|
|
275
|
+
// leg `blocked`, not passed — the runId assertion above is setup.
|
|
276
|
+
return blockedDespiteAssertions(
|
|
272
277
|
`the seam emitted ${JSON.stringify(types)} under \`fail: false\` — a trigger fired, so this run ` +
|
|
273
278
|
'is not the healthy case the leg needs',
|
|
274
279
|
);
|
|
@@ -97,14 +97,20 @@ describe('RFC 0148 §B — conformance-advertised-seam-required', () => {
|
|
|
97
97
|
it('strict mode: seamAbsent (advertised, seam answered 404/403 mid-scenario) FAILS; default mode notes blocked with the reason', () => {
|
|
98
98
|
process.env['OPENWOP_REQUIRE_BEHAVIOR'] = 'true';
|
|
99
99
|
__resetEnvCacheForTests();
|
|
100
|
+
// partial-witness-ok: seamAbsent is the subject under test in this server-free unit test of
|
|
101
|
+
// the helper, not an early return.
|
|
100
102
|
expect(() => seamAbsent('host advertises X but /v1/host/sample/x answered 404'), req('openwop.it.conformance-advertised-seam-required.strict-mode-seamabsent-advertised-seam-answered-404-403-mid-scenario-fails-defau', 'RFC 0148 §B', 'strict mode: seamAbsent (advertised, seam answered 404/403 mid-scenario) FAILS; default mode notes blocked with the reason')).toThrow(/RFC 0148 §B/);
|
|
101
103
|
delete process.env['OPENWOP_REQUIRE_BEHAVIOR'];
|
|
102
104
|
__resetEnvCacheForTests();
|
|
103
105
|
resetSoftSkips();
|
|
106
|
+
// partial-witness-ok: seamAbsent is the subject under test in this server-free unit test of
|
|
107
|
+
// the helper, not an early return.
|
|
104
108
|
expect(seamAbsent('host advertises X but /v1/host/sample/x answered 403')).toBeUndefined();
|
|
105
109
|
const noted = softSkipDisposition('conformance-advertised-seam-required.test.ts');
|
|
106
110
|
expect(noted?.kind).toBe('blocked');
|
|
107
111
|
expect(noted?.reason).toContain('answered 403');
|
|
112
|
+
// The note above is this test's subject, not a disposition of the leg.
|
|
113
|
+
resetSoftSkips();
|
|
108
114
|
});
|
|
109
115
|
|
|
110
116
|
it('the sanctioned escape is an explicit opt-out, which records skipped with the reason — still never a pass', () => {
|
|
@@ -48,7 +48,7 @@ import { driver } from '../lib/driver.js';
|
|
|
48
48
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
49
49
|
import { readCapabilityFamily } from '../lib/discovery-capabilities.js';
|
|
50
50
|
import { req } from '../lib/requirement-ids.js';
|
|
51
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
51
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
52
52
|
|
|
53
53
|
const SCHEMA_PATH = join(SCHEMAS_DIR, 'connection-pack-manifest.schema.json');
|
|
54
54
|
const APIHOSTS_FIXTURE = join(FIXTURES_DIR, 'connection-packs', 'connection-pack-apihosts-valid.json');
|
|
@@ -188,7 +188,7 @@ describe('connection-pack apiHosts — behavioral egress allow-list (RFC 0120 it
|
|
|
188
188
|
|
|
189
189
|
for (const evil of ['evil.com', 'notfacebook.com', 'facebook.com.evil.com']) {
|
|
190
190
|
const denied = await probe(evil);
|
|
191
|
-
if (denied === undefined) return
|
|
191
|
+
if (denied === undefined) return blockedDespiteAssertions('the egress-check seam stopped answering on a non-matching host — fail-closed egress is unobserved');
|
|
192
192
|
expect(
|
|
193
193
|
denied.allowed,
|
|
194
194
|
req('openwop.it.connection-pack-apihosts.permits-a-credential-bearing-egress-to-an-apihosts-match-fails-closed-otherwise', 'connection-packs.md §Manifest item 10', `egress to "${evil}" (no apiHosts match) MUST fail closed — no credential sent`),
|
|
@@ -35,7 +35,7 @@ import { driver } from '../lib/driver.js';
|
|
|
35
35
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
36
36
|
import { readCapabilityFamily } from '../lib/discovery-capabilities.js';
|
|
37
37
|
import { req } from '../lib/requirement-ids.js';
|
|
38
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
38
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
39
39
|
|
|
40
40
|
const FIXTURE_PATH = join(FIXTURES_DIR, 'connection-packs', 'connection-pack-github.json');
|
|
41
41
|
|
|
@@ -117,7 +117,7 @@ describe('connection-provider-resolution (RFC 0095 §B.6/§B.8)', () => {
|
|
|
117
117
|
provider: 'conformance-prerelease-probe',
|
|
118
118
|
simulateBuiltinVersion: '1.0.0',
|
|
119
119
|
});
|
|
120
|
-
if (res.status === 404 || res.status === 403) return
|
|
120
|
+
if (res.status === 404 || res.status === 403) return blockedDespiteAssertions('the resolve seam (simulateBuiltinVersion) answered 404/403 — the prerelease-vs-release conflict is unobserved');
|
|
121
121
|
const body = res.json as ResolveResult | undefined;
|
|
122
122
|
expect(
|
|
123
123
|
body?.code,
|
|
@@ -104,13 +104,13 @@ describe('context-budget-transcript-bound (RFC 0111 §"Context economy")', () =>
|
|
|
104
104
|
if (typeof budget !== 'number') return softSkip('inapplicable', 'transcriptTokenBudget not advertised');
|
|
105
105
|
if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', `the live fixture ${FIXTURE} is not advertised — the scripted multiturn fixture drives a mock supervisor, which RFC 0111 §Scope forbids from advertising contextBudget`);
|
|
106
106
|
expect(typeof advertisedCounter === 'string', req(ID, 'RFC 0111', 'tokenCounter MUST be advertised when transcriptTokenBudget is present (schema if/then)')).toBe(true);
|
|
107
|
-
if (typeof advertisedCounter !== 'string')
|
|
107
|
+
if (typeof advertisedCounter !== 'string') throw new Error('unreachable: the assertion above fails first');
|
|
108
108
|
|
|
109
109
|
const create = await driver.post(runsPath(), { workflowId: FIXTURE });
|
|
110
110
|
expect(create.status, req(ID, 'RFC 0111', `POST ${runsPath()} MUST create the live-fixture run`)).toBe(201);
|
|
111
111
|
const runId = runIdOf(create.json);
|
|
112
112
|
expect(runId, req(ID, 'RFC 0111', 'the create response MUST carry a runId')).toBeDefined();
|
|
113
|
-
if (runId === undefined)
|
|
113
|
+
if (runId === undefined) throw new Error('unreachable: the assertion above fails first');
|
|
114
114
|
await pollUntilTerminal(runId, { timeoutMs: LIVE_RUN_POLL_MS });
|
|
115
115
|
|
|
116
116
|
const windows: Array<{ iteration: number; window: TranscriptWindow }> = [];
|
|
@@ -121,6 +121,10 @@ describe('context-budget-transcript-bound (RFC 0111 §"Context economy")', () =>
|
|
|
121
121
|
// `softSkip('blocked')` after the create/runId asserts, which records a
|
|
122
122
|
// partial-witness PASS — a host with no transcript-window seam passed
|
|
123
123
|
// the transcript bound without a single window being read.
|
|
124
|
+
// partial-witness-ok: not a partial witness at either major — seamAbsent
|
|
125
|
+
// runs only when major === 2, where a blocked note after assertions
|
|
126
|
+
// stands as `blocked` (2.35.0) and strict mode fails it; major 1 takes
|
|
127
|
+
// blockedDespiteAssertions.
|
|
124
128
|
if (iteration === 1) return major === 2 ? seamAbsent(`contextBudget is advertised but the transcript-window seam answered ${res.status} (host-sample-test-seams.md §14)`) : blockedDespiteAssertions(`the transcript-window seam answered ${res.status} (host-sample-test-seams.md §14)`);
|
|
125
129
|
break;
|
|
126
130
|
}
|
|
@@ -128,7 +132,7 @@ describe('context-budget-transcript-bound (RFC 0111 §"Context economy")', () =>
|
|
|
128
132
|
expect(res.status, req(ID, 'host-sample-test-seams.md §14', `iteration ${iteration}: the transcript-window seam MUST return 200 for a valid iteration`)).toBe(200);
|
|
129
133
|
const window = parseTranscriptWindow(res.json);
|
|
130
134
|
expect(window, req(ID, 'host-sample-test-seams.md §14', `iteration ${iteration}: the seam MUST return { tokenCounter, tokenCount, eventIds, summarizedRanges, entries? } with a well-formed entries[] when present`)).toBeDefined();
|
|
131
|
-
if (window === undefined)
|
|
135
|
+
if (window === undefined) throw new Error('unreachable: the assertion above fails first');
|
|
132
136
|
windows.push({ iteration, window });
|
|
133
137
|
}
|
|
134
138
|
expect(windows.length, req(ID, 'host-sample-test-seams.md §14', 'a wired transcript-window seam MUST report at least one orchestrator iteration')).toBeGreaterThan(0);
|
|
@@ -159,6 +159,8 @@ describe('context-summarization-replay (RFC 0111 §"Replay determinism")', () =>
|
|
|
159
159
|
// The model-facing half: the summary TEXT the host fed on the fork.
|
|
160
160
|
const sourceTexts = await fedSummaryTexts(sourceRunId, toLog(sourceQ.events));
|
|
161
161
|
const forkTexts = await fedSummaryTexts(forkRunId, toLog(forkQ.events));
|
|
162
|
+
// partial-witness-ok: summaryRef reuse, the replay MUST, was asserted above; the summary-
|
|
163
|
+
// text comparison needs the transcript-window seam's entries[], an optional extra.
|
|
162
164
|
if (sourceTexts === null || forkTexts === null) return softSkip('inapplicable', 'the transcript-window seam serves no entries[] for these runs, so the model-facing summary text was not compared (summaryRef reuse was)');
|
|
163
165
|
expect(sourceTexts.length, req(ID, 'RFC 0111 §"Replay determinism"', 'the source run summarized, so its transcript windows MUST carry the summary text it fed')).toBeGreaterThan(0);
|
|
164
166
|
expect(forkTexts, req(ID, 'RFC 0111 §"Replay determinism"', 'the replay MUST feed the model the recorded summary text, byte for byte — never a re-summarization')).toEqual(sourceTexts);
|
|
@@ -93,7 +93,9 @@ describe.skipIf(SKIP_NO_NOOP)('cost-attribution: metrics.openwopCost forward-com
|
|
|
93
93
|
// passes trivially; don't force a value on a workflow that produces
|
|
94
94
|
// no cost.
|
|
95
95
|
expect(openwopCost).toBeUndefined();
|
|
96
|
-
|
|
96
|
+
// partial-witness-ok: an IF-present clause. The noop fixture records no cost and absence
|
|
97
|
+
// is spec-allowed, so there is no openwopCost shape to bind; the run itself completed.
|
|
98
|
+
return softSkip('inapplicable', 'metrics.openwopCost is absent on the noop run (spec-allowed) — no shape to check');
|
|
97
99
|
}
|
|
98
100
|
|
|
99
101
|
// When present, validate the canonical shape per
|
|
@@ -43,7 +43,7 @@ import { pollUntilTerminal } from '../lib/polling.js';
|
|
|
43
43
|
import { CANARY_MARKER, getCanary } from '../lib/canaries.js';
|
|
44
44
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
45
45
|
import { req } from '../lib/requirement-ids.js';
|
|
46
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
46
|
+
import { blockedDespiteAssertions, softSkip } from '../lib/soft-skip.js';
|
|
47
47
|
import { makeTraceparent } from '../lib/trace-context.js';
|
|
48
48
|
|
|
49
49
|
const NOOP_WORKFLOW_ID = 'conformance-noop';
|
|
@@ -188,7 +188,9 @@ describe.skipIf(SKIP_NO_NOOP)('debug-bundle: invariants per debug-bundle.md', ()
|
|
|
188
188
|
|
|
189
189
|
const bundleRes = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
|
|
190
190
|
const eventsRes = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
|
|
191
|
-
|
|
191
|
+
// rest-endpoints.md §"Required endpoints": every host MUST expose
|
|
192
|
+
// /events/poll, so a conforming host always makes this comparison observable.
|
|
193
|
+
if (eventsRes.status !== 200) return blockedDespiteAssertions(`GET /v1/runs/{runId}/events/poll answered ${eventsRes.status} — the bundle/poll agreement was never observed`);
|
|
192
194
|
|
|
193
195
|
const bundle = bundleRes.json as DebugBundleShape;
|
|
194
196
|
const polledEvents = (eventsRes.json as { events?: unknown[] }).events ?? [];
|
|
@@ -238,12 +240,15 @@ describe.skipIf(SKIP_NO_NOOP)('debug-bundle: spans join the trace (RFC 0207 §C,
|
|
|
238
240
|
it('span traceId / kind / status have the RFC 0207 shape and the run span joins the caller trace', async () => {
|
|
239
241
|
if (!(await isAdvertised())) return softSkip('inapplicable', 'debugBundle not advertised by this host');
|
|
240
242
|
const tp = makeTraceparent();
|
|
243
|
+
// Run create and the bundle read are asserted by the legs above; here they
|
|
244
|
+
// are setup, and returning before any assertion lets a host that emits no
|
|
245
|
+
// spans record `inapplicable` rather than a partial-witness pass.
|
|
241
246
|
const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID }, { headers: { traceparent: tp.header } });
|
|
242
|
-
|
|
247
|
+
if (create.status !== 201) return softSkip('blocked', `POST /v1/runs answered ${create.status} — no run to read spans from`);
|
|
243
248
|
const runId = (create.json as { runId: string }).runId;
|
|
244
249
|
await pollUntilTerminal(runId);
|
|
245
250
|
const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
|
|
246
|
-
|
|
251
|
+
if (res.status !== 200) return softSkip('blocked', `GET /v1/runs/{runId}/debug-bundle answered ${res.status} — no spans to read`);
|
|
247
252
|
const spans = ((res.json as DebugBundleShape | undefined)?.spans ?? []) as Array<Record<string, unknown>>;
|
|
248
253
|
if (spans.length === 0) return softSkip('inapplicable', 'host emits no spans (spans: []) — the RFC 0207 §C trace-join rule has nothing to hold');
|
|
249
254
|
for (const s of spans) {
|
|
@@ -259,6 +264,9 @@ describe.skipIf(SKIP_NO_NOOP)('debug-bundle: spans join the trace (RFC 0207 §C,
|
|
|
259
264
|
}
|
|
260
265
|
}
|
|
261
266
|
const run = spans.find((s) => s['name'] === 'openwop.run');
|
|
267
|
+
// partial-witness-ok: the span shape rules above were observed on every
|
|
268
|
+
// span; the trace-join half needs an openwop.run span with a traceId, which
|
|
269
|
+
// is SHOULD, so its absence is recorded rather than failed.
|
|
262
270
|
if (run === undefined || run['traceId'] === undefined) return softSkip('inapplicable', 'the bundle carries no openwop.run span with a traceId (traceId is SHOULD) — the trace-join half has nothing to compare');
|
|
263
271
|
expect(run['traceId'], req(LEG, 'debug-bundle.md §"spans field" (RFC 0207 §C)', 'the openwop.run span of a run started with an honoured inbound traceparent MUST carry that trace id')).toBe(tp.traceId);
|
|
264
272
|
});
|
|
@@ -64,7 +64,9 @@ describe('discovery: /.well-known/openwop', () => {
|
|
|
64
64
|
'capabilities-change-detection.md §Capabilities-Etag',
|
|
65
65
|
'Capabilities-Etag is optional; hosts that omit it remain conformant',
|
|
66
66
|
)).toBeNull();
|
|
67
|
-
|
|
67
|
+
// partial-witness-ok: Capabilities-Etag is OPTIONAL; when absent, the IF-present MUSTs
|
|
68
|
+
// below bind nothing.
|
|
69
|
+
return softSkip('inapplicable', 'Capabilities-Etag not sent (optional) — no etag to check');
|
|
68
70
|
}
|
|
69
71
|
|
|
70
72
|
expect(firstEtag.trim().length, req('openwop.it.discovery.if-capabilities-etag-is-present-it-is-non-empty-and-stable-within-the-cache-wind',
|
|
@@ -116,7 +118,9 @@ describe('discovery: /.well-known/openwop fixtures field shape per RFC 0003', ()
|
|
|
116
118
|
// RFC 0003 makes the field OPTIONAL — pre-RFC hosts and hosts
|
|
117
119
|
// that opt out advertise nothing. Assertion passes trivially.
|
|
118
120
|
expect(fixtures).toBeUndefined();
|
|
119
|
-
|
|
121
|
+
// partial-witness-ok: discovery answered 200 and `fixtures` is OPTIONAL (RFC 0003); when
|
|
122
|
+
// absent, the IF-present MUSTs below bind nothing.
|
|
123
|
+
return softSkip('inapplicable', 'discovery carries no fixtures[] (optional, RFC 0003) — no entries to check');
|
|
120
124
|
}
|
|
121
125
|
|
|
122
126
|
expect(Array.isArray(fixtures), req('openwop.it.discovery.if-fixtures-is-present-it-must-be-a-string-of-unique-non-empty-entries',
|
|
@@ -26,7 +26,7 @@ import { driver } from '../lib/driver.js';
|
|
|
26
26
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
27
27
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
28
28
|
import { req } from '../lib/requirement-ids.js';
|
|
29
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
29
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
30
30
|
|
|
31
31
|
const PARENT = 'conformance-dispatch-cross-worker-handoff';
|
|
32
32
|
const CHILD_A = 'conformance-dispatch-cross-worker-handoff-child-a';
|
|
@@ -106,7 +106,7 @@ describe.skipIf(SKIP)('dispatch-cross-worker-handoff: sequential child→parent
|
|
|
106
106
|
const events = ((eventsRes.json as { events?: RunEvent[] } | undefined)?.events ?? []);
|
|
107
107
|
const dispatchedA = events.find((e) => e.type === 'node.dispatched' && e.payload?.childWorkflowId === CHILD_A);
|
|
108
108
|
const dispatchedB = events.find((e) => e.type === 'node.dispatched' && e.payload?.childWorkflowId === CHILD_B);
|
|
109
|
-
if (!dispatchedA || !dispatchedB) return
|
|
109
|
+
if (!dispatchedA || !dispatchedB) return blockedDespiteAssertions('the advertised fixture\'s parent run emitted no node.dispatched for child-a and child-b — per-worker mapping precedence is unobserved');
|
|
110
110
|
|
|
111
111
|
const childARes = await driver.get(`/v1/runs/${encodeURIComponent(dispatchedA.payload!.childRunId!)}`);
|
|
112
112
|
const childBRes = await driver.get(`/v1/runs/${encodeURIComponent(dispatchedB.payload!.childRunId!)}`);
|
|
@@ -26,7 +26,7 @@ import { pollUntilTerminal } from '../lib/polling.js';
|
|
|
26
26
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
27
27
|
import { setHostCapability, resetHostCapabilities, isToggleAvailable } from '../lib/host-toggle.js';
|
|
28
28
|
import { req } from '../lib/requirement-ids.js';
|
|
29
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
29
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
30
30
|
|
|
31
31
|
const PARENT = 'conformance-dispatch-input-mapping';
|
|
32
32
|
const CHILD = 'conformance-dispatch-input-mapping-child';
|
|
@@ -100,7 +100,7 @@ describe.skipIf(SKIP)('dispatch-input-mapping: parent → child variable project
|
|
|
100
100
|
const dispatched = events.find(
|
|
101
101
|
(e) => e.type === 'node.dispatched' && e.payload?.childWorkflowId === CHILD,
|
|
102
102
|
);
|
|
103
|
-
if (!dispatched) return
|
|
103
|
+
if (!dispatched) return blockedDespiteAssertions('the advertised fixture\'s parent run emitted no node.dispatched — the unset-variable projection is unobserved');
|
|
104
104
|
const childRunId = dispatched.payload?.childRunId;
|
|
105
105
|
|
|
106
106
|
const childRes = await driver.get(`/v1/runs/${encodeURIComponent(childRunId!)}`);
|
|
@@ -124,7 +124,9 @@ describe.skipIf(!isFixtureAdvertised('conformance-dispatch-cancellable-child'))(
|
|
|
124
124
|
const parentId = await registerParent('conformance-dispatch-cancellable-child');
|
|
125
125
|
if (!parentId) return softSkip('blocked', 'precondition not met — `!parentId` returned early (soft-skip) (seam, prior step, or fixture unavailable)'); // soft-skip
|
|
126
126
|
const create = await driver.post('/v1/runs', { workflowId: parentId });
|
|
127
|
-
|
|
127
|
+
// Asserted only when it fails, so no assertion has passed when the skip below
|
|
128
|
+
// decides the row (see there).
|
|
129
|
+
if (create.status !== 201) expect(create.status).toBe(201);
|
|
128
130
|
const parentRunId = (create.json as { runId: string }).runId;
|
|
129
131
|
|
|
130
132
|
// Poll for the node.dispatched event so we can cancel the child mid-flight.
|
|
@@ -140,7 +142,18 @@ describe.skipIf(!isFixtureAdvertised('conformance-dispatch-cancellable-child'))(
|
|
|
140
142
|
}
|
|
141
143
|
await new Promise((r) => setTimeout(r, 250));
|
|
142
144
|
}
|
|
143
|
-
if (!childRunId)
|
|
145
|
+
if (!childRunId) {
|
|
146
|
+
// Not `blocked`: a conforming host may emit node.dispatched only when the dispatch
|
|
147
|
+
// step RETURNS. The payload's `childStatus` is "the status of the child run at the
|
|
148
|
+
// moment the dispatch step returned" (run-event-payloads.schema.json
|
|
149
|
+
// $defs.nodeDispatched), and openwop-app emits it after the child terminates, so the
|
|
150
|
+
// 30 s child cannot be located and cancelled mid-flight. Recording `blocked` here
|
|
151
|
+
// denied certification to a conforming host (measured on openwop-app, 2026-09-27).
|
|
152
|
+
await driver.post(`/v1/runs/${encodeURIComponent(parentRunId)}/cancel`, { reason: 'conformance-cleanup' });
|
|
153
|
+
// partial-witness-ok: no assertion has passed at runtime; the only earlier expect
|
|
154
|
+
// runs solely when the create is not 201, and then it throws.
|
|
155
|
+
return softSkip('inapplicable', 'no node.dispatched carried a childRunId while the child was running; this host emits it when the dispatch step returns, so the child cannot be cancelled mid-flight and the cancelled-child case cannot be driven');
|
|
156
|
+
}
|
|
144
157
|
const cancelRes = await driver.post(`/v1/runs/${encodeURIComponent(childRunId)}/cancel`, { reason: 'hvmap-1b-cancelled test' });
|
|
145
158
|
expect(cancelRes.status === 200 || cancelRes.status === 202).toBe(true);
|
|
146
159
|
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { describe, it, expect } from 'vitest';
|
|
14
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
14
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
15
15
|
import { readDistillationCap, invokeDistill } from '../lib/distillation.js';
|
|
16
16
|
import { req } from '../lib/requirement-ids.js';
|
|
17
17
|
|
|
@@ -33,7 +33,7 @@ describe('distillation-token-budget (RFC 0062 §B)', () => {
|
|
|
33
33
|
|
|
34
34
|
// A budget too small to distill the corpus MUST fail closed, no partial archive.
|
|
35
35
|
const tooSmall = await invokeDistill({ memoryRef: 'conformance-distill', tokenBudget: 1 });
|
|
36
|
-
if (tooSmall === null) return
|
|
36
|
+
if (tooSmall === null) return blockedDespiteAssertions('the distill seam answered the in-budget call but not the un-meetable one — the atomic token_budget_exceeded failure is unobserved');
|
|
37
37
|
expect(
|
|
38
38
|
tooSmall.status >= 400 && tooSmall.body.error === 'token_budget_exceeded',
|
|
39
39
|
req('openwop.it.distillation-token-budget.within-budget-tokensused-tokenbudget-an-un-meetable-budget-fails-atomically', 'RFC 0062 §B', 'an un-meetable budget MUST fail with token_budget_exceeded'),
|
|
@@ -47,7 +47,7 @@ import { describe, expect, it } from 'vitest';
|
|
|
47
47
|
import { driver } from '../lib/driver.js';
|
|
48
48
|
import { pollUntilTerminal, scaledTimeoutMs } from '../lib/polling.js';
|
|
49
49
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
50
|
-
import {
|
|
50
|
+
import { blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
51
51
|
import { queryTestEvents, requireEvents } from '../lib/event-log-query.js';
|
|
52
52
|
import { recordRequirement } from '../lib/requirement-ledger.js';
|
|
53
53
|
import { requirementIdForFile } from '../lib/scenario-disposition.js';
|
|
@@ -112,7 +112,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('RFC 0158 §C.8 — poison work terminates with
|
|
|
112
112
|
? 'event-log seam /v1/host/sample/test/runs/{runId}/events not wired — attempts are unobservable'
|
|
113
113
|
: `event-log seam returned HTTP ${first.status}`;
|
|
114
114
|
recordRequirement(REQ, 'blocked', why);
|
|
115
|
-
return
|
|
115
|
+
return blockedDespiteAssertions(why);
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
const events = requireEvents(first, 'RFC 0158 §C.8 attempt counting');
|
|
@@ -33,7 +33,8 @@ import { pollUntilTerminal } from '../lib/polling.js';
|
|
|
33
33
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
34
34
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
35
35
|
import { req } from '../lib/requirement-ids.js';
|
|
36
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
36
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
37
|
+
import { readDispatchBudgets } from '../lib/mock-ai-budgets.js';
|
|
37
38
|
|
|
38
39
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
39
40
|
/**
|
|
@@ -210,26 +211,61 @@ describe.skipIf(HTTP_SKIP)('envelope-completion-distinguishes-truncation: schema
|
|
|
210
211
|
]);
|
|
211
212
|
if (seed.status === 404) return softSkip('blocked', 'precondition not met — `seed.status === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
212
213
|
|
|
213
|
-
await startRunAndRead(SCHEMA_VIOLATION_FIXTURE);
|
|
214
|
-
const
|
|
215
|
-
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
//
|
|
219
|
-
//
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
|
|
225
|
-
if (
|
|
214
|
+
const result = await startRunAndRead(SCHEMA_VIOLATION_FIXTURE);
|
|
215
|
+
const REQ_ID = 'openwop.it.envelope-completion-distinguishes-truncation.schema-violation-retry-budget-unchanged-from-initial-no-budget-multiplication-on';
|
|
216
|
+
|
|
217
|
+
// Suite 2.42.7: the per-attempt witness. `last-dispatch-budget` reports only
|
|
218
|
+
// the most recent call, so the retry's budget could not be compared with the
|
|
219
|
+
// attempt before it — a host that doubled 256 → 512 on a schema-violation
|
|
220
|
+
// retry passed the `< 20_000` range check below. `dispatch-budgets`
|
|
221
|
+
// (host-sample-test-seams.md §28) returns every call's budget in call order,
|
|
222
|
+
// and the fixture now sets `maxTokens: 256` so attempt 1 carries one. A served
|
|
223
|
+
// seam that breaks its contract is `blocked`; an UNSERVED one falls through to
|
|
224
|
+
// the pre-seam partial witness below (the seam is newer than this leg).
|
|
225
|
+
const budgets = await readDispatchBudgets(NODE_OF[SCHEMA_VIOLATION_FIXTURE] as string);
|
|
226
|
+
if (!budgets.ok && !budgets.unserved) return blockedDespiteAssertions(budgets.reason);
|
|
227
|
+
if (!budgets.ok) {
|
|
228
|
+
const budget = await lastBudget(SCHEMA_VIOLATION_FIXTURE);
|
|
229
|
+
if (budget === null) return softSkip('blocked', 'precondition not met — `budget === null` returned early (seam, prior step, or fixture unavailable)');
|
|
230
|
+
// Pre-seam path: only the most recent call's budget is visible, so it is
|
|
231
|
+
// range-checked against the truncation path (≥2× the default — typically
|
|
232
|
+
// 8000 for the sample's structuredOutput dispatch path), not compared.
|
|
226
233
|
expect(
|
|
227
234
|
budget,
|
|
228
|
-
req(
|
|
235
|
+
req(REQ_ID,
|
|
229
236
|
'RFCS/0033-envelope-completion-contract.md §C',
|
|
230
237
|
'schema-violation retry MUST NOT multiply maxTokens — budget stays at the original value (host default)',
|
|
231
238
|
),
|
|
232
239
|
).toBeLessThan(20_000);
|
|
240
|
+
// partial-witness-ok: the host does not serve the optional dispatch-budgets
|
|
241
|
+
// seam (host-sample-test-seams.md §28), so the retry's budget is range-checked,
|
|
242
|
+
// not compared with the attempt before it. The acceptance predicate refuses a
|
|
243
|
+
// partial-witness row.
|
|
244
|
+
return softSkip('inapplicable', `partial witness — the §C budget rule is only range-checked: ${budgets.reason}; the most recent call's maxTokens (${budget}) is below the truncation-path range, but a retry that grew the budget within that range is not detected`);
|
|
245
|
+
}
|
|
246
|
+
if (result === null) return blockedDespiteAssertions('the run create or event-log read failed — the retry count the budgets are checked against was never observed');
|
|
247
|
+
const retries = result.events.filter((e) => e.type === 'envelope.retry.attempted').length;
|
|
248
|
+
// RFC 0033 §C: a host MAY retry. No retry means no retry budget to compare;
|
|
249
|
+
// the sibling leg above convicts a host that should have retried and did not.
|
|
250
|
+
// partial-witness-ok: no retry happened, so there is no retry budget to observe.
|
|
251
|
+
if (retries === 0) return softSkip('inapplicable', 'the host made no schema-violation retry (RFC 0033 §C: MAY retry) — there is no retry budget to compare');
|
|
252
|
+
if (budgets.attempts.length < retries + 1) {
|
|
253
|
+
return blockedDespiteAssertions(`the dispatch-budgets seam recorded ${budgets.attempts.length} call(s) but the run emitted ${retries} envelope.retry.attempted event(s), i.e. ${retries + 1} calls — the seam does not report every attempt (host-sample-test-seams.md §28)`);
|
|
254
|
+
}
|
|
255
|
+
const attempts = budgets.attempts;
|
|
256
|
+
if (!attempts.every((m): m is number => m !== null)) {
|
|
257
|
+
// partial-witness-ok: the seam is served but at least one call carried no
|
|
258
|
+
// budget (maxTokens null), so no two attempts' budgets can be compared.
|
|
259
|
+
return softSkip('inapplicable', `partial witness — the §C budget rule is UNWITNESSED: the dispatch-budgets seam shows a call with no output budget (per-attempt budgets: ${JSON.stringify(attempts)}), so no retry's budget can be compared with the attempt before it`);
|
|
260
|
+
}
|
|
261
|
+
for (let i = 1; i < attempts.length; i++) {
|
|
262
|
+
expect(
|
|
263
|
+
attempts[i]! <= attempts[i - 1]!,
|
|
264
|
+
req(REQ_ID,
|
|
265
|
+
'RFCS/0033-envelope-completion-contract.md §C',
|
|
266
|
+
`schema-violation retries SHALL NOT include an increased output budget — attempt ${i + 1} carried maxTokens ${attempts[i]} after attempt ${i} carried ${attempts[i - 1]} (per-attempt budgets: ${JSON.stringify(attempts)})`,
|
|
267
|
+
),
|
|
268
|
+
).toBe(true);
|
|
233
269
|
}
|
|
234
270
|
});
|
|
235
271
|
});
|
|
@@ -29,7 +29,7 @@ import { driver } from '../lib/driver.js';
|
|
|
29
29
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
30
30
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
31
31
|
import { req } from '../lib/requirement-ids.js';
|
|
32
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
32
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
33
33
|
|
|
34
34
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
35
35
|
const FIXTURE = 'conformance-envelope-nl-to-format-engaged';
|
|
@@ -77,7 +77,7 @@ describe.skipIf(HTTP_SKIP)('envelope-nl-to-format-engaged: runtime behavior (RFC
|
|
|
77
77
|
expect(seed.status).toBe(200);
|
|
78
78
|
|
|
79
79
|
const events = await runAndReadEvents();
|
|
80
|
-
if (events === null) return
|
|
80
|
+
if (events === null) return blockedDespiteAssertions('the advertised fixture run did not start (≠201) or its event log did not read (≠200) — envelope.nlToFormat.engaged is unobserved');
|
|
81
81
|
const engagements = events.filter((e) => e.type === 'envelope.nlToFormat.engaged');
|
|
82
82
|
expect(
|
|
83
83
|
engagements.length,
|
|
@@ -123,7 +123,7 @@ describe.skipIf(HTTP_SKIP)('envelope-recovery-applied: SECURITY invariant envelo
|
|
|
123
123
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
124
124
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
125
125
|
import { req } from '../lib/requirement-ids.js';
|
|
126
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
126
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
127
127
|
|
|
128
128
|
const RECOVERY_FIXTURE = 'conformance-envelope-recovery-applied';
|
|
129
129
|
const RECOVERY_NODE_ID = 'recovery-applied-structured-call';
|
|
@@ -163,7 +163,7 @@ describe.skipIf(HTTP_SKIP)('envelope-recovery-applied: end-to-end through the en
|
|
|
163
163
|
expect(seed.status).toBe(200);
|
|
164
164
|
|
|
165
165
|
const events = await runAndReadEvents();
|
|
166
|
-
if (events === null) return
|
|
166
|
+
if (events === null) return blockedDespiteAssertions('the advertised fixture run did not start (≠201) or its event log did not read (≠200) — envelope.recovery.applied is unobserved');
|
|
167
167
|
const recoveries = events.filter((e) => e.type === 'envelope.recovery.applied');
|
|
168
168
|
expect(
|
|
169
169
|
recoveries.length,
|
|
@@ -193,7 +193,7 @@ import { pollUntilTerminal } from '../lib/polling.js';
|
|
|
193
193
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
194
194
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
195
195
|
import { req } from '../lib/requirement-ids.js';
|
|
196
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
196
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
197
197
|
|
|
198
198
|
const E2E_FIXTURE = 'conformance-envelope-refusal';
|
|
199
199
|
const E2E_NODE_ID = 'refusal-structured-call';
|
|
@@ -232,7 +232,7 @@ describe.skipIf(HTTP_SKIP)('envelope-refusal-shape: end-to-end refusal through d
|
|
|
232
232
|
expect(seed.status).toBe(200);
|
|
233
233
|
|
|
234
234
|
const result = await runE2eAndRead();
|
|
235
|
-
if (result === null) return
|
|
235
|
+
if (result === null) return blockedDespiteAssertions('the advertised fixture run did not start (≠201) or its event log did not read (≠200) — envelope.refusal and the no-retry rule are unobserved');
|
|
236
236
|
const refusals = result.events.filter((e) => e.type === 'envelope.refusal');
|
|
237
237
|
expect(
|
|
238
238
|
refusals.length,
|