@openwop/openwop-conformance 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +156 -1
- package/README.md +3 -2
- package/api/asyncapi.yaml +8 -0
- package/api/openapi.yaml +371 -1
- package/api/redocly.yaml +15 -0
- package/coverage.md +26 -5
- package/fixtures/conformance-agent-reasoning-streaming.json +37 -0
- package/fixtures/conformance-dispatch-cancellable-child.json +27 -0
- package/fixtures/conformance-dispatch-deterministic-fail-child.json +30 -0
- package/fixtures/conformance-dispatch-input-mapping-no-default.json +49 -0
- package/fixtures/conformance-dispatch-per-worker-override.json +59 -0
- package/fixtures/conformance-envelope-nl-to-format-engaged.json +41 -0
- package/fixtures/conformance-envelope-recovery-applied.json +39 -0
- package/fixtures/conformance-envelope-refusal.json +38 -0
- package/fixtures/conformance-envelope-retry-attempted.json +39 -0
- package/fixtures/conformance-envelope-retry-exhausted.json +38 -0
- package/fixtures/conformance-envelope-truncated.json +39 -0
- package/fixtures/conformance-envelope-truncation-cap-exhaustion.json +39 -0
- package/fixtures/conformance-model-capability-insufficient.json +25 -0
- package/fixtures/conformance-multi-agent-confidence-escalation.json +49 -0
- package/fixtures/conformance-multi-agent-handoff-child.json +27 -0
- package/fixtures/conformance-multi-agent-handoff.json +49 -0
- package/fixtures/conformance-prompt-all-four-kinds.json +39 -0
- package/fixtures/conformance-prompt-end-to-end.json +33 -0
- package/fixtures/conformance-subworkflow-input-mapping-no-default.json +33 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation-child.json +31 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation.json +33 -0
- package/fixtures/openwop-smoke-cost-emit.json +37 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot-2.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-schema-hint.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-secret-redaction.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-trust-marker.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-system.json +15 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-user.json +15 -0
- package/fixtures.md +45 -0
- package/package.json +1 -1
- package/schemas/README.md +5 -0
- package/schemas/agent-manifest.schema.json +16 -0
- package/schemas/capabilities.schema.json +390 -0
- package/schemas/core-conformance-mock-agent-config.schema.json +5 -0
- package/schemas/envelopes/clarification.request.schema.json +9 -0
- package/schemas/envelopes/error.schema.json +4 -0
- package/schemas/envelopes/schema.request.schema.json +4 -0
- package/schemas/envelopes/schema.response.schema.json +1 -1
- package/schemas/node-pack-manifest.schema.json +28 -0
- package/schemas/orchestrator-decision.schema.json +12 -0
- package/schemas/prompt-kind.schema.json +8 -0
- package/schemas/prompt-pack-manifest.schema.json +80 -0
- package/schemas/prompt-ref.schema.json +40 -0
- package/schemas/prompt-template.schema.json +149 -0
- package/schemas/registry-version-manifest.schema.json +5 -0
- package/schemas/run-ancestry-response.schema.json +54 -0
- package/schemas/run-event-payloads.schema.json +513 -11
- package/schemas/run-event.schema.json +17 -1
- package/schemas/run-snapshot.schema.json +3 -2
- package/schemas/workflow-definition.schema.json +19 -1
- package/src/lib/driver.ts +15 -0
- package/src/lib/env.ts +51 -0
- package/src/lib/event-log-query.ts +62 -0
- package/src/lib/fixtures.ts +38 -1
- package/src/lib/host-toggle.ts +54 -0
- package/src/lib/llm-cache-key-recipe.ts +68 -0
- package/src/lib/multi-agent-capabilities.ts +10 -0
- package/src/lib/otel-scrape.ts +59 -0
- package/src/scenarios/agentReasoningStreaming.test.ts +193 -0
- package/src/scenarios/aiEnvelope.capBreached.test.ts +97 -9
- package/src/scenarios/aiEnvelope.contractRefusal.test.ts +224 -15
- package/src/scenarios/aiEnvelope.correlationReplay.test.ts +257 -25
- package/src/scenarios/aiEnvelope.redaction.test.ts +210 -29
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +163 -24
- package/src/scenarios/aiEnvelope.trustBoundaryPropagation.test.ts +262 -12
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +107 -16
- package/src/scenarios/blob-presign-expiry.test.ts +42 -9
- package/src/scenarios/blob-roundtrip.test.ts +0 -0
- package/src/scenarios/cache-ttl-expiry.test.ts +34 -8
- package/src/scenarios/cost-attribution.test.ts +124 -11
- package/src/scenarios/cross-engine-append-ordering.test.ts +99 -0
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +136 -0
- package/src/scenarios/cross-host-causation-shape.test.ts +117 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +60 -0
- package/src/scenarios/dispatch-cross-worker-handoff.test.ts +34 -3
- package/src/scenarios/dispatch-input-mapping.test.ts +75 -6
- package/src/scenarios/dispatch-output-mapping.test.ts +96 -6
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +223 -0
- package/src/scenarios/envelope-nl-to-format-engaged.test.ts +152 -0
- package/src/scenarios/envelope-reasoning-secret-redaction.test.ts +343 -0
- package/src/scenarios/envelope-reasoning-shape.test.ts +190 -0
- package/src/scenarios/envelope-recovery-applied.test.ts +229 -0
- package/src/scenarios/envelope-refusal-shape.test.ts +289 -0
- package/src/scenarios/envelope-retry-attempted.test.ts +258 -0
- package/src/scenarios/envelope-retry-exhausted.test.ts +168 -0
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +229 -0
- package/src/scenarios/envelope-truncated.test.ts +136 -0
- package/src/scenarios/envelope-truncation-cap-exhaustion.test.ts +144 -0
- package/src/scenarios/envelope-variant-discriminator-static.test.ts +152 -0
- package/src/scenarios/fixtures-gating.test.ts +139 -1
- package/src/scenarios/fixtures-valid.test.ts +123 -15
- package/src/scenarios/kv-ttl-expiry.test.ts +40 -9
- package/src/scenarios/model-capability-insufficient.test.ts +221 -0
- package/src/scenarios/model-capability-substituted.test.ts +203 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +164 -0
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +167 -0
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +124 -0
- package/src/scenarios/multi-region-idempotency.test.ts +58 -0
- package/src/scenarios/node-module-required-capabilities-shape.test.ts +185 -0
- package/src/scenarios/otel-trace-propagation-subworkflow.test.ts +19 -0
- package/src/scenarios/pack-registry-publish.test.ts +231 -51
- package/src/scenarios/prompt-all-four-kinds-events.test.ts +198 -0
- package/src/scenarios/prompt-composed-secret-redaction.test.ts +178 -0
- package/src/scenarios/prompt-composed-trust-marker.test.ts +165 -0
- package/src/scenarios/prompt-end-to-end-events.test.ts +202 -0
- package/src/scenarios/prompt-list-and-fetch.test.ts +207 -0
- package/src/scenarios/prompt-mutable-lifecycle.test.ts +216 -0
- package/src/scenarios/prompt-pack-install.test.ts +187 -0
- package/src/scenarios/prompt-render-deterministic.test.ts +240 -0
- package/src/scenarios/prompt-resolution-chain-agent-intrinsic.test.ts +140 -0
- package/src/scenarios/prompt-resolution-chain-fallback-cascade.test.ts +172 -0
- package/src/scenarios/prompt-resolution-chain-node-wins.test.ts +144 -0
- package/src/scenarios/prompt-template-shape.test.ts +359 -0
- package/src/scenarios/provider-usage.test.ts +185 -0
- package/src/scenarios/queue-ack-nack-dlq.test.ts +64 -10
- package/src/scenarios/queue-publish-consume-roundtrip.test.ts +50 -10
- package/src/scenarios/replay-divergence-at-refusal.test.ts +134 -0
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +197 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +127 -25
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +80 -0
- package/src/scenarios/sandbox-capability-gate-respected.test.ts +31 -0
- package/src/scenarios/sandbox-memory-cap.test.ts +61 -0
- package/src/scenarios/sandbox-no-cross-pack-mutation.test.ts +35 -0
- package/src/scenarios/sandbox-no-host-env-leak.test.ts +38 -0
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +91 -0
- package/src/scenarios/sandbox-no-host-process-escape.test.ts +30 -0
- package/src/scenarios/sandbox-no-network-escape.test.ts +49 -0
- package/src/scenarios/sandbox-timeout-cap.test.ts +61 -0
- package/src/scenarios/search-bm25-roundtrip.test.ts +54 -9
- package/src/scenarios/spec-corpus-validity.test.ts +34 -6
- package/src/scenarios/sql-transaction-atomicity.test.ts +37 -8
- package/src/scenarios/stream-subscribe-from-beginning.test.ts +46 -9
- package/src/scenarios/subworkflow-input-mapping.test.ts +146 -10
- package/src/scenarios/table-cursor-pagination.test.ts +47 -9
- package/src/scenarios/table-schema-enforcement.test.ts +46 -9
- package/src/scenarios/vector-knn-roundtrip.test.ts +50 -10
- package/src/scenarios/workflow-chain-host-expansion.test.ts +202 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0024 — streaming `agent.reasoning.delta` events.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that hosts advertising `capabilities.agents.reasoning.streaming: true`
|
|
5
|
+
* emit incremental `agent.reasoning.delta` events while a reasoning
|
|
6
|
+
* block is still open, followed by exactly one closing `agent.reasoned`
|
|
7
|
+
* event carrying the full authoritative content.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when the host doesn't advertise
|
|
10
|
+
* `capabilities.agents.supported: true` AND
|
|
11
|
+
* `capabilities.agents.reasoning.streaming: true`, OR when reasoning
|
|
12
|
+
* verbosity is `'off'`.
|
|
13
|
+
*
|
|
14
|
+
* Driven by the `core.conformance.mock-agent` typeId (RFC 0023)
|
|
15
|
+
* extended with `mockReasoning.streamChunks` per RFC 0024 §"Conformance"
|
|
16
|
+
* (see `schemas/core-conformance-mock-agent-config.schema.json`).
|
|
17
|
+
*
|
|
18
|
+
* @see RFCS/0024-agent-reasoning-streaming.md
|
|
19
|
+
* @see schemas/run-event-payloads.schema.json §`agentReasoningDelta`
|
|
20
|
+
* @see schemas/capabilities.schema.json §`agents.reasoning.streaming`
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { describe, it, expect } from 'vitest';
|
|
24
|
+
import { driver } from '../lib/driver.js';
|
|
25
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
26
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
27
|
+
import {
|
|
28
|
+
isAgentSupported,
|
|
29
|
+
isReasoningStreamingSupported,
|
|
30
|
+
getReasoningVerbosity,
|
|
31
|
+
} from '../lib/multi-agent-capabilities.js';
|
|
32
|
+
|
|
33
|
+
const FIXTURE = 'conformance-agent-reasoning-streaming';
|
|
34
|
+
/** Expected concatenation of the fixture's `streamChunks` — kept in sync
|
|
35
|
+
* with `conformance/fixtures/conformance-agent-reasoning-streaming.json`.
|
|
36
|
+
* When the fixture changes, this constant changes with it. */
|
|
37
|
+
const EXPECTED_CHUNKS = [
|
|
38
|
+
'Let me think about this. ',
|
|
39
|
+
'First, the user is asking a question. ',
|
|
40
|
+
'Therefore, I should respond clearly.',
|
|
41
|
+
] as const;
|
|
42
|
+
const EXPECTED_FULL = EXPECTED_CHUNKS.join('');
|
|
43
|
+
|
|
44
|
+
const SKIP =
|
|
45
|
+
!isAgentSupported() ||
|
|
46
|
+
!isReasoningStreamingSupported() ||
|
|
47
|
+
getReasoningVerbosity() === 'off' ||
|
|
48
|
+
!isFixtureAdvertised(FIXTURE);
|
|
49
|
+
|
|
50
|
+
describe.skipIf(SKIP)('agentReasoningStreaming: RFC 0024 incremental + closing event contract', () => {
|
|
51
|
+
it('emits N agent.reasoning.delta events followed by exactly one closing agent.reasoned', async () => {
|
|
52
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
53
|
+
expect(create.status).toBe(201);
|
|
54
|
+
const runId = (create.json as { runId: string }).runId;
|
|
55
|
+
|
|
56
|
+
await pollUntilTerminal(runId);
|
|
57
|
+
|
|
58
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
59
|
+
expect(events.status).toBe(200);
|
|
60
|
+
const list = (events.json as { events: Array<{ type: string; payload?: Record<string, unknown> }> }).events;
|
|
61
|
+
|
|
62
|
+
const deltas = list.filter((e) => e.type === 'agent.reasoning.delta');
|
|
63
|
+
const finals = list.filter((e) => e.type === 'agent.reasoned');
|
|
64
|
+
|
|
65
|
+
expect(
|
|
66
|
+
deltas.length,
|
|
67
|
+
driver.describe(
|
|
68
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
69
|
+
'streaming host MUST emit one agent.reasoning.delta per streamChunks entry',
|
|
70
|
+
),
|
|
71
|
+
).toBe(EXPECTED_CHUNKS.length);
|
|
72
|
+
expect(
|
|
73
|
+
finals.length,
|
|
74
|
+
driver.describe(
|
|
75
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
76
|
+
'streaming host MUST emit exactly one closing agent.reasoned event after the deltas',
|
|
77
|
+
),
|
|
78
|
+
).toBe(1);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('agent.reasoning.delta `sequence` starts at 0 and increments by 1 within the block', async () => {
|
|
82
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
83
|
+
expect(create.status).toBe(201);
|
|
84
|
+
const runId = (create.json as { runId: string }).runId;
|
|
85
|
+
|
|
86
|
+
await pollUntilTerminal(runId);
|
|
87
|
+
|
|
88
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
89
|
+
const list = (events.json as { events: Array<{ type: string; payload?: Record<string, unknown> }> }).events;
|
|
90
|
+
|
|
91
|
+
const deltas = list.filter((e) => e.type === 'agent.reasoning.delta');
|
|
92
|
+
const sequences = deltas
|
|
93
|
+
.map((e) => e.payload?.sequence)
|
|
94
|
+
.filter((s): s is number => typeof s === 'number');
|
|
95
|
+
|
|
96
|
+
expect(
|
|
97
|
+
sequences,
|
|
98
|
+
driver.describe(
|
|
99
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
100
|
+
'`sequence` MUST start at 0 and increment by 1 per delta within a block',
|
|
101
|
+
),
|
|
102
|
+
).toEqual(EXPECTED_CHUNKS.map((_, i) => i));
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('closing agent.reasoned.reasoning is the concatenation of the deltas (authoritative)', async () => {
|
|
106
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
107
|
+
expect(create.status).toBe(201);
|
|
108
|
+
const runId = (create.json as { runId: string }).runId;
|
|
109
|
+
|
|
110
|
+
await pollUntilTerminal(runId);
|
|
111
|
+
|
|
112
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
113
|
+
const list = (events.json as { events: Array<{ type: string; payload?: Record<string, unknown> }> }).events;
|
|
114
|
+
|
|
115
|
+
const finalEvent = list.find((e) => e.type === 'agent.reasoned');
|
|
116
|
+
expect(finalEvent, 'closing agent.reasoned must be present').toBeDefined();
|
|
117
|
+
const reasoning = finalEvent?.payload?.reasoning;
|
|
118
|
+
expect(typeof reasoning, 'closing event MUST carry a reasoning string').toBe('string');
|
|
119
|
+
// The mock-agent's contract: closing reasoning equals concat(streamChunks).
|
|
120
|
+
// Real hosts MAY transform at finalize (summary truncation, redaction);
|
|
121
|
+
// for the mock-agent fixture, no transform applies — exact equality.
|
|
122
|
+
expect(
|
|
123
|
+
reasoning,
|
|
124
|
+
driver.describe(
|
|
125
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
126
|
+
'closing agent.reasoned.reasoning is authoritative; for the mock-agent fixture, equals delta concatenation',
|
|
127
|
+
),
|
|
128
|
+
).toBe(EXPECTED_FULL);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('agentId is consistent across all streaming + closing events in a block', async () => {
|
|
132
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
133
|
+
expect(create.status).toBe(201);
|
|
134
|
+
const runId = (create.json as { runId: string }).runId;
|
|
135
|
+
|
|
136
|
+
await pollUntilTerminal(runId);
|
|
137
|
+
|
|
138
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
139
|
+
const list = (events.json as { events: Array<{ type: string; payload?: Record<string, unknown> }> }).events;
|
|
140
|
+
|
|
141
|
+
const relevant = list.filter(
|
|
142
|
+
(e) => e.type === 'agent.reasoning.delta' || e.type === 'agent.reasoned',
|
|
143
|
+
);
|
|
144
|
+
const agentIds = new Set(
|
|
145
|
+
relevant
|
|
146
|
+
.map((e) => e.payload?.agentId)
|
|
147
|
+
.filter((a): a is string => typeof a === 'string' && a.length > 0),
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
expect(
|
|
151
|
+
agentIds.size,
|
|
152
|
+
driver.describe(
|
|
153
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
154
|
+
'agentId MUST be consistent across all `agent.reasoning.delta` events AND the closing `agent.reasoned` for a given block',
|
|
155
|
+
),
|
|
156
|
+
).toBe(1);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('all agent.reasoning.delta events arrive BEFORE the closing agent.reasoned', async () => {
|
|
160
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
161
|
+
expect(create.status).toBe(201);
|
|
162
|
+
const runId = (create.json as { runId: string }).runId;
|
|
163
|
+
|
|
164
|
+
await pollUntilTerminal(runId);
|
|
165
|
+
|
|
166
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
167
|
+
const list = (events.json as Array<{ type: string }> | { events: Array<{ type: string }> });
|
|
168
|
+
const arr = Array.isArray(list) ? list : list.events;
|
|
169
|
+
|
|
170
|
+
const closingIdx = arr.findIndex((e) => e.type === 'agent.reasoned');
|
|
171
|
+
expect(closingIdx, 'closing event present').toBeGreaterThan(-1);
|
|
172
|
+
const lastDeltaIdx = arr.map((e) => e.type).lastIndexOf('agent.reasoning.delta');
|
|
173
|
+
|
|
174
|
+
// Guard against vacuous pass: a host advertising streaming but
|
|
175
|
+
// emitting ZERO deltas would otherwise pass `-1 < closingIdx`
|
|
176
|
+
// trivially. The fixture configures 3 streamChunks, so at least
|
|
177
|
+
// one delta MUST appear in the event log.
|
|
178
|
+
expect(
|
|
179
|
+
lastDeltaIdx,
|
|
180
|
+
driver.describe(
|
|
181
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
182
|
+
'streaming host MUST emit at least one `agent.reasoning.delta` for a fixture with non-empty `streamChunks`',
|
|
183
|
+
),
|
|
184
|
+
).toBeGreaterThan(-1);
|
|
185
|
+
expect(
|
|
186
|
+
lastDeltaIdx,
|
|
187
|
+
driver.describe(
|
|
188
|
+
'RFCS/0024-agent-reasoning-streaming.md §Proposal',
|
|
189
|
+
'every `agent.reasoning.delta` MUST precede the closing `agent.reasoned` for the same block',
|
|
190
|
+
),
|
|
191
|
+
).toBeLessThan(closingIdx);
|
|
192
|
+
});
|
|
193
|
+
});
|
|
@@ -161,13 +161,101 @@ describe('aiEnvelope.capBreached: behavioral cap enforcement (FINAL v1.1)', () =
|
|
|
161
161
|
});
|
|
162
162
|
});
|
|
163
163
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
it
|
|
172
|
-
|
|
164
|
+
// E.1 engine-projection via the test-only event-log seam. The acceptor
|
|
165
|
+
// returns the breached outcome; the seam projects it onto cap.breached +
|
|
166
|
+
// node.failed per capabilities.md §"Engine-enforced limits". Tests
|
|
167
|
+
// soft-skip on HTTP 404 when the seam isn't exposed.
|
|
168
|
+
import { queryTestEvents, isEventLogSeamAvailable, resetTestSeam } from '../lib/event-log-query.js';
|
|
169
|
+
|
|
170
|
+
describe('aiEnvelope.capBreached: engine projection via event-log seam (capabilities.md §"cap.breached")', () => {
|
|
171
|
+
it('breached outcome projects to cap.breached { kind: "envelopes" } event with causationId chain', async () => {
|
|
172
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
173
|
+
const runId = `r-cap-env-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
174
|
+
const correlationId = `${runId}:node-1:turn-0:cap-env`;
|
|
175
|
+
const r = await accept(
|
|
176
|
+
{
|
|
177
|
+
type: 'error',
|
|
178
|
+
schemaVersion: 1,
|
|
179
|
+
envelopeId: 'env-proj-cap-env',
|
|
180
|
+
correlationId,
|
|
181
|
+
payload: { code: 'x', message: 'y' },
|
|
182
|
+
meta: baseMeta,
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
counters: { envelopesPerTurn: { current: 32, cap: 32 } },
|
|
186
|
+
projectTo: { runId, nodeId: 'node-1' },
|
|
187
|
+
},
|
|
188
|
+
);
|
|
189
|
+
if (r.status === 404) return;
|
|
190
|
+
expect(r.body.status).toBe('breached');
|
|
191
|
+
|
|
192
|
+
const events = await queryTestEvents(runId, { type: 'cap.breached' });
|
|
193
|
+
if (!events.ok) return;
|
|
194
|
+
expect(
|
|
195
|
+
events.events.length,
|
|
196
|
+
driver.describe('capabilities.md §"Engine-enforced limits and the cap.breached event"', 'breached outcome MUST project to exactly one cap.breached event'),
|
|
197
|
+
).toBe(1);
|
|
198
|
+
const evt = events.events[0]!;
|
|
199
|
+
expect(evt.payload.kind).toBe('envelopes');
|
|
200
|
+
expect(evt.causationId).toBe(correlationId);
|
|
201
|
+
await resetTestSeam();
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it('cap.breached payload includes limit, observed, and nodeId per capabilities.md', async () => {
|
|
205
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
206
|
+
const runId = `r-cap-payload-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
207
|
+
await accept(
|
|
208
|
+
{
|
|
209
|
+
type: 'clarification.request',
|
|
210
|
+
schemaVersion: 1,
|
|
211
|
+
envelopeId: 'env-proj-cap-clar',
|
|
212
|
+
correlationId: `${runId}:node-2:turn-0:cap`,
|
|
213
|
+
payload: { questions: [{ id: 'q1', question: 'why?' }] },
|
|
214
|
+
meta: baseMeta,
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
counters: { clarificationRounds: { current: 5, cap: 5 } },
|
|
218
|
+
projectTo: { runId, nodeId: 'node-2' },
|
|
219
|
+
},
|
|
220
|
+
);
|
|
221
|
+
const events = await queryTestEvents(runId, { type: 'cap.breached' });
|
|
222
|
+
if (!events.ok || events.events.length === 0) return;
|
|
223
|
+
const evt = events.events[0]!;
|
|
224
|
+
expect(evt.payload.kind).toBe('clarification');
|
|
225
|
+
expect(
|
|
226
|
+
typeof evt.payload.limit,
|
|
227
|
+
driver.describe('capabilities.md §"cap.breached"', 'payload.limit MUST be present as a number'),
|
|
228
|
+
).toBe('number');
|
|
229
|
+
expect(evt.payload.nodeId).toBe('node-2');
|
|
230
|
+
await resetTestSeam();
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it('cap.breached MUST be paired with a terminal node.failed transition', async () => {
|
|
234
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
235
|
+
const runId = `r-cap-fail-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
236
|
+
await accept(
|
|
237
|
+
{
|
|
238
|
+
type: 'schema.request',
|
|
239
|
+
schemaVersion: 1,
|
|
240
|
+
envelopeId: 'env-proj-cap-fail',
|
|
241
|
+
correlationId: `${runId}:node-3:turn-0:cap`,
|
|
242
|
+
payload: { envelopeType: 'vendor.acme.foo' },
|
|
243
|
+
meta: baseMeta,
|
|
244
|
+
},
|
|
245
|
+
{
|
|
246
|
+
counters: { schemaRounds: { current: 3, cap: 3 } },
|
|
247
|
+
projectTo: { runId, nodeId: 'node-3' },
|
|
248
|
+
},
|
|
249
|
+
);
|
|
250
|
+
const breached = await queryTestEvents(runId, { type: 'cap.breached' });
|
|
251
|
+
const failed = await queryTestEvents(runId, { type: 'node.failed' });
|
|
252
|
+
if (!breached.ok || !failed.ok) return;
|
|
253
|
+
expect(breached.events.length).toBe(1);
|
|
254
|
+
expect(
|
|
255
|
+
failed.events.length,
|
|
256
|
+
driver.describe('capabilities.md §"cap.breached"', 'cap.breached MUST be paired with a terminal node.failed event'),
|
|
257
|
+
).toBe(1);
|
|
258
|
+
expect((failed.events[0]!.payload.error as { code?: string }).code).toBe('cap_breached');
|
|
259
|
+
await resetTestSeam();
|
|
260
|
+
});
|
|
173
261
|
});
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* aiEnvelope.contractRefusal — FINAL v1.1 advertisement-shape
|
|
2
|
+
* aiEnvelope.contractRefusal — FINAL v1.1 advertisement-shape + behavioral.
|
|
3
3
|
*
|
|
4
|
-
* Status:
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). `spec/v1/ai-envelope.md`
|
|
5
|
+
* promoted Draft → FINAL v1.1 2026-05-18. Live behavioral via the
|
|
6
|
+
* `POST /v1/host/sample/envelope/accept` seam + the capability-toggle seam
|
|
7
|
+
* (soft-skip when either is absent).
|
|
7
8
|
*
|
|
8
9
|
* Summary: an Envelope Contract is a per-typeId declaration of which envelope
|
|
9
10
|
* kinds that node accepts (`accepts: string[]` plus implicit universals). When
|
|
@@ -19,8 +20,9 @@
|
|
|
19
20
|
* @see spec/v1/ai-envelope.md §"Envelope Contract"
|
|
20
21
|
*/
|
|
21
22
|
|
|
22
|
-
import { describe, it, expect } from 'vitest';
|
|
23
|
+
import { describe, it, expect, afterEach } from 'vitest';
|
|
23
24
|
import { driver } from '../lib/driver.js';
|
|
25
|
+
import { setHostCapability, resetHostCapabilities, isToggleAvailable } from '../lib/host-toggle.js';
|
|
24
26
|
|
|
25
27
|
interface DiscoveryDoc {
|
|
26
28
|
capabilities?: Record<string, unknown>;
|
|
@@ -137,14 +139,221 @@ describe('aiEnvelope.contractRefusal: behavioral accept-gate (FINAL v1.1)', () =
|
|
|
137
139
|
});
|
|
138
140
|
});
|
|
139
141
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
142
|
+
// E.1 engine-projection via the test-only event-log seam.
|
|
143
|
+
import { queryTestEvents, isEventLogSeamAvailable, resetTestSeam } from '../lib/event-log-query.js';
|
|
144
|
+
|
|
145
|
+
describe('aiEnvelope.contractRefusal: engine projection via event-log seam', () => {
|
|
146
|
+
it('gated (fail-node) → node.failed { error.code: "envelope_contract_violation" }', async () => {
|
|
147
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
148
|
+
const runId = `r-cr-fail-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
149
|
+
const r = await accept(
|
|
150
|
+
{
|
|
151
|
+
type: 'vendor.x.bar.create',
|
|
152
|
+
schemaVersion: 1,
|
|
153
|
+
envelopeId: 'env-cr-proj-1',
|
|
154
|
+
correlationId: `${runId}:n:0:cr-proj-1`,
|
|
155
|
+
payload: {},
|
|
156
|
+
meta: baseMeta,
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
hostSupportedEnvelopes: ['vendor.x.bar.create', 'vendor.x.foo.create'],
|
|
160
|
+
nodeAllowedKinds: ['vendor.x.foo.create'],
|
|
161
|
+
projectTo: { runId, nodeId: 'n', refusalMode: 'fail-node' },
|
|
162
|
+
},
|
|
163
|
+
);
|
|
164
|
+
if (r.status === 404) return;
|
|
165
|
+
expect(r.body.status).toBe('gated');
|
|
166
|
+
const events = await queryTestEvents(runId, { type: 'node.failed' });
|
|
167
|
+
if (!events.ok || events.events.length === 0) return;
|
|
168
|
+
const err = events.events[0]!.payload.error as { code?: string; details?: { refusedType?: string; acceptedTypes?: string[] } };
|
|
169
|
+
expect(
|
|
170
|
+
err.code,
|
|
171
|
+
driver.describe('ai-envelope.md §"Envelope Contract"', 'gated outcome MUST project to node.failed with error.code = envelope_contract_violation'),
|
|
172
|
+
).toBe('envelope_contract_violation');
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('refused envelope: error.details.refusedType names emitted kind; acceptedTypes lists allowed kinds', async () => {
|
|
176
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
177
|
+
const runId = `r-cr-details-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
178
|
+
await accept(
|
|
179
|
+
{
|
|
180
|
+
type: 'vendor.x.bar.create',
|
|
181
|
+
schemaVersion: 1,
|
|
182
|
+
envelopeId: 'env-cr-proj-details',
|
|
183
|
+
correlationId: `${runId}:n:0:cr-details`,
|
|
184
|
+
payload: {},
|
|
185
|
+
meta: baseMeta,
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
hostSupportedEnvelopes: ['vendor.x.bar.create', 'vendor.x.foo.create'],
|
|
189
|
+
nodeAllowedKinds: ['vendor.x.foo.create'],
|
|
190
|
+
projectTo: { runId, nodeId: 'n' },
|
|
191
|
+
},
|
|
192
|
+
);
|
|
193
|
+
const events = await queryTestEvents(runId, { type: 'node.failed' });
|
|
194
|
+
if (!events.ok || events.events.length === 0) return;
|
|
195
|
+
const details = (events.events[0]!.payload.error as { details?: { refusedType?: string; acceptedTypes?: string[] } }).details;
|
|
196
|
+
expect(details?.refusedType).toBe('vendor.x.bar.create');
|
|
197
|
+
expect(
|
|
198
|
+
Array.isArray(details?.acceptedTypes) && details!.acceptedTypes!.includes('vendor.x.foo.create'),
|
|
199
|
+
driver.describe('ai-envelope.md §"Envelope Contract"', 'error.details.acceptedTypes MUST list the node\'s declared accepts[] (plus universals)'),
|
|
200
|
+
).toBe(true);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it('refusalMode:"discard-and-warn" → log.appended { level: "warn" } instead of node.failed', async () => {
|
|
204
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
205
|
+
const runId = `r-cr-warn-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
206
|
+
await accept(
|
|
207
|
+
{
|
|
208
|
+
type: 'vendor.x.bar.create',
|
|
209
|
+
schemaVersion: 1,
|
|
210
|
+
envelopeId: 'env-cr-proj-warn',
|
|
211
|
+
correlationId: `${runId}:n:0:cr-warn`,
|
|
212
|
+
payload: {},
|
|
213
|
+
meta: baseMeta,
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
hostSupportedEnvelopes: ['vendor.x.bar.create'],
|
|
217
|
+
nodeAllowedKinds: ['vendor.x.foo.create'], // gated
|
|
218
|
+
projectTo: { runId, nodeId: 'n', refusalMode: 'discard-and-warn' },
|
|
219
|
+
},
|
|
220
|
+
);
|
|
221
|
+
const warnEvents = await queryTestEvents(runId, { type: 'log.appended' });
|
|
222
|
+
const failEvents = await queryTestEvents(runId, { type: 'node.failed' });
|
|
223
|
+
if (!warnEvents.ok || !failEvents.ok) return;
|
|
224
|
+
expect(
|
|
225
|
+
warnEvents.events.some((e) => (e.payload as { level?: string }).level === 'warn'),
|
|
226
|
+
driver.describe('ai-envelope.md §"Envelope Contract"', 'discard-and-warn MUST emit log.appended at warn level'),
|
|
227
|
+
).toBe(true);
|
|
228
|
+
expect(
|
|
229
|
+
failEvents.events.length,
|
|
230
|
+
driver.describe('ai-envelope.md §"Envelope Contract"', 'discard-and-warn MUST NOT emit node.failed'),
|
|
231
|
+
).toBe(0);
|
|
232
|
+
await resetTestSeam();
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
it('host-gate refusal (hostSupportedEnvelopes) projects to node.failed with envelope_contract_violation', async () => {
|
|
236
|
+
if (!(await isEventLogSeamAvailable())) return;
|
|
237
|
+
const runId = `r-cr-host-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
238
|
+
await accept(
|
|
239
|
+
{
|
|
240
|
+
type: 'vendor.unadvertised.kind',
|
|
241
|
+
schemaVersion: 1,
|
|
242
|
+
envelopeId: 'env-cr-proj-host',
|
|
243
|
+
correlationId: `${runId}:n:0:cr-host`,
|
|
244
|
+
payload: {},
|
|
245
|
+
meta: baseMeta,
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
hostSupportedEnvelopes: ['vendor.advertised.only'],
|
|
249
|
+
nodeAllowedKinds: ['vendor.unadvertised.kind'],
|
|
250
|
+
projectTo: { runId, nodeId: 'n' },
|
|
251
|
+
},
|
|
252
|
+
);
|
|
253
|
+
const events = await queryTestEvents(runId, { type: 'node.failed' });
|
|
254
|
+
if (!events.ok || events.events.length === 0) return;
|
|
255
|
+
expect(
|
|
256
|
+
(events.events[0]!.payload.error as { code?: string }).code,
|
|
257
|
+
driver.describe('ai-envelope.md §"Capability handshake integration"', 'host-gate refusal MUST project to node.failed envelope_contract_violation (stacks above node-gate)'),
|
|
258
|
+
).toBe('envelope_contract_violation');
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
// Capability-stacking — backed by the `host.aiEnvelope.supported`
|
|
263
|
+
// flag in the workflow-engine's capability overlay. Per ai-envelope.md
|
|
264
|
+
// §"Capability handshake integration" line 305: capability-gated
|
|
265
|
+
// typeId refusal MUST stack atop envelope-contract refusal. When the
|
|
266
|
+
// host doesn't advertise `host.aiEnvelope: supported`, every
|
|
267
|
+
// envelope/accept call refuses BEFORE the per-envelope contract
|
|
268
|
+
// gates (host-gate, node-gate, schema-floor) fire — observable as
|
|
269
|
+
// `reason: "capability_required"` (NOT "envelope_contract_violation").
|
|
270
|
+
|
|
271
|
+
describe('aiEnvelope.contractRefusal: capability-stacking (FINAL v1.1)', () => {
|
|
272
|
+
afterEach(async () => {
|
|
273
|
+
// Restore overlay after each test so subsequent scenarios see the
|
|
274
|
+
// default advertisement.
|
|
275
|
+
await resetHostCapabilities();
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
it('host.aiEnvelope.supported = false → envelope/accept refuses with capability_required BEFORE envelope contract gates', async () => {
|
|
279
|
+
if (!(await isToggleAvailable())) return; // seam not exposed — soft-skip
|
|
280
|
+
|
|
281
|
+
const toggle = await setHostCapability('host.aiEnvelope.supported', false);
|
|
282
|
+
if (!toggle.ok) return;
|
|
283
|
+
|
|
284
|
+
// Same envelope shape that the existing host-gate scenario uses
|
|
285
|
+
// (line 233-257 above) — the type IS in hostSupportedEnvelopes AND
|
|
286
|
+
// matches nodeAllowedKinds, so the envelope-contract gate would
|
|
287
|
+
// normally accept. The capability gate must fire FIRST and return
|
|
288
|
+
// capability_required regardless.
|
|
289
|
+
const r = await accept(
|
|
290
|
+
{
|
|
291
|
+
type: 'vendor.advertised.kind',
|
|
292
|
+
schemaVersion: 1,
|
|
293
|
+
envelopeId: 'env-cr-capstack-1',
|
|
294
|
+
correlationId: 'r:n:0:cr-capstack',
|
|
295
|
+
payload: {},
|
|
296
|
+
meta: baseMeta,
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
hostSupportedEnvelopes: ['vendor.advertised.kind'],
|
|
300
|
+
nodeAllowedKinds: ['vendor.advertised.kind'],
|
|
301
|
+
},
|
|
302
|
+
);
|
|
303
|
+
if (r.status === 404) return;
|
|
304
|
+
expect(
|
|
305
|
+
r.body.status,
|
|
306
|
+
driver.describe(
|
|
307
|
+
'ai-envelope.md §"Capability handshake integration"',
|
|
308
|
+
'capability-absent host MUST refuse envelope acceptance regardless of host-gate / node-gate match',
|
|
309
|
+
),
|
|
310
|
+
).toBe('invalid');
|
|
311
|
+
expect(
|
|
312
|
+
r.body.reason,
|
|
313
|
+
driver.describe(
|
|
314
|
+
'capabilities.md §"Unsupported capability — refusal contract"',
|
|
315
|
+
'refusal reason MUST be capability_required (NOT envelope_contract_violation) — capability gate stacks above the envelope-contract gate',
|
|
316
|
+
),
|
|
317
|
+
).toBe('capability_required');
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
it('host.aiEnvelope.supported = true → envelope/accept falls through to envelope-contract gates', async () => {
|
|
321
|
+
if (!(await isToggleAvailable())) return;
|
|
322
|
+
const toggle = await setHostCapability('host.aiEnvelope.supported', true);
|
|
323
|
+
if (!toggle.ok) return;
|
|
324
|
+
|
|
325
|
+
// With capability advertised, a normally-rejected envelope (type
|
|
326
|
+
// not in hostSupportedEnvelopes) reaches the envelope-contract
|
|
327
|
+
// gate and refuses with `envelope_contract_violation`, NOT
|
|
328
|
+
// `capability_required`. Proves the capability gate is gated on
|
|
329
|
+
// the flag and doesn't short-circuit the contract path when the
|
|
330
|
+
// capability IS advertised.
|
|
331
|
+
const r = await accept(
|
|
332
|
+
{
|
|
333
|
+
type: 'vendor.unadvertised.kind',
|
|
334
|
+
schemaVersion: 1,
|
|
335
|
+
envelopeId: 'env-cr-capstack-2',
|
|
336
|
+
correlationId: 'r:n:0:cr-capstack-fallthrough',
|
|
337
|
+
payload: {},
|
|
338
|
+
meta: baseMeta,
|
|
339
|
+
},
|
|
340
|
+
{
|
|
341
|
+
hostSupportedEnvelopes: ['vendor.advertised.only'],
|
|
342
|
+
nodeAllowedKinds: ['vendor.unadvertised.kind'],
|
|
343
|
+
},
|
|
344
|
+
);
|
|
345
|
+
if (r.status === 404) return;
|
|
346
|
+
expect(
|
|
347
|
+
r.body.status,
|
|
348
|
+
driver.describe(
|
|
349
|
+
'ai-envelope.md §"Capability handshake integration"',
|
|
350
|
+
'when capability IS advertised, envelope-contract gates run normally',
|
|
351
|
+
),
|
|
352
|
+
).toBe('gated');
|
|
353
|
+
// `gated` is the envelope-contract-gate outcome (host-gate +
|
|
354
|
+
// node-gate); reason text varies. The key contract: status is NOT
|
|
355
|
+
// `invalid` with `capability_required` — the capability layer
|
|
356
|
+
// didn't intercept.
|
|
357
|
+
expect(r.body.reason).not.toBe('capability_required');
|
|
358
|
+
});
|
|
150
359
|
});
|