@openwop/openwop-conformance 2.45.0 → 2.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/spec-artifacts.lock.json +2 -2
- package/package.json +2 -2
- package/requirements.json +132 -127
- package/schemas/CORPUS-STAMP.json +9 -9
- package/src/lib/run-list-walk.ts +65 -0
- package/src/scenarios/agentConfidenceEscalation.test.ts +8 -1
- package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +8 -1
- package/src/scenarios/agentMemoryRedactionContract.test.ts +8 -1
- package/src/scenarios/agentMemoryRoundTrip.test.ts +8 -1
- package/src/scenarios/agentMemoryTtlExpiry.test.ts +8 -1
- package/src/scenarios/agentMessageReducer.test.ts +8 -1
- package/src/scenarios/agentMetadata.test.ts +8 -1
- package/src/scenarios/agentPackCatalog.test.ts +13 -5
- package/src/scenarios/agentPackExport.test.ts +8 -2
- package/src/scenarios/agentPackHandoffSchemaValidation.test.ts +17 -2
- package/src/scenarios/agentPackInstall.test.ts +8 -2
- package/src/scenarios/agentPackProvenance.test.ts +8 -1
- package/src/scenarios/agentReasoningEvents.test.ts +9 -1
- package/src/scenarios/agentReasoningStreaming.test.ts +14 -1
- package/src/scenarios/append-ordering.test.ts +7 -2
- package/src/scenarios/bulk-cancel.test.ts +11 -1
- package/src/scenarios/cap-breach.test.ts +8 -1
- package/src/scenarios/channel-ttl.test.ts +7 -1
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +8 -1
- package/src/scenarios/conversationLifecycle.test.ts +8 -1
- package/src/scenarios/conversationReplayDeterminism.test.ts +8 -2
- package/src/scenarios/conversationVsLegacySuspend.test.ts +8 -1
- package/src/scenarios/cost-attribution.test.ts +17 -4
- package/src/scenarios/debugBundle.test.ts +16 -5
- package/src/scenarios/dispatch-cross-worker-handoff.test.ts +10 -2
- package/src/scenarios/dispatch-output-mapping.test.ts +20 -4
- package/src/scenarios/dispatchLoop.test.ts +7 -1
- package/src/scenarios/durability-poison-exhaustion.test.ts +7 -2
- package/src/scenarios/highConcurrency.test.ts +11 -1
- package/src/scenarios/host-callback-declaration.test.ts +10 -1
- package/src/scenarios/identity-passthrough.test.ts +7 -1
- package/src/scenarios/interrupt-approval.test.ts +25 -5
- package/src/scenarios/interrupt-auth-required-resume.test.ts +9 -3
- package/src/scenarios/interrupt-external-event-correlation.test.ts +9 -2
- package/src/scenarios/interrupt-parent-child-cascade.test.ts +10 -2
- package/src/scenarios/interrupt-quorum-resolution.test.ts +9 -2
- package/src/scenarios/interrupt-token-matrix.test.ts +11 -3
- package/src/scenarios/interruptRace.test.ts +13 -3
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +14 -3
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +15 -3
- package/src/scenarios/multi-node-ordering.test.ts +7 -1
- package/src/scenarios/node-failed-payload-shape.test.ts +7 -1
- package/src/scenarios/orchestratorConservativePath.test.ts +8 -1
- package/src/scenarios/orchestratorDispatch.test.ts +9 -1
- package/src/scenarios/orchestratorTermination.test.ts +8 -1
- package/src/scenarios/owner-subject-echo.test.ts +9 -2
- package/src/scenarios/pause-resume.test.ts +19 -8
- package/src/scenarios/prompt-all-four-kinds-events.test.ts +9 -2
- package/src/scenarios/prompt-end-to-end-events.test.ts +10 -2
- package/src/scenarios/replay-fork-arbitrary.test.ts +11 -4
- package/src/scenarios/restart-during-run.test.ts +7 -1
- package/src/scenarios/secret-leakage-otel-attribute.test.ts +14 -5
- package/src/scenarios/stream-text-fixture.test.ts +8 -2
- package/src/scenarios/streamReconnect.test.ts +8 -2
- package/src/scenarios/subworkflow.test.ts +9 -1
- package/src/scenarios/v2-run-list.test.ts +14 -16
- package/src/scenarios/version-fold.test.ts +8 -2
|
@@ -37,7 +37,7 @@ import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
|
37
37
|
import { pollUntilStatus, pollUntilTerminal } from '../lib/polling.js';
|
|
38
38
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
39
39
|
import { req } from '../lib/requirement-ids.js';
|
|
40
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
40
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
41
41
|
|
|
42
42
|
const FIXTURE =
|
|
43
43
|
(isFixtureAdvertised('conformance-cancellable') && 'conformance-cancellable') ||
|
|
@@ -77,8 +77,13 @@ async function eventTypes(runId: string): Promise<string[]> {
|
|
|
77
77
|
return Array.isArray(events) ? events.map((e) => String((e as { type?: unknown }).type)) : [];
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
|
|
80
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
81
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
82
|
+
(!FIXTURE) ? ['blocked', `no candidate fixture for this scenario is advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
83
|
+
|
|
84
|
+
describe('pause/resume: running → paused → running → terminal', () => {
|
|
81
85
|
it('pause transitions to paused; resume returns the run to running', async () => {
|
|
86
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
82
87
|
const create = await driver.post('/v1/runs', {
|
|
83
88
|
workflowId: FIXTURE!,
|
|
84
89
|
// 3 s, not 30 s: the resumed node.started is a FRESH attempt (runs.md
|
|
@@ -125,8 +130,9 @@ describe.skipIf(SKIP)('pause/resume: running → paused → running → terminal
|
|
|
125
130
|
});
|
|
126
131
|
});
|
|
127
132
|
|
|
128
|
-
describe
|
|
133
|
+
describe('pause/resume: :resume on a non-paused run returns 409', () => {
|
|
129
134
|
it('resuming a running (not paused) run returns 409 with details.runStatus', async () => {
|
|
135
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
130
136
|
const create = await driver.post('/v1/runs', {
|
|
131
137
|
workflowId: FIXTURE!,
|
|
132
138
|
inputs: { delayMs: 30_000 },
|
|
@@ -159,8 +165,9 @@ describe.skipIf(SKIP)('pause/resume: :resume on a non-paused run returns 409', (
|
|
|
159
165
|
});
|
|
160
166
|
});
|
|
161
167
|
|
|
162
|
-
describe
|
|
168
|
+
describe('pause/resume: a second :pause is 409 without a matching Idempotency-Key and 202 with one', () => {
|
|
163
169
|
it(':pause on an already-paused run returns 409 with details.runStatus paused, unless the request carries the original Idempotency-Key', async () => {
|
|
170
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
164
171
|
const create = await driver.post('/v1/runs', {
|
|
165
172
|
workflowId: FIXTURE!,
|
|
166
173
|
inputs: { delayMs: 30_000 },
|
|
@@ -220,8 +227,9 @@ describe.skipIf(SKIP)('pause/resume: a second :pause is 409 without a matching I
|
|
|
220
227
|
});
|
|
221
228
|
});
|
|
222
229
|
|
|
223
|
-
describe
|
|
230
|
+
describe('pause/resume: :pause on a terminal run returns 409', () => {
|
|
224
231
|
it(':pause on a completed/cancelled/failed run MUST return 409', async () => {
|
|
232
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
225
233
|
const create = await driver.post('/v1/runs', {
|
|
226
234
|
workflowId: 'conformance-noop',
|
|
227
235
|
});
|
|
@@ -244,8 +252,9 @@ describe.skipIf(SKIP)('pause/resume: :pause on a terminal run returns 409', () =
|
|
|
244
252
|
});
|
|
245
253
|
});
|
|
246
254
|
|
|
247
|
-
describe
|
|
255
|
+
describe('pause/resume: drain-current-node lets the executing node reach a terminal first', () => {
|
|
248
256
|
it('under drain-current-node node.completed precedes run.paused in the log', async () => {
|
|
257
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
249
258
|
// A short node: drain has something to wait for that finishes inside the
|
|
250
259
|
// poll window. A 30 s node under drain pauses after 30 s — that is the
|
|
251
260
|
// semantic, not a bug (rest-endpoints.md: "lets the executing node reach
|
|
@@ -283,8 +292,9 @@ describe.skipIf(SKIP)('pause/resume: drain-current-node lets the executing node
|
|
|
283
292
|
});
|
|
284
293
|
});
|
|
285
294
|
|
|
286
|
-
describe
|
|
295
|
+
describe('pause/resume: :pause-during-suspend race', () => {
|
|
287
296
|
it(':pause MUST NOT silently override an active interrupt suspend', async () => {
|
|
297
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
288
298
|
// If the host seeds an approval fixture, drive a suspend then attempt
|
|
289
299
|
// :pause. The expected behavior is that :pause either (a) noops with
|
|
290
300
|
// 409 because the run is already waiting-approval (not in a pausable
|
|
@@ -339,8 +349,9 @@ describe.skipIf(SKIP)('pause/resume: :pause-during-suspend race', () => {
|
|
|
339
349
|
// `capabilities.md` §`runs.pauseResume`. When a host advertises
|
|
340
350
|
// `drainPolicies[]`, each advertised value MUST be accepted with 202.
|
|
341
351
|
// Skips entirely when no advertisement is present.
|
|
342
|
-
describe
|
|
352
|
+
describe('pause/resume: drainPolicy discrimination per capabilities advertisement', () => {
|
|
343
353
|
it('every drainPolicy advertised by the host is accepted on :pause', async () => {
|
|
354
|
+
if (SKIP) return softSkip(...GATE_WHY);
|
|
344
355
|
const disco = await driver.get('/.well-known/openwop');
|
|
345
356
|
const drainPolicies =
|
|
346
357
|
capabilityFamily<{ pauseResume?: { drainPolicies?: string[] } }>(disco.json, 'runs')
|
|
@@ -36,7 +36,7 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
36
36
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
37
37
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
38
38
|
import { req } from '../lib/requirement-ids.js';
|
|
39
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
39
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
40
40
|
|
|
41
41
|
const WORKFLOW_ID = 'conformance-prompt-all-four-kinds';
|
|
42
42
|
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
@@ -79,8 +79,14 @@ async function readAllEvents(runId: string): Promise<RunEventDoc[]> {
|
|
|
79
79
|
|
|
80
80
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
83
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
84
|
+
(!process.env.OPENWOP_BASE_URL) ? ['inapplicable', `no target: OPENWOP_BASE_URL is unset`] as const :
|
|
85
|
+
(!isFixtureAdvertised(WORKFLOW_ID)) ? ['blocked', `the \`${WORKFLOW_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
86
|
+
|
|
87
|
+
describe('prompt-all-four-kinds-events: each PromptKind dispatches end-to-end (RFC 0027 §A)', () => {
|
|
83
88
|
it('emits agent.promptResolved + prompt.composed for system, user, schema-hint, and few-shot kinds', async () => {
|
|
89
|
+
if (SKIP_NO_FIXTURE || HTTP_SKIP) return softSkip(...GATE_WHY);
|
|
84
90
|
const d = await readDiscovery();
|
|
85
91
|
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
86
92
|
|
|
@@ -171,6 +177,7 @@ describe.skipIf(SKIP_NO_FIXTURE || HTTP_SKIP)('prompt-all-four-kinds-events: eac
|
|
|
171
177
|
});
|
|
172
178
|
|
|
173
179
|
it('emits the first agent.promptResolved before the first prompt.composed (resolution-precedes-composition ordering)', async () => {
|
|
180
|
+
if (SKIP_NO_FIXTURE || HTTP_SKIP) return softSkip(...GATE_WHY);
|
|
174
181
|
const d = await readDiscovery();
|
|
175
182
|
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
176
183
|
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
@@ -36,7 +36,7 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
36
36
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
37
37
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
38
38
|
import { req } from '../lib/requirement-ids.js';
|
|
39
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
39
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
40
40
|
|
|
41
41
|
const WORKFLOW_ID = 'conformance-prompt-end-to-end';
|
|
42
42
|
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
@@ -82,8 +82,14 @@ async function readAllEvents(runId: string): Promise<RunEventDoc[]> {
|
|
|
82
82
|
|
|
83
83
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
86
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
87
|
+
(!process.env.OPENWOP_BASE_URL) ? ['inapplicable', `no target: OPENWOP_BASE_URL is unset`] as const :
|
|
88
|
+
(!isFixtureAdvertised(WORKFLOW_ID)) ? ['blocked', `the \`${WORKFLOW_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
89
|
+
|
|
90
|
+
describe('prompt-end-to-end-events: real dispatch emits agent.promptResolved + prompt.composed (RFC 0027/0029)', () => {
|
|
86
91
|
it('emits agent.promptResolved with chain[].applied: true for layer "node" when systemPromptRef is set on node.config', async () => {
|
|
92
|
+
if (SKIP_NO_FIXTURE || HTTP_SKIP) return softSkip(...GATE_WHY);
|
|
87
93
|
const d = await readDiscovery();
|
|
88
94
|
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
89
95
|
|
|
@@ -136,6 +142,7 @@ describe.skipIf(SKIP_NO_FIXTURE || HTTP_SKIP)('prompt-end-to-end-events: real di
|
|
|
136
142
|
});
|
|
137
143
|
|
|
138
144
|
it('emits prompt.composed with sha256:<hex64> hash + non-empty composed body for system-kind template', async () => {
|
|
145
|
+
if (SKIP_NO_FIXTURE || HTTP_SKIP) return softSkip(...GATE_WHY);
|
|
139
146
|
const d = await readDiscovery();
|
|
140
147
|
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
141
148
|
|
|
@@ -182,6 +189,7 @@ describe.skipIf(SKIP_NO_FIXTURE || HTTP_SKIP)('prompt-end-to-end-events: real di
|
|
|
182
189
|
});
|
|
183
190
|
|
|
184
191
|
it('emits agent.promptResolved before prompt.composed (causal ordering)', async () => {
|
|
192
|
+
if (SKIP_NO_FIXTURE || HTTP_SKIP) return softSkip(...GATE_WHY);
|
|
185
193
|
const d = await readDiscovery();
|
|
186
194
|
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
187
195
|
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
import { describe, it, expect } from 'vitest';
|
|
32
32
|
import { driver } from '../lib/driver.js';
|
|
33
|
-
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
33
|
+
import { softSkip, blockedDespiteAssertions, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
34
34
|
import { forkDeclined } from '../lib/fork-availability.js';
|
|
35
35
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
36
36
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
@@ -119,10 +119,15 @@ function structuralShape(
|
|
|
119
119
|
}));
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
describe.
|
|
122
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
123
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
124
|
+
(!isFixtureAdvertised(MULTI_NODE_WORKFLOW_ID)) ? ['blocked', `the \`${MULTI_NODE_WORKFLOW_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
125
|
+
|
|
126
|
+
describe(
|
|
123
127
|
'replay-fork-arbitrary: fork from mid-range fromSeq in replay mode reaches terminal',
|
|
124
128
|
() => {
|
|
125
129
|
it('mid-fromSeq replay fork produces a new run that reaches `completed`', async (ctx) => {
|
|
130
|
+
if (SKIP_NO_MULTI) return softSkip(...GATE_WHY);
|
|
126
131
|
const replay = await fetchReplayCapability();
|
|
127
132
|
if (replay?.supported !== true) {
|
|
128
133
|
softSkip('inapplicable', "host does not advertise `replay.supported: true` — the replay contract does not apply to it");
|
|
@@ -197,12 +202,13 @@ describe.skipIf(SKIP_NO_MULTI)(
|
|
|
197
202
|
},
|
|
198
203
|
);
|
|
199
204
|
|
|
200
|
-
describe
|
|
205
|
+
describe(
|
|
201
206
|
'replay-fork-arbitrary: two replay forks at the same mid-range fromSeq yield identical event shape',
|
|
202
207
|
() => {
|
|
203
208
|
it(
|
|
204
209
|
'mid-fromSeq determinism — same source + same fromSeq → identical post-fork events (modulo IDs + timestamps)',
|
|
205
210
|
async (ctx) => {
|
|
211
|
+
if (SKIP_NO_MULTI) return softSkip(...GATE_WHY);
|
|
206
212
|
const replay = await fetchReplayCapability();
|
|
207
213
|
if (replay?.supported !== true) {
|
|
208
214
|
ctx.skip();
|
|
@@ -294,10 +300,11 @@ describe.skipIf(SKIP_NO_MULTI)(
|
|
|
294
300
|
},
|
|
295
301
|
);
|
|
296
302
|
|
|
297
|
-
describe
|
|
303
|
+
describe(
|
|
298
304
|
'replay-fork-arbitrary: fork from mid-range fromSeq in branch mode reaches terminal with overlay',
|
|
299
305
|
() => {
|
|
300
306
|
it('mid-fromSeq branch fork with empty overlay produces a new run that reaches `completed`', async (ctx) => {
|
|
307
|
+
if (SKIP_NO_MULTI) return softSkip(...GATE_WHY);
|
|
301
308
|
const replay = await fetchReplayCapability();
|
|
302
309
|
if (replay?.supported !== true) {
|
|
303
310
|
softSkip('inapplicable', "host does not advertise `replay.supported: true` — the replay contract does not apply to it");
|
|
@@ -37,6 +37,7 @@ import { tmpdir } from 'node:os';
|
|
|
37
37
|
import { join } from 'node:path';
|
|
38
38
|
import { spawnHost, type SpawnedHost } from '../lib/multiProcess.js';
|
|
39
39
|
import { req } from '../lib/requirement-ids.js';
|
|
40
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
40
41
|
|
|
41
42
|
const HOST_PACKAGE_DIR =
|
|
42
43
|
process.env.OPENWOP_RESTART_DURING_RUN_HOST_DIR ?? 'examples/hosts/sqlite';
|
|
@@ -78,12 +79,17 @@ afterEach(async () => {
|
|
|
78
79
|
}
|
|
79
80
|
});
|
|
80
81
|
|
|
81
|
-
describe.
|
|
82
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
83
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
84
|
+
(!RUN_THIS_SCENARIO) ? ['skipped', `the operator did not opt in (OPENWOP_RUN_RESTART_DURING_RUN=1)`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
85
|
+
|
|
86
|
+
describe(
|
|
82
87
|
'restart-during-run: SIGKILL + same-port restart resumes orphaned run',
|
|
83
88
|
() => {
|
|
84
89
|
it(
|
|
85
90
|
'mid-run SIGKILL on the host process; restart at the same port + DB resumes to terminal',
|
|
86
91
|
async () => {
|
|
92
|
+
if (!RUN_THIS_SCENARIO) return softSkip(...GATE_WHY);
|
|
87
93
|
workdir = mkdtempSync(join(tmpdir(), 'openwop-restart-during-run-'));
|
|
88
94
|
const dbPath = join(workdir, 'openwop-host.sqlite');
|
|
89
95
|
|
|
@@ -58,7 +58,7 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
58
58
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
59
59
|
import { getCollector, waitForRunSpans } from '../lib/otel-collector.js';
|
|
60
60
|
import { req } from '../lib/requirement-ids.js';
|
|
61
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
61
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
62
62
|
|
|
63
63
|
/**
|
|
64
64
|
* Callback-shaped: the host exports OTLP spans to the suite's collector, which scans them for the BYOK canary.
|
|
@@ -102,10 +102,16 @@ async function startByokRun(): Promise<string | null> {
|
|
|
102
102
|
return (create.json as { runId: string }).runId;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
|
|
105
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
106
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
107
|
+
(!process.env.OPENWOP_BASE_URL) ? ['inapplicable', `no target: OPENWOP_BASE_URL is unset`] as const :
|
|
108
|
+
(!isFixtureAdvertised(BYOK_WORKFLOW_ID)) ? ['blocked', `the \`${BYOK_WORKFLOW_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
109
|
+
|
|
110
|
+
describe(
|
|
106
111
|
'secret-leakage-otel-attribute: OTel span scrape (RFC 0034 §B)',
|
|
107
112
|
() => {
|
|
108
113
|
it('NO OTel span attribute MUST contain the BYOK canary plaintext for a run that resolved it', async (ctx) => {
|
|
114
|
+
if (HTTP_SKIP || FIXTURE_SKIP) return softSkip(...GATE_WHY);
|
|
109
115
|
if (!CANARY_VALUE) {
|
|
110
116
|
ctx.skip();
|
|
111
117
|
return softSkip('blocked', 'precondition not met — `!CANARY_VALUE` returned early (seam, prior step, or fixture unavailable)');
|
|
@@ -171,10 +177,11 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
171
177
|
},
|
|
172
178
|
);
|
|
173
179
|
|
|
174
|
-
describe
|
|
180
|
+
describe(
|
|
175
181
|
'secret-leakage-debug-bundle-otel: debug-bundle export scrape (RFC 0034 §B)',
|
|
176
182
|
() => {
|
|
177
183
|
it('NO debug-bundle field MUST contain the BYOK canary plaintext for a run that resolved it', async (ctx) => {
|
|
184
|
+
if (HTTP_SKIP || FIXTURE_SKIP) return softSkip(...GATE_WHY);
|
|
178
185
|
if (!CANARY_VALUE) {
|
|
179
186
|
ctx.skip();
|
|
180
187
|
return softSkip('blocked', 'precondition not met — `!CANARY_VALUE` returned early (seam, prior step, or fixture unavailable)');
|
|
@@ -216,7 +223,7 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
216
223
|
},
|
|
217
224
|
);
|
|
218
225
|
|
|
219
|
-
describe
|
|
226
|
+
describe(
|
|
220
227
|
'secret-leakage-otel-attribute: real OTLP export scrape (collector-side)',
|
|
221
228
|
() => {
|
|
222
229
|
// Distinct from the scrape-seam probe above: this asserts against what
|
|
@@ -228,6 +235,7 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
228
235
|
// attributes" gap. Gated on the in-process collector being active
|
|
229
236
|
// (`OPENWOP_OTEL_COLLECTOR=true` + the host configured to export to it).
|
|
230
237
|
it('NO real-exported OTel span/metric attribute MUST contain the BYOK canary plaintext', async (ctx) => {
|
|
238
|
+
if (HTTP_SKIP || FIXTURE_SKIP) return softSkip(...GATE_WHY);
|
|
231
239
|
const collector = getCollector();
|
|
232
240
|
if (!collector || !CANARY_VALUE) {
|
|
233
241
|
ctx.skip();
|
|
@@ -267,10 +275,11 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
267
275
|
},
|
|
268
276
|
);
|
|
269
277
|
|
|
270
|
-
describe
|
|
278
|
+
describe(
|
|
271
279
|
'secret-leakage-otel-attribute: advertisement-shape probe (RFC 0034 §A)',
|
|
272
280
|
() => {
|
|
273
281
|
it('when secrets.supported is true, observability.testSeams advertisements MUST be boolean if present', async (ctx) => {
|
|
282
|
+
if (HTTP_SKIP || FIXTURE_SKIP) return softSkip(...GATE_WHY);
|
|
274
283
|
const d = await readDiscovery();
|
|
275
284
|
if (capabilityFamily<{ supported?: unknown }>(d, 'secrets')?.supported !== true) {
|
|
276
285
|
ctx.skip();
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
*/
|
|
33
33
|
|
|
34
34
|
import { describe, it, expect } from 'vitest';
|
|
35
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
35
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
36
36
|
import { driver, type OpenWOPResponse } from '../lib/driver.js';
|
|
37
37
|
import { subscribe } from '../lib/sse.js';
|
|
38
38
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
@@ -87,8 +87,13 @@ async function isStreamTextMockAdvertised(): Promise<boolean> {
|
|
|
87
87
|
return Array.isArray(testing?.mockProviders) && testing.mockProviders.includes('stream-text');
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
|
|
90
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
91
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
92
|
+
(!isFixtureAdvertised(FIXTURE_ID)) ? ['blocked', `the \`${FIXTURE_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
93
|
+
|
|
94
|
+
describe('stream-text-fixture: messages-mode fold through the stream-text mock (F1)', () => {
|
|
91
95
|
it('delivers the mocked tokens in order, terminates with isLast + meta, and server-closes', async () => {
|
|
96
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
92
97
|
if (!(await isStreamTextMockAdvertised())) {
|
|
93
98
|
// eslint-disable-next-line no-console
|
|
94
99
|
console.warn(
|
|
@@ -191,6 +196,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('stream-text-fixture: messages-mode fold throug
|
|
|
191
196
|
});
|
|
192
197
|
|
|
193
198
|
it('an unknown mockProvider.id is rejected with 400 unsupported_mock_provider', async () => {
|
|
199
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
194
200
|
if (!(await isStreamTextMockAdvertised())) {
|
|
195
201
|
// eslint-disable-next-line no-console
|
|
196
202
|
console.warn('[stream-text-fixture] stream-text mock not advertised; skipping negative leg');
|
|
@@ -22,7 +22,7 @@ import { subscribe, type SseEvent } from '../lib/sse.js';
|
|
|
22
22
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
23
23
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
24
24
|
import { req } from '../lib/requirement-ids.js';
|
|
25
|
-
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
25
|
+
import { softSkip, blockedDespiteAssertions, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
26
26
|
|
|
27
27
|
const WORKFLOW_ID = 'conformance-cancellable';
|
|
28
28
|
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
@@ -50,10 +50,15 @@ function getSeq(event: SseEvent): number | null {
|
|
|
50
50
|
return null;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
54
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
55
|
+
(!isFixtureAdvertised(WORKFLOW_ID)) ? ['blocked', `the \`${WORKFLOW_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
56
|
+
|
|
57
|
+
describe('stream-reconnect: Last-Event-ID resume per spec/v1/stream-modes.md', () => {
|
|
54
58
|
it(
|
|
55
59
|
'reconnect with Last-Event-ID resumes without loss or duplication',
|
|
56
60
|
async () => {
|
|
61
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
57
62
|
// Phase 1: kick off a long-running run.
|
|
58
63
|
const create = await driver.post('/v1/runs', {
|
|
59
64
|
workflowId: WORKFLOW_ID,
|
|
@@ -119,6 +124,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('stream-reconnect: Last-Event-ID resume per spe
|
|
|
119
124
|
it(
|
|
120
125
|
'reconnect with Last-Event-ID equal to terminal seq closes immediately',
|
|
121
126
|
async () => {
|
|
127
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
122
128
|
// Quick run, observe terminal seq, then attempt a reconnect after
|
|
123
129
|
// terminal — server SHOULD close immediately with no events.
|
|
124
130
|
const create = await driver.post('/v1/runs', { workflowId: 'conformance-noop' });
|
|
@@ -21,6 +21,7 @@ import { driver } from '../lib/driver.js';
|
|
|
21
21
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
22
22
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
23
23
|
import { req } from '../lib/requirement-ids.js';
|
|
24
|
+
import { softSkip, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
24
25
|
|
|
25
26
|
const PARENT_WORKFLOW_ID = 'conformance-subworkflow-parent';
|
|
26
27
|
const CHILD_WORKFLOW_ID = 'conformance-subworkflow-child';
|
|
@@ -44,8 +45,14 @@ interface RunEvent {
|
|
|
44
45
|
readonly payload?: unknown;
|
|
45
46
|
}
|
|
46
47
|
|
|
47
|
-
|
|
48
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
49
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
50
|
+
(!isFixtureAdvertised(PARENT_WORKFLOW_ID)) ? ['blocked', `the \`${PARENT_WORKFLOW_ID}\` fixture is not advertised`] as const :
|
|
51
|
+
(!isFixtureAdvertised(CHILD_WORKFLOW_ID)) ? ['blocked', `the \`${CHILD_WORKFLOW_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
52
|
+
|
|
53
|
+
describe('subworkflow: conformance-subworkflow-parent dispatches child + completes', () => {
|
|
48
54
|
it('parent run reaches terminal completed and child variable is propagated via outputMapping', async () => {
|
|
55
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
49
56
|
const create = await driver.post('/v1/runs', { workflowId: PARENT_WORKFLOW_ID });
|
|
50
57
|
expect(create.status).toBe(201);
|
|
51
58
|
const parentRunId = (create.json as { runId: string }).runId;
|
|
@@ -68,6 +75,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('subworkflow: conformance-subworkflow-parent di
|
|
|
68
75
|
});
|
|
69
76
|
|
|
70
77
|
it('child run is created with parent linkage fields and reaches terminal completed', async () => {
|
|
78
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
71
79
|
const create = await driver.post('/v1/runs', { workflowId: PARENT_WORKFLOW_ID });
|
|
72
80
|
const parentRunId = (create.json as { runId: string }).runId;
|
|
73
81
|
|
|
@@ -21,10 +21,13 @@ import { gateFamily, v2Discovery, v2Validator } from '../lib/v2.js';
|
|
|
21
21
|
import { readErrorCode } from '../lib/error-envelope.js';
|
|
22
22
|
import { softSkip } from '../lib/soft-skip.js';
|
|
23
23
|
import { req } from '../lib/requirement-ids.js';
|
|
24
|
+
import { walkRunList, type FetchPage, type RunListPage, type WalkResult } from '../lib/run-list-walk.js';
|
|
24
25
|
|
|
25
26
|
const DOC = 'spec/v2/core/runs.md §List';
|
|
26
27
|
const NOOP_WORKFLOW_ID = 'conformance-noop';
|
|
27
28
|
const MAX_PAGES = 50;
|
|
29
|
+
/** Pages still followed after the created runs are seen, so a host-minted cursor is exercised. */
|
|
30
|
+
const TAIL_PAGES = 2;
|
|
28
31
|
|
|
29
32
|
async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> {
|
|
30
33
|
try { return await fn(); } catch { return null; }
|
|
@@ -40,22 +43,16 @@ async function createRun(): Promise<{ runId: string } | { reason: string }> {
|
|
|
40
43
|
|
|
41
44
|
function tenantOf(runId: string): string { return runId.slice(0, runId.indexOf('/')); }
|
|
42
45
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
async function walk(query: string, maxPageSize: number): Promise<{ ids: string[]; pages: Page[]; reason?: string }> {
|
|
47
|
-
const ids: string[] = []; const pages: Page[] = []; let cursor: string | undefined;
|
|
48
|
-
for (let i = 0; i < MAX_PAGES; i++) {
|
|
46
|
+
/** The newest-first, stop-when-found walk (`lib/run-list-walk.ts`) over this host's `GET /runs`. */
|
|
47
|
+
async function walk(query: string, maxPageSize: number, want: string[]): Promise<WalkResult> {
|
|
48
|
+
const fetchPage: FetchPage = async (cursor) => {
|
|
49
49
|
const q = `${query}${query ? '&' : '?'}limit=${maxPageSize}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`;
|
|
50
50
|
const res = await http(() => driver.get(`/runs${q}`));
|
|
51
|
-
if (res === null) return {
|
|
52
|
-
if (res.status !== 200) return {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
cursor = page.nextCursor;
|
|
57
|
-
}
|
|
58
|
-
return { ids, pages };
|
|
51
|
+
if (res === null) return { reason: `GET /runs${q} unreachable (fetch failed)` };
|
|
52
|
+
if (res.status !== 200) return { reason: `GET /runs${q} answered ${res.status} ${readErrorCode(res.json) ?? ''}` };
|
|
53
|
+
return { page: res.json as RunListPage };
|
|
54
|
+
};
|
|
55
|
+
return walkRunList(fetchPage, { want, maxPages: MAX_PAGES, tailPages: TAIL_PAGES });
|
|
59
56
|
}
|
|
60
57
|
|
|
61
58
|
describe('RFC 0182 — run-list (gated on runList)', () => {
|
|
@@ -68,7 +65,7 @@ describe('RFC 0182 — run-list (gated on runList)', () => {
|
|
|
68
65
|
const a = await createRun(); if ('reason' in a) return softSkip('blocked', a.reason);
|
|
69
66
|
const b = await createRun(); if ('reason' in b) return softSkip('blocked', b.reason);
|
|
70
67
|
const tenant = tenantOf(a.runId);
|
|
71
|
-
const walked = await walk('', maxPageSize);
|
|
68
|
+
const walked = await walk('', maxPageSize, [a.runId, b.runId]);
|
|
72
69
|
if (walked.reason && walked.pages.length === 0) return softSkip('blocked', walked.reason);
|
|
73
70
|
const validate = v2Validator('run-list-response');
|
|
74
71
|
for (const page of walked.pages) {
|
|
@@ -77,6 +74,7 @@ describe('RFC 0182 — run-list (gated on runList)', () => {
|
|
|
77
74
|
expect((page.runs as unknown[]).length <= maxPageSize, req('openwop.requirement.0182.run-list.tenant-scoped', DOC, `a page MUST NOT exceed the advertised runList.maxPageSize (${maxPageSize}); got ${(page.runs as unknown[]).length}`)).toBe(true);
|
|
78
75
|
}
|
|
79
76
|
expect(walked.ids.includes(a.runId) && walked.ids.includes(b.runId), req('openwop.requirement.0182.run-list.tenant-scoped', DOC, `a run the caller just created MUST appear in its unfiltered list — created ${a.runId} and ${b.runId}; walked ${walked.ids.length} id(s) over ${walked.pages.length} page(s)${walked.reason ? ` (${walked.reason})` : ''}`)).toBe(true);
|
|
77
|
+
expect(walked.ids.indexOf(b.runId) < walked.ids.indexOf(a.runId), req('openwop.requirement.0182.run-list.tenant-scoped', DOC, `the list MUST be newest first — ${b.runId} was created after ${a.runId}, so it MUST be listed before it`)).toBe(true);
|
|
80
78
|
const foreign = walked.ids.filter((id) => tenantOf(id) !== tenant);
|
|
81
79
|
expect(foreign, req('openwop.requirement.0182.run-list.tenant-scoped', 'spec/v2/core/identity.md §5', `every runId in the list MUST carry the caller's tenant segment (${tenant}) — the list is tenant-scoped by construction; found ${foreign.length} other(s): ${foreign.slice(0, 3).join(', ')}`)).toEqual([]);
|
|
82
80
|
});
|
|
@@ -98,7 +96,7 @@ describe('RFC 0182 — run-list (gated on runList)', () => {
|
|
|
98
96
|
if (!filters.includes('workflowId')) return softSkip('inapplicable', 'runList.filters does not name workflowId — the filter leg has no obligation');
|
|
99
97
|
const maxPageSize = typeof fam['maxPageSize'] === 'number' ? fam['maxPageSize'] : 1;
|
|
100
98
|
const c = await createRun(); if ('reason' in c) return softSkip('blocked', c.reason);
|
|
101
|
-
const walked = await walk(`?workflowId=${encodeURIComponent(NOOP_WORKFLOW_ID)}`, maxPageSize);
|
|
99
|
+
const walked = await walk(`?workflowId=${encodeURIComponent(NOOP_WORKFLOW_ID)}`, maxPageSize, [c.runId]);
|
|
102
100
|
if (walked.reason && walked.pages.length === 0) return softSkip('blocked', walked.reason);
|
|
103
101
|
const others = walked.pages.flatMap((p) => (Array.isArray(p.runs) ? p.runs : [])).filter((r) => (r as { workflowId?: unknown }).workflowId !== NOOP_WORKFLOW_ID);
|
|
104
102
|
expect(others.length, req('openwop.requirement.0182.run-list.filter-exact', DOC, `a workflowId filter MUST be exact — every listed run's workflowId MUST equal ${NOOP_WORKFLOW_ID}; found ${others.length} other(s)`)).toBe(0);
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
36
|
import { describe, it, expect } from 'vitest';
|
|
37
|
-
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
37
|
+
import { softSkip, blockedDespiteAssertions, type SoftSkipKind } from '../lib/soft-skip.js';
|
|
38
38
|
import { driver, type OpenWOPResponse } from '../lib/driver.js';
|
|
39
39
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
40
40
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
@@ -77,8 +77,13 @@ async function createForcedRun(version: number): Promise<OpenWOPResponse> {
|
|
|
77
77
|
);
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
|
|
80
|
+
/** Why the gate below holds, as RFC 0148 §A names it (openwop#1686: a describe-level skip recorded no disposition). */
|
|
81
|
+
const GATE_WHY: readonly [SoftSkipKind, string] =
|
|
82
|
+
(!isFixtureAdvertised(FIXTURE_ID)) ? ['blocked', `the \`${FIXTURE_ID}\` fixture is not advertised`] as const : ['blocked', 'the gate held for no named reason'] as const;
|
|
83
|
+
|
|
84
|
+
describe('version-fold: forced engine versions fold-best-effort across the advertised range', () => {
|
|
81
85
|
it('each version in [min, current, max] completes + snapshot and event log stay readable', async () => {
|
|
86
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
82
87
|
const seam = await readForceSeam();
|
|
83
88
|
if (seam === null) {
|
|
84
89
|
// eslint-disable-next-line no-console
|
|
@@ -163,6 +168,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('version-fold: forced engine versions fold-best
|
|
|
163
168
|
});
|
|
164
169
|
|
|
165
170
|
it('a forced version outside the advertised range is rejected with 400 unsupported_force_engine_version', async () => {
|
|
171
|
+
if (SKIP_NO_FIXTURE) return softSkip(...GATE_WHY);
|
|
166
172
|
const seam = await readForceSeam();
|
|
167
173
|
if (seam === null) {
|
|
168
174
|
// eslint-disable-next-line no-console
|