@openwop/openwop-conformance 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +31 -6
- package/api/grpc/openwop.proto +251 -0
- package/api/openapi.yaml +109 -3
- package/coverage.md +48 -9
- package/fixtures/conformance-configurable-schema.json +39 -0
- package/fixtures/conformance-subworkflow-parent.json +1 -1
- package/fixtures/conformance-wasm-pack-memory-cap-breach.json +23 -0
- package/fixtures/openwop-smoke-byok-roundtrip.json +25 -0
- package/fixtures.md +21 -0
- package/package.json +3 -1
- package/schemas/README.md +4 -0
- package/schemas/audit-verify-result.schema.json +90 -0
- package/schemas/capabilities.schema.json +342 -1
- package/schemas/node-pack-manifest.schema.json +4 -4
- package/schemas/pack-lockfile.schema.json +92 -0
- package/schemas/registry-version-manifest.schema.json +145 -0
- package/schemas/run-event-payloads.schema.json +20 -4
- package/schemas/run-event.schema.json +2 -1
- package/schemas/security-advisory.schema.json +109 -0
- package/src/lib/a2a-fake-peer.ts +143 -56
- package/src/lib/behavior-gate.ts +107 -0
- package/src/lib/env.ts +37 -0
- package/src/lib/grpc-framing.test.ts +96 -0
- package/src/lib/grpc-framing.ts +76 -0
- package/src/lib/oidc-issuer.test.ts +328 -0
- package/src/lib/oidc-issuer.ts +241 -0
- package/src/lib/otel-collector-grpc.test.ts +191 -0
- package/src/lib/otel-collector.test.ts +303 -0
- package/src/lib/otel-collector.ts +318 -14
- package/src/lib/otlp-protobuf.test.ts +461 -0
- package/src/lib/otlp-protobuf.ts +529 -0
- package/src/scenarios/a2a-task-roundtrip.test.ts +147 -28
- package/src/scenarios/agentConfidenceEscalation.test.ts +1 -0
- package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +1 -0
- package/src/scenarios/agentMemoryRedactionContract.test.ts +1 -0
- package/src/scenarios/agentMemoryRoundTrip.test.ts +1 -0
- package/src/scenarios/agentMemoryTtlExpiry.test.ts +1 -0
- package/src/scenarios/agentMessageReducer.test.ts +1 -0
- package/src/scenarios/agentMetadata.test.ts +1 -0
- package/src/scenarios/agentPackExport.test.ts +1 -0
- package/src/scenarios/agentPackInstall.test.ts +1 -0
- package/src/scenarios/agentPackProvenance.test.ts +1 -0
- package/src/scenarios/audit-log-integrity.test.ts +3 -6
- package/src/scenarios/auth-api-key-rotation.test.ts +182 -0
- package/src/scenarios/auth-mtls.test.ts +274 -0
- package/src/scenarios/auth-oauth2-client-credentials.test.ts +259 -0
- package/src/scenarios/auth-oidc-user-bearer.test.ts +361 -0
- package/src/scenarios/bulk-cancel.test.ts +111 -0
- package/src/scenarios/configurable-schema.test.ts +48 -0
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +1 -0
- package/src/scenarios/conversationLifecycle.test.ts +1 -0
- package/src/scenarios/conversationReplayDeterminism.test.ts +1 -0
- package/src/scenarios/conversationVsLegacySuspend.test.ts +1 -0
- package/src/scenarios/debug-bundle-truncation.test.ts +95 -0
- package/src/scenarios/discovery.test.ts +183 -0
- package/src/scenarios/http-client-ssrf.test.ts +71 -0
- package/src/scenarios/idempotency.test.ts +6 -0
- package/src/scenarios/idempotencyRetry.test.ts +3 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +205 -34
- package/src/scenarios/mcp-toolcall-redaction.test.ts +66 -0
- package/src/scenarios/memory-compaction-event-emitted.test.ts +121 -0
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +116 -0
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +127 -0
- package/src/scenarios/metric-emission.test.ts +113 -0
- package/src/scenarios/multi-region-idempotency.test.ts +39 -4
- package/src/scenarios/orchestratorConservativePath.test.ts +1 -0
- package/src/scenarios/orchestratorDispatch.test.ts +1 -0
- package/src/scenarios/orchestratorTermination.test.ts +1 -0
- package/src/scenarios/otel-emission-grpc.test.ts +98 -0
- package/src/scenarios/otel-trace-propagation-subworkflow.test.ts +139 -0
- package/src/scenarios/pause-resume.test.ts +119 -0
- package/src/scenarios/production-backpressure.test.ts +342 -0
- package/src/scenarios/production-retention-expiry.test.ts +164 -0
- package/src/scenarios/registry-public.test.ts +222 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +35 -0
- package/src/scenarios/replay-retention-expiry.test.ts +178 -0
- package/src/scenarios/restart-during-run.test.ts +177 -0
- package/src/scenarios/spec-corpus-validity.test.ts +59 -26
- package/src/scenarios/staleClaim.test.ts +3 -0
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +67 -10
- package/src/scenarios/wasm-pack-memory-cap.test.ts +64 -9
- package/src/scenarios/webhook-negative.test.ts +90 -0
- package/src/scenarios/webhook-signed-delivery.test.ts +178 -0
- package/src/setup.ts +25 -1
- package/vitest.config.ts +5 -1
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Track 11 close-out: cross-run trace-context propagation across
|
|
3
|
+
* `core.subWorkflow` invocation.
|
|
4
|
+
*
|
|
5
|
+
* `otel-trace-propagation.test.ts` verifies that a single run's spans
|
|
6
|
+
* inherit an inbound `traceparent`'s traceId. This scenario closes the
|
|
7
|
+
* remaining gap (`conformance/coverage.md` row 52: "Cross-host
|
|
8
|
+
* propagation across `core.subWorkflow` invocation"): when a parent
|
|
9
|
+
* run with a known inbound traceparent dispatches a child run via
|
|
10
|
+
* `core.subWorkflow`, the CHILD run's emitted spans MUST also share
|
|
11
|
+
* the parent's traceId — distributed traces stitch across the
|
|
12
|
+
* sub-workflow boundary without operator-side correlation hacks.
|
|
13
|
+
*
|
|
14
|
+
* Operator-tier value: in production deployments, a sub-workflow may
|
|
15
|
+
* execute on a different host instance (`core.subWorkflow` is a
|
|
16
|
+
* dispatch boundary, not necessarily an in-process call). The
|
|
17
|
+
* traceparent-propagation contract guarantees the operator's OTel
|
|
18
|
+
* backend can render parent + child as one trace tree even when
|
|
19
|
+
* they're on separate hosts.
|
|
20
|
+
*
|
|
21
|
+
* Skip conditions:
|
|
22
|
+
* - Collector disabled.
|
|
23
|
+
* - Host doesn't advertise `capabilities.observability`.
|
|
24
|
+
* - `conformance-subworkflow-parent` fixture not advertised (host
|
|
25
|
+
* doesn't implement `core.subWorkflow`).
|
|
26
|
+
*
|
|
27
|
+
* @see spec/v1/observability.md §"Trace context propagation"
|
|
28
|
+
* @see spec/v1/node-packs.md §`core.subWorkflow`
|
|
29
|
+
* @see conformance/coverage.md row 52
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { describe, it, expect } from 'vitest';
|
|
33
|
+
import { driver } from '../lib/driver.js';
|
|
34
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
35
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
36
|
+
import { getCollector, waitForRunSpans } from '../lib/otel-collector.js';
|
|
37
|
+
|
|
38
|
+
const PARENT_FIXTURE = 'conformance-subworkflow-parent';
|
|
39
|
+
|
|
40
|
+
interface RunEvent {
|
|
41
|
+
type: string;
|
|
42
|
+
nodeId?: string;
|
|
43
|
+
payload?: { outputs?: { childRunId?: string } };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function makeTraceparent(): { header: string; traceId: string } {
|
|
47
|
+
// W3C format: 00-<32 hex traceId>-<16 hex spanId>-01.
|
|
48
|
+
// Use a distinct id from the parent-only scenario so collector
|
|
49
|
+
// matching is unambiguous when both scenarios run back-to-back.
|
|
50
|
+
const traceId = '7c3e51b9d2a04e6f8b1c0d2e3f4a5b6c';
|
|
51
|
+
const spanId = '00f067aa0ba902b7';
|
|
52
|
+
return { header: `00-${traceId}-${spanId}-01`, traceId };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function isObservabilityAdvertised(): Promise<boolean> {
|
|
56
|
+
try {
|
|
57
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
58
|
+
const caps = (disco.json as { capabilities?: { observability?: unknown } }).capabilities ?? {};
|
|
59
|
+
return caps.observability !== undefined;
|
|
60
|
+
} catch {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
describe('otel-trace-propagation-subworkflow: traceparent threads parent → child via core.subWorkflow', () => {
|
|
66
|
+
it('child run spans inherit the parent run\'s inbound traceId', async () => {
|
|
67
|
+
if (!getCollector()) {
|
|
68
|
+
// eslint-disable-next-line no-console
|
|
69
|
+
console.warn('[otel-trace-propagation-subworkflow] collector not started; skipping');
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (!isFixtureAdvertised(PARENT_FIXTURE)) {
|
|
73
|
+
// eslint-disable-next-line no-console
|
|
74
|
+
console.warn(`[otel-trace-propagation-subworkflow] ${PARENT_FIXTURE} not advertised; skipping`);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
if (!(await isObservabilityAdvertised())) {
|
|
78
|
+
// eslint-disable-next-line no-console
|
|
79
|
+
console.warn('[otel-trace-propagation-subworkflow] capabilities.observability not advertised; skipping');
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const collector = getCollector()!;
|
|
84
|
+
collector.reset();
|
|
85
|
+
|
|
86
|
+
const { header, traceId } = makeTraceparent();
|
|
87
|
+
const create = await driver.post(
|
|
88
|
+
'/v1/runs',
|
|
89
|
+
{ workflowId: PARENT_FIXTURE },
|
|
90
|
+
{ headers: { traceparent: header } },
|
|
91
|
+
);
|
|
92
|
+
expect(create.status).toBe(201);
|
|
93
|
+
const parentRunId = (create.json as { runId: string }).runId;
|
|
94
|
+
|
|
95
|
+
await pollUntilTerminal(parentRunId, { timeoutMs: 30_000 });
|
|
96
|
+
|
|
97
|
+
// Walk the parent's event log to discover the child run id.
|
|
98
|
+
const eventsRes = await driver.get(
|
|
99
|
+
`/v1/runs/${encodeURIComponent(parentRunId)}/events/poll?lastSequence=0&timeout=1`,
|
|
100
|
+
);
|
|
101
|
+
expect(eventsRes.status).toBe(200);
|
|
102
|
+
const events = (eventsRes.json as { events?: RunEvent[] } | undefined)?.events ?? [];
|
|
103
|
+
|
|
104
|
+
const subwfCompleted = events.find(
|
|
105
|
+
(e) => e.type === 'node.completed' && e.nodeId === 'subwf-call',
|
|
106
|
+
);
|
|
107
|
+
expect(subwfCompleted, driver.describe(
|
|
108
|
+
'node-packs.md §core.subWorkflow',
|
|
109
|
+
'parent event log MUST include node.completed for the subwf-call node',
|
|
110
|
+
)).toBeDefined();
|
|
111
|
+
|
|
112
|
+
const childRunId = subwfCompleted?.payload?.outputs?.childRunId;
|
|
113
|
+
expect(typeof childRunId, driver.describe(
|
|
114
|
+
'node-packs.md §core.subWorkflow outputSchema',
|
|
115
|
+
'subwf-call node.completed payload MUST carry outputs.childRunId',
|
|
116
|
+
)).toBe('string');
|
|
117
|
+
|
|
118
|
+
// Both parent + child spans MUST share the inbound traceId.
|
|
119
|
+
const parentSpans = await waitForRunSpans(parentRunId, { timeoutMs: 10_000, minCount: 1 });
|
|
120
|
+
const childSpans = await waitForRunSpans(childRunId!, { timeoutMs: 10_000, minCount: 1 });
|
|
121
|
+
|
|
122
|
+
expect(parentSpans.length, 'collector MUST receive ≥1 span for the parent run').toBeGreaterThan(0);
|
|
123
|
+
expect(childSpans.length, 'collector MUST receive ≥1 span for the child run').toBeGreaterThan(0);
|
|
124
|
+
|
|
125
|
+
const wantTrace = traceId.toLowerCase();
|
|
126
|
+
|
|
127
|
+
const parentMatching = parentSpans.filter((s) => s.traceId.toLowerCase() === wantTrace);
|
|
128
|
+
expect(parentMatching.length, driver.describe(
|
|
129
|
+
'observability.md §"Trace context propagation"',
|
|
130
|
+
'parent-run spans MUST share the inbound traceparent traceId',
|
|
131
|
+
)).toBeGreaterThan(0);
|
|
132
|
+
|
|
133
|
+
const childMatching = childSpans.filter((s) => s.traceId.toLowerCase() === wantTrace);
|
|
134
|
+
expect(childMatching.length, driver.describe(
|
|
135
|
+
'observability.md §"Trace context propagation" + node-packs.md §core.subWorkflow',
|
|
136
|
+
'child-run spans dispatched via core.subWorkflow MUST inherit the parent run\'s traceId so distributed traces stitch across the dispatch boundary',
|
|
137
|
+
)).toBeGreaterThan(0);
|
|
138
|
+
});
|
|
139
|
+
});
|
|
@@ -107,3 +107,122 @@ describe.skipIf(SKIP)('pause/resume: :resume on a non-paused run returns 409', (
|
|
|
107
107
|
});
|
|
108
108
|
});
|
|
109
109
|
});
|
|
110
|
+
|
|
111
|
+
describe.skipIf(SKIP)('pause/resume: pause is idempotent when already paused', () => {
|
|
112
|
+
it(':pause on an already-paused run is a no-op (200/202) — idempotent', async () => {
|
|
113
|
+
const create = await driver.post('/v1/runs', {
|
|
114
|
+
workflowId: FIXTURE!,
|
|
115
|
+
inputs: { delaySeconds: 30 },
|
|
116
|
+
});
|
|
117
|
+
expect(create.status).toBe(201);
|
|
118
|
+
const runId = (create.json as { runId: string }).runId;
|
|
119
|
+
await pollUntilStatus(runId, 'running', { timeoutMs: 10_000 });
|
|
120
|
+
|
|
121
|
+
const first = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {});
|
|
122
|
+
if (first.status === 404) {
|
|
123
|
+
await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
|
|
124
|
+
reason: 'conformance-cleanup',
|
|
125
|
+
});
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
expect([200, 202]).toContain(first.status);
|
|
129
|
+
await pollUntilStatus(runId, 'paused', { timeoutMs: 10_000 });
|
|
130
|
+
|
|
131
|
+
// Idempotent second :pause — MUST NOT 409 just because the run is
|
|
132
|
+
// already paused. 200/202 are both acceptable per the additive
|
|
133
|
+
// contract; 409 would force callers to read state before calling.
|
|
134
|
+
const second = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {});
|
|
135
|
+
expect(
|
|
136
|
+
[200, 202].includes(second.status),
|
|
137
|
+
driver.describe(
|
|
138
|
+
'rest-endpoints.md POST /v1/runs/{runId}:pause',
|
|
139
|
+
':pause on an already-paused run MUST be idempotent (200/202), not 409',
|
|
140
|
+
),
|
|
141
|
+
).toBe(true);
|
|
142
|
+
|
|
143
|
+
await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
|
|
144
|
+
reason: 'conformance-cleanup',
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
describe.skipIf(SKIP)('pause/resume: :pause on a terminal run returns 409', () => {
|
|
150
|
+
it(':pause on a completed/cancelled/failed run MUST return 409', async () => {
|
|
151
|
+
const create = await driver.post('/v1/runs', {
|
|
152
|
+
workflowId: 'conformance-noop',
|
|
153
|
+
});
|
|
154
|
+
if (create.status !== 201) return; // conformance-noop not seeded; skip cleanly
|
|
155
|
+
const runId = (create.json as { runId: string }).runId;
|
|
156
|
+
await pollUntilTerminal(runId, { timeoutMs: 10_000 });
|
|
157
|
+
|
|
158
|
+
const pause = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {});
|
|
159
|
+
if (pause.status === 404) return;
|
|
160
|
+
expect(pause.status, driver.describe(
|
|
161
|
+
'rest-endpoints.md POST /v1/runs/{runId}:pause',
|
|
162
|
+
':pause on a terminal run MUST return 409',
|
|
163
|
+
)).toBe(409);
|
|
164
|
+
|
|
165
|
+
const body = pause.json as { error?: string; details?: { runStatus?: string } };
|
|
166
|
+
expect(body.error).toBe('conflict');
|
|
167
|
+
// Spec requires `details.runStatus` to disclose the terminal state so
|
|
168
|
+
// the caller can decide whether to retry or surface the conflict.
|
|
169
|
+
expect(['completed', 'failed', 'cancelled']).toContain(body.details?.runStatus);
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
describe.skipIf(SKIP)('pause/resume: :pause-during-suspend race', () => {
|
|
174
|
+
it(':pause MUST NOT silently override an active interrupt suspend', async () => {
|
|
175
|
+
// If the host seeds an approval fixture, drive a suspend then attempt
|
|
176
|
+
// :pause. The expected behavior is that :pause either (a) noops with
|
|
177
|
+
// 409 because the run is already waiting-approval (not in a pausable
|
|
178
|
+
// state), or (b) accepts and stacks pause atop the suspend with the
|
|
179
|
+
// run's terminal state still being waiting-approval. Either is
|
|
180
|
+
// acceptable; what's NOT acceptable is the host quietly flipping
|
|
181
|
+
// status to `paused` and discarding the suspended interrupt.
|
|
182
|
+
if (!isFixtureAdvertised('conformance-approval')) {
|
|
183
|
+
// eslint-disable-next-line no-console
|
|
184
|
+
console.warn(
|
|
185
|
+
'[pause-resume] conformance-approval not advertised; skipping :pause-during-suspend race subtest',
|
|
186
|
+
);
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
const create = await driver.post('/v1/runs', { workflowId: 'conformance-approval' });
|
|
190
|
+
expect(create.status).toBe(201);
|
|
191
|
+
const runId = (create.json as { runId: string }).runId;
|
|
192
|
+
await pollUntilStatus(runId, 'waiting-approval', { timeoutMs: 10_000 });
|
|
193
|
+
|
|
194
|
+
const pause = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {
|
|
195
|
+
reason: 'race-test',
|
|
196
|
+
});
|
|
197
|
+
if (pause.status === 404) {
|
|
198
|
+
// Cleanup.
|
|
199
|
+
await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
|
|
200
|
+
reason: 'conformance-cleanup',
|
|
201
|
+
});
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Either rejection (preferred) or stacked-pause is OK; silent override is not.
|
|
206
|
+
if (pause.status === 409) {
|
|
207
|
+
const body = pause.json as { details?: { runStatus?: string } };
|
|
208
|
+
expect(body.details?.runStatus, driver.describe(
|
|
209
|
+
'rest-endpoints.md POST /v1/runs/{runId}:pause',
|
|
210
|
+
':pause-during-suspend MUST surface the active waiting-* status in the conflict envelope',
|
|
211
|
+
)).toMatch(/^waiting-/);
|
|
212
|
+
} else {
|
|
213
|
+
// Stacked-pause accepted: verify the run's reported status still
|
|
214
|
+
// surfaces the underlying suspend — the host MUST NOT lose track
|
|
215
|
+
// of the interrupt waiting for resolution.
|
|
216
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
217
|
+
const status = (snap.json as { status: string }).status;
|
|
218
|
+
expect(
|
|
219
|
+
status === 'paused' || status.startsWith('waiting-'),
|
|
220
|
+
':pause-during-suspend MUST NOT silently discard the active interrupt',
|
|
221
|
+
).toBe(true);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
|
|
225
|
+
reason: 'conformance-cleanup',
|
|
226
|
+
});
|
|
227
|
+
});
|
|
228
|
+
});
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0009 §B: production-profile backpressure envelope.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that hosts claiming the `openwop-production` profile satisfy
|
|
5
|
+
* `spec/v1/production-profile.md` §Backpressure:
|
|
6
|
+
*
|
|
7
|
+
* 1. `capabilities.production.backpressure.supported: true` is
|
|
8
|
+
* advertised when the profile claim is `supported: true`.
|
|
9
|
+
* 2. When the host advertises an `inflightCap`, the suite saturates
|
|
10
|
+
* it with `inflightCap + 1` concurrent long-lived requests and
|
|
11
|
+
* asserts the 503 envelope shape: status 503 + `Retry-After`
|
|
12
|
+
* header + body `{error: "service_unavailable", message, details:
|
|
13
|
+
* {retryAfter: <integer>}}` where `details.retryAfter` equals the
|
|
14
|
+
* `Retry-After` header in seconds.
|
|
15
|
+
* 3. When `retryAfterSeconds` is also advertised, both equal that
|
|
16
|
+
* advertised value.
|
|
17
|
+
* 4. Discovery (`GET /.well-known/openwop`) is exempt from the
|
|
18
|
+
* inflight cap (health probes MUST answer even under load).
|
|
19
|
+
*
|
|
20
|
+
* Hosts that don't advertise `inflightCap` soft-skip the saturation
|
|
21
|
+
* step (the envelope assertion only fires if a 503 happens to be
|
|
22
|
+
* observed via other means). RFC 0009 unresolved question #3 — the
|
|
23
|
+
* alternative of probing via Little's Law is rejected by default;
|
|
24
|
+
* inflightCap advertisement is the chosen forcing mechanism.
|
|
25
|
+
*
|
|
26
|
+
* **Run with `--no-file-parallelism`** — saturating the host's
|
|
27
|
+
* inflight cap leaves no headroom for other scenarios that issue
|
|
28
|
+
* concurrent POSTs (e.g., `idempotencyRetry.test.ts`'s 5-retry burst).
|
|
29
|
+
* Conformance runs that include this scenario MUST pass `--no-file-
|
|
30
|
+
* parallelism` to vitest or scope each file to its own worker. The
|
|
31
|
+
* otel-emission scenario has the same constraint (`conformance/README.md`
|
|
32
|
+
* line ~69).
|
|
33
|
+
*
|
|
34
|
+
* @see RFCS/0009-production-profile-conformance.md §B
|
|
35
|
+
* @see spec/v1/production-profile.md §Backpressure
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { describe, it, expect } from 'vitest';
|
|
39
|
+
import { driver } from '../lib/driver.js';
|
|
40
|
+
import { loadEnv } from '../lib/env.js';
|
|
41
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
42
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
43
|
+
|
|
44
|
+
interface BackpressureCaps {
|
|
45
|
+
supported?: boolean;
|
|
46
|
+
inflightCap?: number;
|
|
47
|
+
retryAfterSeconds?: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface ProductionCaps {
|
|
51
|
+
supported?: boolean;
|
|
52
|
+
backpressure?: BackpressureCaps;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function readProductionCaps(): Promise<ProductionCaps | undefined> {
|
|
56
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
57
|
+
return (disco.json as { capabilities?: { production?: ProductionCaps } })
|
|
58
|
+
.capabilities?.production;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function isProfileAdvertised(prod: ProductionCaps | undefined): boolean {
|
|
62
|
+
return (
|
|
63
|
+
prod?.supported === true && prod?.backpressure?.supported === true
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe('production-backpressure: capability shape', () => {
|
|
68
|
+
it('host that claims openwop-production with backpressure advertises required fields', async () => {
|
|
69
|
+
const prod = await readProductionCaps();
|
|
70
|
+
|
|
71
|
+
if (!behaviorGate('openwop-production', isProfileAdvertised(prod))) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
expect(prod?.supported, driver.describe(
|
|
76
|
+
'production-profile.md §Compatibility baseline',
|
|
77
|
+
'capabilities.production.supported MUST be true when claiming the openwop-production profile',
|
|
78
|
+
)).toBe(true);
|
|
79
|
+
|
|
80
|
+
expect(prod?.backpressure?.supported, driver.describe(
|
|
81
|
+
'production-profile.md §Backpressure',
|
|
82
|
+
'capabilities.production.backpressure.supported MUST be true for production-profile claimants',
|
|
83
|
+
)).toBe(true);
|
|
84
|
+
|
|
85
|
+
if (prod?.backpressure?.inflightCap !== undefined) {
|
|
86
|
+
expect(
|
|
87
|
+
Number.isInteger(prod.backpressure.inflightCap) &&
|
|
88
|
+
prod.backpressure.inflightCap >= 1,
|
|
89
|
+
driver.describe(
|
|
90
|
+
'capabilities.schema.json production.backpressure.inflightCap',
|
|
91
|
+
'inflightCap MUST be an integer ≥ 1 when advertised',
|
|
92
|
+
),
|
|
93
|
+
).toBe(true);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (prod?.backpressure?.retryAfterSeconds !== undefined) {
|
|
97
|
+
expect(
|
|
98
|
+
Number.isInteger(prod.backpressure.retryAfterSeconds) &&
|
|
99
|
+
prod.backpressure.retryAfterSeconds >= 0,
|
|
100
|
+
driver.describe(
|
|
101
|
+
'capabilities.schema.json production.backpressure.retryAfterSeconds',
|
|
102
|
+
'retryAfterSeconds MUST be a non-negative integer when advertised',
|
|
103
|
+
),
|
|
104
|
+
).toBe(true);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
describe('production-backpressure: 503 envelope under saturation', () => {
|
|
110
|
+
it('saturating advertised inflightCap returns 503 + Retry-After + canonical envelope', async () => {
|
|
111
|
+
const prod = await readProductionCaps();
|
|
112
|
+
|
|
113
|
+
if (!behaviorGate('openwop-production', isProfileAdvertised(prod))) {
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const cap = prod?.backpressure?.inflightCap;
|
|
118
|
+
if (cap === undefined) {
|
|
119
|
+
// eslint-disable-next-line no-console
|
|
120
|
+
console.warn(
|
|
121
|
+
'[production-backpressure] host does not advertise inflightCap; skipping saturation assertion',
|
|
122
|
+
);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Use the conformance-delay fixture to hold inflight slots via
|
|
127
|
+
// long-lived SSE connections, matching the Postgres host's
|
|
128
|
+
// internal backpressure test pattern.
|
|
129
|
+
const FIXTURE = 'conformance-delay';
|
|
130
|
+
if (!isFixtureAdvertised(FIXTURE)) {
|
|
131
|
+
// eslint-disable-next-line no-console
|
|
132
|
+
console.warn(
|
|
133
|
+
`[production-backpressure] ${FIXTURE} not advertised; skipping saturation (host doesn't seed the long-running fixture)`,
|
|
134
|
+
);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Create `cap` long-running runs and open SSE streams against each
|
|
139
|
+
// to hold the slots open. Raw fetch is used for SSE because the
|
|
140
|
+
// driver doesn't expose abort signals; abort is required to release
|
|
141
|
+
// the inflight slots in the finally block. The Accept header is
|
|
142
|
+
// required so hosts with /events content negotiation (e.g., the
|
|
143
|
+
// Postgres reference host) actually keep the connection open as a
|
|
144
|
+
// long-lived SSE stream; without it they return a one-shot JSON
|
|
145
|
+
// snapshot and the inflight slot drops immediately.
|
|
146
|
+
const env = loadEnv();
|
|
147
|
+
const authHeaders = {
|
|
148
|
+
Authorization: `Bearer ${env.apiKey}`,
|
|
149
|
+
Accept: 'text/event-stream',
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const slotHolders: AbortController[] = [];
|
|
153
|
+
const slotPromises: Promise<unknown>[] = [];
|
|
154
|
+
// Track the saturating runs so we can explicitly cancel them in
|
|
155
|
+
// `finally`. Without this, aborting only the SSE stream leaves
|
|
156
|
+
// the runs alive on the host until their delay elapses; any
|
|
157
|
+
// neighbor test running in parallel (vitest's default mode) then
|
|
158
|
+
// hits the still-saturated inflight cap and fails with a 503 of
|
|
159
|
+
// its own. Per spec the saturating runs MAY survive their SSE
|
|
160
|
+
// client; the test-isolation contract is the caller's
|
|
161
|
+
// responsibility.
|
|
162
|
+
const saturatingRunIds: string[] = [];
|
|
163
|
+
|
|
164
|
+
let saturationEarlyExit = false;
|
|
165
|
+
for (let i = 0; i < cap; i++) {
|
|
166
|
+
const create = await driver.post('/v1/runs', {
|
|
167
|
+
workflowId: FIXTURE,
|
|
168
|
+
// Long enough that the saturation window holds during the
|
|
169
|
+
// cap+1 probe + envelope assertions (~500ms total) but short
|
|
170
|
+
// enough that any leaked run drains quickly if cancel fails.
|
|
171
|
+
inputs: { delayMs: 2_000 },
|
|
172
|
+
});
|
|
173
|
+
if (create.status === 503) {
|
|
174
|
+
// Inflight cap was already saturated by a parallel test
|
|
175
|
+
// (default vitest mode shares the host across scenarios).
|
|
176
|
+
// The test's docstring notes `--no-file-parallelism` is the
|
|
177
|
+
// canonical execution mode; outside that mode the saturation
|
|
178
|
+
// moment is unreachable. Soft-skip per the existing pattern
|
|
179
|
+
// (consistent with the cap+1 fallback below).
|
|
180
|
+
// eslint-disable-next-line no-console
|
|
181
|
+
console.warn(
|
|
182
|
+
`[production-backpressure] cap already saturated by parallel runs at i=${i} (likely running without --no-file-parallelism); soft-skipping envelope assertions`,
|
|
183
|
+
);
|
|
184
|
+
saturationEarlyExit = true;
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
187
|
+
expect(create.status).toBe(201);
|
|
188
|
+
const runId = (create.json as { runId: string }).runId;
|
|
189
|
+
saturatingRunIds.push(runId);
|
|
190
|
+
|
|
191
|
+
const controller = new AbortController();
|
|
192
|
+
slotHolders.push(controller);
|
|
193
|
+
slotPromises.push(
|
|
194
|
+
fetch(`${env.baseUrl}/v1/runs/${encodeURIComponent(runId)}/events`, {
|
|
195
|
+
headers: authHeaders,
|
|
196
|
+
signal: controller.signal,
|
|
197
|
+
}).catch(() => undefined),
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
if (saturationEarlyExit) {
|
|
201
|
+
// Drain whatever we created so we don't leave residue for the
|
|
202
|
+
// next test. The finally block below handles bulk-cancel
|
|
203
|
+
// unconditionally; just bail before the envelope assertions.
|
|
204
|
+
for (const c of slotHolders) c.abort();
|
|
205
|
+
if (saturatingRunIds.length > 0) {
|
|
206
|
+
try {
|
|
207
|
+
await driver.post('/v1/runs:bulk-cancel', { runIds: saturatingRunIds });
|
|
208
|
+
} catch {
|
|
209
|
+
// Best-effort.
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
await Promise.allSettled(slotPromises);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Let SSE connections register.
|
|
217
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
218
|
+
|
|
219
|
+
try {
|
|
220
|
+
// The `cap + 1`-th authenticated request MUST 503.
|
|
221
|
+
const blocked = await driver.post('/v1/runs', {
|
|
222
|
+
workflowId: 'conformance-noop',
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
// Soft-skip if the host didn't actually 503 (e.g., it accepts
|
|
226
|
+
// more concurrency than its advertised cap, or the SSE slots
|
|
227
|
+
// didn't register in time). The envelope assertion only fires
|
|
228
|
+
// when 503 was observed.
|
|
229
|
+
if (blocked.status !== 503) {
|
|
230
|
+
// eslint-disable-next-line no-console
|
|
231
|
+
console.warn(
|
|
232
|
+
`[production-backpressure] expected 503 at cap+1=${cap + 1}, got ${blocked.status}; saturation may need tuning`,
|
|
233
|
+
);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const retryAfterHeader = blocked.headers.get('retry-after');
|
|
238
|
+
expect(retryAfterHeader, driver.describe(
|
|
239
|
+
'production-profile.md §Backpressure',
|
|
240
|
+
'503 response MUST include a Retry-After header',
|
|
241
|
+
)).toBeTruthy();
|
|
242
|
+
expect((retryAfterHeader ?? '').length).toBeGreaterThan(0);
|
|
243
|
+
|
|
244
|
+
const body = blocked.json as {
|
|
245
|
+
error?: string;
|
|
246
|
+
message?: string;
|
|
247
|
+
details?: { retryAfter?: number };
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
expect(body.error, driver.describe(
|
|
251
|
+
'production-profile.md §Backpressure',
|
|
252
|
+
'503 envelope MUST have error: "service_unavailable"',
|
|
253
|
+
)).toBe('service_unavailable');
|
|
254
|
+
|
|
255
|
+
expect(typeof body.message).toBe('string');
|
|
256
|
+
expect((body.message ?? '').length).toBeGreaterThan(0);
|
|
257
|
+
|
|
258
|
+
expect(typeof body.details?.retryAfter, driver.describe(
|
|
259
|
+
'production-profile.md §Backpressure',
|
|
260
|
+
'details.retryAfter MUST be present and numeric',
|
|
261
|
+
)).toBe('number');
|
|
262
|
+
|
|
263
|
+
// details.retryAfter MUST equal the Retry-After header in seconds.
|
|
264
|
+
const headerSeconds = Number.parseInt(retryAfterHeader ?? '', 10);
|
|
265
|
+
expect(body.details?.retryAfter, driver.describe(
|
|
266
|
+
'production-profile.md §Backpressure',
|
|
267
|
+
'details.retryAfter MUST equal Retry-After header in seconds',
|
|
268
|
+
)).toBe(headerSeconds);
|
|
269
|
+
|
|
270
|
+
// If retryAfterSeconds is advertised, both equal it.
|
|
271
|
+
const advertised = prod?.backpressure?.retryAfterSeconds;
|
|
272
|
+
if (advertised !== undefined) {
|
|
273
|
+
expect(headerSeconds, driver.describe(
|
|
274
|
+
'capabilities.schema.json production.backpressure.retryAfterSeconds',
|
|
275
|
+
'Retry-After MUST equal advertised retryAfterSeconds when both are present',
|
|
276
|
+
)).toBe(advertised);
|
|
277
|
+
}
|
|
278
|
+
} finally {
|
|
279
|
+
// Test isolation: explicitly cancel every saturating run so
|
|
280
|
+
// neighbor tests running in parallel see the inflight cap
|
|
281
|
+
// released immediately. The SSE-stream aborts come first so
|
|
282
|
+
// the host's SSE handlers exit; the bulk-cancel then drains
|
|
283
|
+
// the executor queue. POST /v1/runs:bulk-cancel is naturally
|
|
284
|
+
// idempotent so even if a run already terminated, the call
|
|
285
|
+
// returns `ok: true, status: "cancelled"` per
|
|
286
|
+
// rest-endpoints.md §"Bulk cancel".
|
|
287
|
+
for (const c of slotHolders) c.abort();
|
|
288
|
+
if (saturatingRunIds.length > 0) {
|
|
289
|
+
try {
|
|
290
|
+
await driver.post('/v1/runs:bulk-cancel', { runIds: saturatingRunIds });
|
|
291
|
+
} catch {
|
|
292
|
+
// Best-effort; a host that doesn't expose bulk-cancel
|
|
293
|
+
// gets per-id cancellation as the fallback.
|
|
294
|
+
for (const id of saturatingRunIds) {
|
|
295
|
+
try {
|
|
296
|
+
await driver.post(`/v1/runs/${encodeURIComponent(id)}/cancel`, {});
|
|
297
|
+
} catch {
|
|
298
|
+
// Swallow — the runs will drain naturally within
|
|
299
|
+
// the (now-shortened) 2-second delay window.
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
await Promise.allSettled(slotPromises);
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
describe('production-backpressure: discovery exempt from cap', () => {
|
|
310
|
+
it('GET /.well-known/openwop returns 200 even when inflight is saturated', async () => {
|
|
311
|
+
const prod = await readProductionCaps();
|
|
312
|
+
|
|
313
|
+
if (!behaviorGate('openwop-production', isProfileAdvertised(prod))) {
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
const cap = prod?.backpressure?.inflightCap;
|
|
318
|
+
if (cap === undefined) {
|
|
319
|
+
// Without advertised cap we can't saturate deterministically;
|
|
320
|
+
// fall back to a single discovery probe.
|
|
321
|
+
const probe = await driver.get('/.well-known/openwop');
|
|
322
|
+
expect(probe.status, driver.describe(
|
|
323
|
+
'production-profile.md §Backpressure',
|
|
324
|
+
'discovery MUST answer regardless of load',
|
|
325
|
+
)).toBe(200);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Issue many concurrent discovery probes; all MUST succeed.
|
|
330
|
+
const probes = await Promise.all(
|
|
331
|
+
Array.from({ length: Math.max(cap + 2, 4) }, () =>
|
|
332
|
+
driver.get('/.well-known/openwop'),
|
|
333
|
+
),
|
|
334
|
+
);
|
|
335
|
+
for (const probe of probes) {
|
|
336
|
+
expect(probe.status, driver.describe(
|
|
337
|
+
'production-profile.md §Backpressure',
|
|
338
|
+
'discovery MUST bypass the inflight cap (health probes always answer)',
|
|
339
|
+
)).toBe(200);
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
});
|