@openwop/openwop-conformance 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +31 -6
- package/api/grpc/openwop.proto +251 -0
- package/api/openapi.yaml +109 -3
- package/coverage.md +48 -9
- package/fixtures/conformance-configurable-schema.json +39 -0
- package/fixtures/conformance-subworkflow-parent.json +1 -1
- package/fixtures/conformance-wasm-pack-memory-cap-breach.json +23 -0
- package/fixtures/openwop-smoke-byok-roundtrip.json +25 -0
- package/fixtures.md +21 -0
- package/package.json +3 -1
- package/schemas/README.md +4 -0
- package/schemas/audit-verify-result.schema.json +90 -0
- package/schemas/capabilities.schema.json +342 -1
- package/schemas/node-pack-manifest.schema.json +4 -4
- package/schemas/pack-lockfile.schema.json +92 -0
- package/schemas/registry-version-manifest.schema.json +145 -0
- package/schemas/run-event-payloads.schema.json +20 -4
- package/schemas/run-event.schema.json +2 -1
- package/schemas/security-advisory.schema.json +109 -0
- package/src/lib/a2a-fake-peer.ts +143 -56
- package/src/lib/behavior-gate.ts +107 -0
- package/src/lib/env.ts +37 -0
- package/src/lib/grpc-framing.test.ts +96 -0
- package/src/lib/grpc-framing.ts +76 -0
- package/src/lib/oidc-issuer.test.ts +328 -0
- package/src/lib/oidc-issuer.ts +241 -0
- package/src/lib/otel-collector-grpc.test.ts +191 -0
- package/src/lib/otel-collector.test.ts +303 -0
- package/src/lib/otel-collector.ts +318 -14
- package/src/lib/otlp-protobuf.test.ts +461 -0
- package/src/lib/otlp-protobuf.ts +529 -0
- package/src/scenarios/a2a-task-roundtrip.test.ts +147 -28
- package/src/scenarios/agentConfidenceEscalation.test.ts +1 -0
- package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +1 -0
- package/src/scenarios/agentMemoryRedactionContract.test.ts +1 -0
- package/src/scenarios/agentMemoryRoundTrip.test.ts +1 -0
- package/src/scenarios/agentMemoryTtlExpiry.test.ts +1 -0
- package/src/scenarios/agentMessageReducer.test.ts +1 -0
- package/src/scenarios/agentMetadata.test.ts +1 -0
- package/src/scenarios/agentPackExport.test.ts +1 -0
- package/src/scenarios/agentPackInstall.test.ts +1 -0
- package/src/scenarios/agentPackProvenance.test.ts +1 -0
- package/src/scenarios/audit-log-integrity.test.ts +3 -6
- package/src/scenarios/auth-api-key-rotation.test.ts +182 -0
- package/src/scenarios/auth-mtls.test.ts +274 -0
- package/src/scenarios/auth-oauth2-client-credentials.test.ts +259 -0
- package/src/scenarios/auth-oidc-user-bearer.test.ts +361 -0
- package/src/scenarios/bulk-cancel.test.ts +111 -0
- package/src/scenarios/configurable-schema.test.ts +48 -0
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +1 -0
- package/src/scenarios/conversationLifecycle.test.ts +1 -0
- package/src/scenarios/conversationReplayDeterminism.test.ts +1 -0
- package/src/scenarios/conversationVsLegacySuspend.test.ts +1 -0
- package/src/scenarios/debug-bundle-truncation.test.ts +95 -0
- package/src/scenarios/discovery.test.ts +183 -0
- package/src/scenarios/http-client-ssrf.test.ts +71 -0
- package/src/scenarios/idempotency.test.ts +6 -0
- package/src/scenarios/idempotencyRetry.test.ts +3 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +205 -34
- package/src/scenarios/mcp-toolcall-redaction.test.ts +66 -0
- package/src/scenarios/memory-compaction-event-emitted.test.ts +121 -0
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +116 -0
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +127 -0
- package/src/scenarios/metric-emission.test.ts +113 -0
- package/src/scenarios/multi-region-idempotency.test.ts +39 -4
- package/src/scenarios/orchestratorConservativePath.test.ts +1 -0
- package/src/scenarios/orchestratorDispatch.test.ts +1 -0
- package/src/scenarios/orchestratorTermination.test.ts +1 -0
- package/src/scenarios/otel-emission-grpc.test.ts +98 -0
- package/src/scenarios/otel-trace-propagation-subworkflow.test.ts +139 -0
- package/src/scenarios/pause-resume.test.ts +119 -0
- package/src/scenarios/production-backpressure.test.ts +342 -0
- package/src/scenarios/production-retention-expiry.test.ts +164 -0
- package/src/scenarios/registry-public.test.ts +222 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +35 -0
- package/src/scenarios/replay-retention-expiry.test.ts +178 -0
- package/src/scenarios/restart-during-run.test.ts +177 -0
- package/src/scenarios/spec-corpus-validity.test.ts +59 -26
- package/src/scenarios/staleClaim.test.ts +3 -0
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +67 -10
- package/src/scenarios/wasm-pack-memory-cap.test.ts +64 -9
- package/src/scenarios/webhook-negative.test.ts +90 -0
- package/src/scenarios/webhook-signed-delivery.test.ts +178 -0
- package/src/setup.ts +25 -1
- package/vitest.config.ts +5 -1
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* End-to-end webhook signed-delivery scenario (webhooks.md).
|
|
3
|
+
*
|
|
4
|
+
* Boots a local HTTP receiver, registers it via
|
|
5
|
+
* `POST /v1/webhooks`, drives a run, and verifies that:
|
|
6
|
+
* 1. Delivery arrives at the receiver.
|
|
7
|
+
* 2. `X-openwop-Signature-Algorithm: v1` header is present.
|
|
8
|
+
* 3. `X-openwop-Signature` is a valid HMAC-SHA256 of
|
|
9
|
+
* `${timestamp}.${rawBody}` under the subscription secret.
|
|
10
|
+
* 4. `X-openwop-Subscription-Id` matches the returned subscription id.
|
|
11
|
+
*
|
|
12
|
+
* Capability-gated: skips when the host does not advertise
|
|
13
|
+
* `capabilities.webhooks.supported = true`.
|
|
14
|
+
*
|
|
15
|
+
* Operator contract: hosts that implement a SSRF guard on
|
|
16
|
+
* `POST /v1/webhooks` (rejecting loopback / RFC1918 / link-local
|
|
17
|
+
* destinations to protect deployer infrastructure) MUST allow the test
|
|
18
|
+
* receiver. The SQLite reference host bypasses the guard when the
|
|
19
|
+
* `OPENWOP_WEBHOOK_ALLOW_PRIVATE=true` env var is set at boot. Test-only
|
|
20
|
+
* hosts SHOULD provide an equivalent opt-in. When the host rejects with
|
|
21
|
+
* `400 webhook_url_rejected`, this scenario skips with a warning.
|
|
22
|
+
*
|
|
23
|
+
* @see spec/v1/webhooks.md §"Signature scheme"
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
27
|
+
import { createHmac } from 'node:crypto';
|
|
28
|
+
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
|
|
29
|
+
import { driver } from '../lib/driver.js';
|
|
30
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
31
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
32
|
+
|
|
33
|
+
interface DeliveredRequest {
|
|
34
|
+
readonly headers: Record<string, string>;
|
|
35
|
+
readonly body: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function startReceiver(): Promise<{ server: Server; url: string; received: DeliveredRequest[] }> {
|
|
39
|
+
const received: DeliveredRequest[] = [];
|
|
40
|
+
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
41
|
+
const chunks: Buffer[] = [];
|
|
42
|
+
req.on('data', (c: Buffer) => chunks.push(c));
|
|
43
|
+
req.on('end', () => {
|
|
44
|
+
const body = Buffer.concat(chunks).toString('utf8');
|
|
45
|
+
const headers: Record<string, string> = {};
|
|
46
|
+
for (const [k, v] of Object.entries(req.headers)) {
|
|
47
|
+
if (typeof v === 'string') headers[k.toLowerCase()] = v;
|
|
48
|
+
else if (Array.isArray(v)) headers[k.toLowerCase()] = v.join(',');
|
|
49
|
+
}
|
|
50
|
+
received.push({ headers, body });
|
|
51
|
+
res.writeHead(204);
|
|
52
|
+
res.end();
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
|
|
56
|
+
const addr = server.address();
|
|
57
|
+
if (typeof addr !== 'object' || addr === null) throw new Error('receiver address unavailable');
|
|
58
|
+
return { server, url: `http://127.0.0.1:${addr.port}/`, received };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let activeServer: Server | null = null;
|
|
62
|
+
afterEach(async () => {
|
|
63
|
+
if (activeServer) {
|
|
64
|
+
await new Promise<void>((resolve) => activeServer!.close(() => resolve()));
|
|
65
|
+
activeServer = null;
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
async function isWebhookSupported(): Promise<boolean> {
|
|
70
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
71
|
+
const caps = (disco.json as { capabilities?: { webhooks?: { supported?: boolean } } }).capabilities;
|
|
72
|
+
return caps?.webhooks?.supported === true;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
describe('webhook-signed-delivery: end-to-end HMAC v1', () => {
|
|
76
|
+
it('host POSTs run events to subscriber with valid X-openwop-Signature', async () => {
|
|
77
|
+
if (!(await isWebhookSupported())) {
|
|
78
|
+
// eslint-disable-next-line no-console
|
|
79
|
+
console.warn('[webhook-signed-delivery] host does not advertise webhook support; skipping');
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
if (!isFixtureAdvertised('conformance-noop')) {
|
|
83
|
+
// eslint-disable-next-line no-console
|
|
84
|
+
console.warn('[webhook-signed-delivery] conformance-noop not advertised; skipping');
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const receiver = await startReceiver();
|
|
89
|
+
activeServer = receiver.server;
|
|
90
|
+
|
|
91
|
+
// Register the webhook.
|
|
92
|
+
const reg = await driver.post('/v1/webhooks', { url: receiver.url });
|
|
93
|
+
|
|
94
|
+
// SSRF guard skip: if the host rejects loopback destinations,
|
|
95
|
+
// honor the operator contract and skip rather than fail.
|
|
96
|
+
if (reg.status === 400) {
|
|
97
|
+
const body = reg.json as { error?: string };
|
|
98
|
+
if (body.error === 'webhook_url_rejected') {
|
|
99
|
+
// eslint-disable-next-line no-console
|
|
100
|
+
console.warn(
|
|
101
|
+
'[webhook-signed-delivery] host SSRF guard rejected the loopback receiver; ' +
|
|
102
|
+
'set OPENWOP_WEBHOOK_ALLOW_PRIVATE=true on the host (or equivalent) to run',
|
|
103
|
+
);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
expect(reg.status, driver.describe(
|
|
109
|
+
'webhooks.md §"Register"',
|
|
110
|
+
'POST /v1/webhooks MUST return 201 with subscriptionId + secret on success',
|
|
111
|
+
)).toBe(201);
|
|
112
|
+
const sub = reg.json as { subscriptionId: string; secret: string };
|
|
113
|
+
expect(typeof sub.subscriptionId).toBe('string');
|
|
114
|
+
expect(typeof sub.secret).toBe('string');
|
|
115
|
+
expect(sub.secret.length).toBeGreaterThan(0);
|
|
116
|
+
|
|
117
|
+
// Drive a run; the host MUST deliver events to the registered receiver.
|
|
118
|
+
const create = await driver.post('/v1/runs', { workflowId: 'conformance-noop' });
|
|
119
|
+
expect(create.status).toBe(201);
|
|
120
|
+
const runId = (create.json as { runId: string }).runId;
|
|
121
|
+
await pollUntilTerminal(runId, { timeoutMs: 10_000 });
|
|
122
|
+
|
|
123
|
+
// Allow a small grace period for fire-and-forget delivery to land.
|
|
124
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
125
|
+
|
|
126
|
+
// Test-isolation note: when this scenario runs concurrently with
|
|
127
|
+
// other webhook-bearing scenarios against a stateful host, the
|
|
128
|
+
// host's webhook registry fans out EVERY run's events to EVERY
|
|
129
|
+
// registered subscription. Receivers in this scenario MAY observe
|
|
130
|
+
// deliveries from other tests' concurrent runs. Filter to events
|
|
131
|
+
// carrying THIS test's runId so the assertion checks the
|
|
132
|
+
// signature shape on a delivery the host emitted for THIS run.
|
|
133
|
+
const ourDeliveries = receiver.received.filter((d) => {
|
|
134
|
+
try {
|
|
135
|
+
const body = JSON.parse(d.body) as { runId?: unknown };
|
|
136
|
+
return body.runId === runId;
|
|
137
|
+
} catch {
|
|
138
|
+
return false;
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
expect(ourDeliveries.length, driver.describe(
|
|
142
|
+
'webhooks.md §"Delivery"',
|
|
143
|
+
'host MUST POST at least one event for THIS run to a registered subscriber after run.completed',
|
|
144
|
+
)).toBeGreaterThan(0);
|
|
145
|
+
|
|
146
|
+
// Validate the FIRST delivery's signature contract. Other deliveries
|
|
147
|
+
// share the same signing rules; checking one is sufficient.
|
|
148
|
+
const first = ourDeliveries[0]!;
|
|
149
|
+
expect(first.headers['x-openwop-signature-algorithm'], driver.describe(
|
|
150
|
+
'webhooks.md §"Signature algorithm versioning"',
|
|
151
|
+
'every delivery MUST set X-openwop-Signature-Algorithm: v1',
|
|
152
|
+
)).toBe('v1');
|
|
153
|
+
expect(first.headers['x-openwop-subscription-id']).toBe(sub.subscriptionId);
|
|
154
|
+
|
|
155
|
+
const timestamp = first.headers['x-openwop-signature-timestamp'];
|
|
156
|
+
expect(typeof timestamp).toBe('string');
|
|
157
|
+
expect((timestamp ?? '').length).toBeGreaterThan(0);
|
|
158
|
+
|
|
159
|
+
const signature = first.headers['x-openwop-signature'];
|
|
160
|
+
const expected = createHmac('sha256', sub.secret)
|
|
161
|
+
.update(`${timestamp}.${first.body}`, 'utf8')
|
|
162
|
+
.digest('hex');
|
|
163
|
+
expect(signature, driver.describe(
|
|
164
|
+
'webhooks.md §"Signature scheme"',
|
|
165
|
+
'X-openwop-Signature MUST be HMAC-SHA256(secret, `${timestamp}.${rawBody}`) hex',
|
|
166
|
+
)).toBe(expected);
|
|
167
|
+
|
|
168
|
+
// Body should parse as JSON with a run event shape.
|
|
169
|
+
const event = JSON.parse(first.body) as { type?: unknown; runId?: unknown };
|
|
170
|
+
expect(typeof event.type).toBe('string');
|
|
171
|
+
expect(event.runId).toBe(runId);
|
|
172
|
+
|
|
173
|
+
// Cleanup: unregister.
|
|
174
|
+
const del = await driver.delete(`/v1/webhooks/${encodeURIComponent(sub.subscriptionId)}`);
|
|
175
|
+
expect(del.status).toBeGreaterThanOrEqual(200);
|
|
176
|
+
expect(del.status).toBeLessThan(300);
|
|
177
|
+
});
|
|
178
|
+
});
|
package/src/setup.ts
CHANGED
|
@@ -117,6 +117,30 @@ async function maybeStartOtelCollector(): Promise<void> {
|
|
|
117
117
|
`Configure the host with OTEL_EXPORTER_OTLP_ENDPOINT=${collector.endpoint()} ` +
|
|
118
118
|
`and OTEL_EXPORTER_OTLP_PROTOCOL=http/json.`,
|
|
119
119
|
);
|
|
120
|
+
|
|
121
|
+
// Track 11 — opt into the parallel OTLP/gRPC collector when
|
|
122
|
+
// `OPENWOP_OTEL_COLLECTOR_GRPC=true`. Same span/metric store; hosts
|
|
123
|
+
// emitting via either transport surface in `getCollector().spans()`.
|
|
124
|
+
if (process.env.OPENWOP_OTEL_COLLECTOR_GRPC === 'true') {
|
|
125
|
+
const grpcPortEnv = process.env.OPENWOP_OTEL_COLLECTOR_GRPC_PORT;
|
|
126
|
+
const grpcRequestedPort = grpcPortEnv ? Number(grpcPortEnv) : 4317;
|
|
127
|
+
try {
|
|
128
|
+
await collector.startGrpc(grpcRequestedPort);
|
|
129
|
+
} catch (err) {
|
|
130
|
+
// eslint-disable-next-line no-console
|
|
131
|
+
console.warn(
|
|
132
|
+
`[openwop-conformance setup] OTLP/gRPC collector failed to bind on port ${grpcRequestedPort} ` +
|
|
133
|
+
`(${(err as Error).message ?? 'unknown'}); falling back to ephemeral port.`,
|
|
134
|
+
);
|
|
135
|
+
await collector.startGrpc(0);
|
|
136
|
+
}
|
|
137
|
+
// eslint-disable-next-line no-console
|
|
138
|
+
console.error(
|
|
139
|
+
`[openwop-conformance setup] OTLP/gRPC collector listening at ${collector.grpcEndpoint()}. ` +
|
|
140
|
+
`Configure the host with OTEL_EXPORTER_OTLP_ENDPOINT=${collector.grpcEndpoint()} ` +
|
|
141
|
+
`and OTEL_EXPORTER_OTLP_PROTOCOL=grpc.`,
|
|
142
|
+
);
|
|
143
|
+
}
|
|
120
144
|
}
|
|
121
145
|
|
|
122
146
|
/**
|
|
@@ -163,7 +187,7 @@ async function maybeStartA2AFakePeer(): Promise<void> {
|
|
|
163
187
|
// eslint-disable-next-line no-console
|
|
164
188
|
console.error(
|
|
165
189
|
`[openwop-conformance setup] A2A fake peer listening at ${peer.endpoint()}. ` +
|
|
166
|
-
`AgentCard at ${peer.endpoint()}/agent.json.`,
|
|
190
|
+
`AgentCard at ${peer.endpoint()}/.well-known/agent-card.json.`,
|
|
167
191
|
);
|
|
168
192
|
}
|
|
169
193
|
|
package/vitest.config.ts
CHANGED
|
@@ -2,7 +2,11 @@ import { defineConfig } from 'vitest/config';
|
|
|
2
2
|
|
|
3
3
|
export default defineConfig({
|
|
4
4
|
test: {
|
|
5
|
-
|
|
5
|
+
// `src/scenarios/**/*.test.ts` are the host-targeted conformance
|
|
6
|
+
// scenarios; `src/lib/**/*.test.ts` are server-free unit tests for
|
|
7
|
+
// suite-internal helpers (e.g., the synthetic OIDC issuer harness)
|
|
8
|
+
// that don't depend on OPENWOP_BASE_URL.
|
|
9
|
+
include: ['src/scenarios/**/*.test.ts', 'src/lib/**/*.test.ts'],
|
|
6
10
|
testTimeout: 30_000,
|
|
7
11
|
hookTimeout: 30_000,
|
|
8
12
|
globals: true,
|