@openwop/openwop-conformance 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -1
- package/README.md +3 -2
- package/api/asyncapi.yaml +8 -0
- package/api/openapi.yaml +371 -1
- package/coverage.md +26 -6
- package/fixtures/conformance-envelope-nl-to-format-engaged.json +41 -0
- package/fixtures/conformance-envelope-recovery-applied.json +39 -0
- package/fixtures/conformance-envelope-refusal.json +38 -0
- package/fixtures/conformance-envelope-retry-attempted.json +39 -0
- package/fixtures/conformance-envelope-retry-exhausted.json +38 -0
- package/fixtures/conformance-envelope-truncated.json +39 -0
- package/fixtures/conformance-envelope-truncation-cap-exhaustion.json +39 -0
- package/fixtures/conformance-model-capability-insufficient.json +25 -0
- package/fixtures/conformance-multi-agent-confidence-escalation.json +49 -0
- package/fixtures/conformance-multi-agent-handoff-child.json +27 -0
- package/fixtures/conformance-multi-agent-handoff.json +49 -0
- package/fixtures/conformance-prompt-all-four-kinds.json +39 -0
- package/fixtures/conformance-prompt-end-to-end.json +33 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation-child.json +31 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation.json +33 -0
- package/fixtures/openwop-smoke-cost-emit.json +37 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot-2.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-schema-hint.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-secret-redaction.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-trust-marker.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-system.json +15 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-user.json +15 -0
- package/fixtures.md +39 -0
- package/package.json +1 -1
- package/schemas/README.md +5 -0
- package/schemas/agent-manifest.schema.json +16 -0
- package/schemas/capabilities.schema.json +384 -1
- package/schemas/envelopes/clarification.request.schema.json +9 -0
- package/schemas/envelopes/error.schema.json +4 -0
- package/schemas/envelopes/schema.request.schema.json +4 -0
- package/schemas/envelopes/schema.response.schema.json +1 -1
- package/schemas/node-pack-manifest.schema.json +28 -0
- package/schemas/orchestrator-decision.schema.json +12 -0
- package/schemas/prompt-kind.schema.json +8 -0
- package/schemas/prompt-pack-manifest.schema.json +80 -0
- package/schemas/prompt-ref.schema.json +40 -0
- package/schemas/prompt-template.schema.json +149 -0
- package/schemas/registry-version-manifest.schema.json +5 -0
- package/schemas/run-ancestry-response.schema.json +54 -0
- package/schemas/run-event-payloads.schema.json +479 -11
- package/schemas/run-event.schema.json +15 -1
- package/schemas/run-snapshot.schema.json +3 -2
- package/schemas/workflow-definition.schema.json +19 -1
- package/src/lib/llm-cache-key-recipe.ts +68 -0
- package/src/scenarios/aiEnvelope.contractRefusal.test.ts +104 -13
- package/src/scenarios/aiEnvelope.correlationReplay.test.ts +32 -15
- package/src/scenarios/aiEnvelope.redaction.test.ts +6 -5
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +5 -5
- package/src/scenarios/aiEnvelope.trustBoundaryPropagation.test.ts +211 -12
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +7 -7
- package/src/scenarios/blob-presign-expiry.test.ts +7 -7
- package/src/scenarios/cache-ttl-expiry.test.ts +6 -6
- package/src/scenarios/cost-attribution.test.ts +124 -11
- package/src/scenarios/cross-engine-append-ordering.test.ts +99 -0
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +136 -0
- package/src/scenarios/cross-host-causation-shape.test.ts +117 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +60 -0
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +223 -0
- package/src/scenarios/envelope-nl-to-format-engaged.test.ts +152 -0
- package/src/scenarios/envelope-reasoning-secret-redaction.test.ts +343 -0
- package/src/scenarios/envelope-reasoning-shape.test.ts +190 -0
- package/src/scenarios/envelope-recovery-applied.test.ts +229 -0
- package/src/scenarios/envelope-refusal-shape.test.ts +289 -0
- package/src/scenarios/envelope-retry-attempted.test.ts +258 -0
- package/src/scenarios/envelope-retry-exhausted.test.ts +168 -0
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +229 -0
- package/src/scenarios/envelope-truncated.test.ts +136 -0
- package/src/scenarios/envelope-truncation-cap-exhaustion.test.ts +144 -0
- package/src/scenarios/envelope-variant-discriminator-static.test.ts +152 -0
- package/src/scenarios/fixtures-valid.test.ts +123 -15
- package/src/scenarios/kv-ttl-expiry.test.ts +7 -7
- package/src/scenarios/model-capability-insufficient.test.ts +221 -0
- package/src/scenarios/model-capability-substituted.test.ts +203 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +201 -0
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +167 -0
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +124 -0
- package/src/scenarios/multi-region-idempotency.test.ts +58 -0
- package/src/scenarios/node-module-required-capabilities-shape.test.ts +185 -0
- package/src/scenarios/prompt-all-four-kinds-events.test.ts +198 -0
- package/src/scenarios/prompt-composed-secret-redaction.test.ts +178 -0
- package/src/scenarios/prompt-composed-trust-marker.test.ts +165 -0
- package/src/scenarios/prompt-end-to-end-events.test.ts +202 -0
- package/src/scenarios/prompt-list-and-fetch.test.ts +207 -0
- package/src/scenarios/prompt-mutable-lifecycle.test.ts +216 -0
- package/src/scenarios/prompt-pack-install.test.ts +187 -0
- package/src/scenarios/prompt-render-deterministic.test.ts +240 -0
- package/src/scenarios/prompt-resolution-chain-agent-intrinsic.test.ts +140 -0
- package/src/scenarios/prompt-resolution-chain-fallback-cascade.test.ts +172 -0
- package/src/scenarios/prompt-resolution-chain-node-wins.test.ts +144 -0
- package/src/scenarios/prompt-template-shape.test.ts +359 -0
- package/src/scenarios/queue-ack-nack-dlq.test.ts +7 -7
- package/src/scenarios/queue-publish-consume-roundtrip.test.ts +7 -7
- package/src/scenarios/replay-divergence-at-refusal.test.ts +134 -0
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +197 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +1 -40
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +80 -0
- package/src/scenarios/sandbox-capability-gate-respected.test.ts +27 -0
- package/src/scenarios/sandbox-memory-cap.test.ts +58 -0
- package/src/scenarios/sandbox-no-cross-pack-mutation.test.ts +30 -0
- package/src/scenarios/sandbox-no-host-env-leak.test.ts +27 -0
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +88 -0
- package/src/scenarios/sandbox-no-host-process-escape.test.ts +31 -0
- package/src/scenarios/sandbox-no-network-escape.test.ts +28 -0
- package/src/scenarios/sandbox-timeout-cap.test.ts +58 -0
- package/src/scenarios/search-bm25-roundtrip.test.ts +7 -7
- package/src/scenarios/spec-corpus-validity.test.ts +34 -6
- package/src/scenarios/sql-transaction-atomicity.test.ts +6 -6
- package/src/scenarios/stream-subscribe-from-beginning.test.ts +7 -7
- package/src/scenarios/subworkflow-input-mapping.test.ts +70 -4
- package/src/scenarios/table-cursor-pagination.test.ts +7 -7
- package/src/scenarios/table-schema-enforcement.test.ts +7 -7
- package/src/scenarios/vector-knn-roundtrip.test.ts +7 -7
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sandbox-no-host-fs-escape — RFC 0035 §B invariant `node-pack-sandbox-no-host-fs-escape`.
|
|
3
|
+
*
|
|
4
|
+
* Capability-gated on `capabilities.sandbox.supported: true`. Hosts that
|
|
5
|
+
* don't advertise sandbox soft-skip cleanly (no host yet serves a
|
|
6
|
+
* sandbox-executing pack runtime — the invariant graduates from
|
|
7
|
+
* reference-impl to protocol tier when one does, per
|
|
8
|
+
* `SECURITY/invariants.yaml node-pack-sandbox-no-host-fs-escape`).
|
|
9
|
+
*
|
|
10
|
+
* Asserts (behavioral when host advertises): a pack from the synthetic
|
|
11
|
+
* `vendor.openwop.misbehaving-sandbox` registry that attempts to read or
|
|
12
|
+
* write files outside the host-advertised sandbox root fails closed with
|
|
13
|
+
* `error.code: "sandbox_escape_attempt"` and `details.escapeKind: "host-fs-escape"`
|
|
14
|
+
* per RFC 0035 §C.
|
|
15
|
+
*
|
|
16
|
+
* Today's scenario lands the advertisement-shape probe + the capability-gated
|
|
17
|
+
* behavioral stub. The behavioral assertion exercises the synthetic
|
|
18
|
+
* misbehaving-fs pack against the host's pack loader; that pack lands in a
|
|
19
|
+
* follow-up commit when the first sandbox-executing reference host is
|
|
20
|
+
* available.
|
|
21
|
+
*
|
|
22
|
+
* @see RFCS/0035-sandbox-execution-contract.md §B (failure-mode invariant table)
|
|
23
|
+
* @see spec/v1/host-capabilities.md §"Sandbox execution contract (RFC 0035)"
|
|
24
|
+
* @see SECURITY/invariants.yaml node-pack-sandbox-no-host-fs-escape
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { describe, it, expect } from 'vitest';
|
|
28
|
+
import { driver } from '../lib/driver.js';
|
|
29
|
+
|
|
30
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
31
|
+
|
|
32
|
+
interface SandboxCaps {
|
|
33
|
+
supported?: unknown;
|
|
34
|
+
isolationModel?: unknown;
|
|
35
|
+
allowedHostCalls?: unknown;
|
|
36
|
+
memoryLimitBytes?: unknown;
|
|
37
|
+
wallClockLimitMs?: unknown;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface DiscoveryDoc {
|
|
41
|
+
capabilities?: { sandbox?: SandboxCaps };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function readSandboxCaps(): Promise<SandboxCaps | null> {
|
|
45
|
+
try {
|
|
46
|
+
const res = await driver.get('/.well-known/openwop');
|
|
47
|
+
if (res.status !== 200) return null;
|
|
48
|
+
return (res.json as DiscoveryDoc).capabilities?.sandbox ?? null;
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
describe.skipIf(HTTP_SKIP)('sandbox-no-host-fs-escape: capability shape (RFC 0035 §A)', () => {
|
|
55
|
+
it('capabilities.sandbox (when present) conforms to RFC 0035 §A', async () => {
|
|
56
|
+
const sb = await readSandboxCaps();
|
|
57
|
+
if (sb === null) return; // host omits the block — soft-skip cleanly
|
|
58
|
+
|
|
59
|
+
expect(typeof sb.supported, 'capabilities.sandbox.supported MUST be boolean when present').toBe('boolean');
|
|
60
|
+
|
|
61
|
+
if (sb.supported === true) {
|
|
62
|
+
const m = sb.isolationModel as string;
|
|
63
|
+
const isCategorical = m === 'wasm' || m === 'process' || m === 'container' || m === 'vm';
|
|
64
|
+
const isExtension = /^x-host-[a-z][a-z0-9-]*-[a-z][a-z0-9-]*$/.test(m);
|
|
65
|
+
expect(
|
|
66
|
+
isCategorical || isExtension,
|
|
67
|
+
driver.describe(
|
|
68
|
+
'RFCS/0035-sandbox-execution-contract.md §A',
|
|
69
|
+
'isolationModel MUST be one of {wasm, process, container, vm} OR match ^x-host-<host>-<key>$ pattern',
|
|
70
|
+
),
|
|
71
|
+
).toBe(true);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// Behavioral assertion lands when the vendor.openwop.misbehaving-sandbox
|
|
77
|
+
// synthetic pack ships + a host advertises capabilities.sandbox.supported.
|
|
78
|
+
// Expected wire shape:
|
|
79
|
+
// POST /v1/host/sample/test/sandbox-load { packId: 'vendor.openwop.misbehaving-sandbox' }
|
|
80
|
+
// → 200 OK
|
|
81
|
+
// POST /v1/host/sample/test/sandbox-invoke { typeId: 'misbehave.fs-escape-read', args: { path: '/etc/passwd' } }
|
|
82
|
+
// → response.error.code === 'sandbox_escape_attempt'
|
|
83
|
+
// → response.error.details.escapeKind === 'host-fs-escape'
|
|
84
|
+
// Surfaced as `todo` so test reporters track the gap rather than reporting
|
|
85
|
+
// a vacuous PASS.
|
|
86
|
+
describe('sandbox-no-host-fs-escape: behavioral (RFC 0035 §B node-pack-sandbox-no-host-fs-escape)', () => {
|
|
87
|
+
it.todo('a misbehaving pack that reads outside the sandbox root fails closed with sandbox_escape_attempt + escapeKind: "host-fs-escape"');
|
|
88
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sandbox-no-host-process-escape — RFC 0035 §B invariant `node-pack-sandbox-no-host-process-escape`.
|
|
3
|
+
*
|
|
4
|
+
* Capability-gated on `capabilities.sandbox.supported: true`.
|
|
5
|
+
*
|
|
6
|
+
* Asserts (behavioral when host advertises): a pack invocation that attempts
|
|
7
|
+
* to spawn a host process, fork, or call exec-family syscalls fails closed
|
|
8
|
+
* with `error.code: "sandbox_escape_attempt"` AND
|
|
9
|
+
* `details.escapeKind: "host-process-escape"`.
|
|
10
|
+
*
|
|
11
|
+
* @see RFCS/0035-sandbox-execution-contract.md §B
|
|
12
|
+
* @see SECURITY/invariants.yaml node-pack-sandbox-no-host-process-escape
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { describe, it } from 'vitest';
|
|
16
|
+
|
|
17
|
+
// Behavioral assertion lands when a sandbox-executing host advertises
|
|
18
|
+
// `capabilities.sandbox.supported: true` AND ships a misbehaving-process-escape
|
|
19
|
+
// typeId (e.g., vendor.openwop.misbehaving-process). The assertion drives:
|
|
20
|
+
// 1. POST /v1/runs { workflowId: 'conformance-sandbox-process-escape' }
|
|
21
|
+
// where the workflow includes a node loading the misbehaving typeId.
|
|
22
|
+
// 2. The node attempts spawn/fork/exec inside the sandbox.
|
|
23
|
+
// 3. Assert the run terminates with error.code === 'sandbox_escape_attempt'
|
|
24
|
+
// AND error.details.escapeKind === 'host-process-escape'.
|
|
25
|
+
// 4. Assert no host process was actually spawned (host-side probe).
|
|
26
|
+
// Surfaced as `todo` so test reporters track the gap rather than reporting
|
|
27
|
+
// a vacuous PASS.
|
|
28
|
+
|
|
29
|
+
describe('sandbox-no-host-process-escape: behavioral (RFC 0035 §B)', () => {
|
|
30
|
+
it.todo('a misbehaving pack calling spawn/fork/exec fails closed with sandbox_escape_attempt + escapeKind: "host-process-escape"');
|
|
31
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sandbox-no-network-escape — RFC 0035 §B invariant `node-pack-sandbox-no-network-escape`.
|
|
3
|
+
*
|
|
4
|
+
* Capability-gated on `capabilities.sandbox.supported: true`.
|
|
5
|
+
*
|
|
6
|
+
* Asserts (behavioral when host advertises): a pack invocation that initiates
|
|
7
|
+
* a network request (fetch/connect/etc.) fails closed with
|
|
8
|
+
* `sandbox_capability_denied` AND `details.requestedCapability: "host.fetch"`
|
|
9
|
+
* (or equivalent) UNLESS `host.fetch` appears in
|
|
10
|
+
* `capabilities.sandbox.allowedHostCalls`.
|
|
11
|
+
*
|
|
12
|
+
* @see RFCS/0035-sandbox-execution-contract.md §B + §C
|
|
13
|
+
* @see SECURITY/invariants.yaml node-pack-sandbox-no-network-escape
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it } from 'vitest';
|
|
17
|
+
|
|
18
|
+
// Behavioral assertion lands when a sandbox-executing host advertises
|
|
19
|
+
// `capabilities.sandbox.supported: true` (with `host.fetch` NOT in
|
|
20
|
+
// `allowedHostCalls`) AND ships a misbehaving-network-escape typeId.
|
|
21
|
+
// The assertion drives the pack to fetch() inside the sandbox and asserts
|
|
22
|
+
// error.code === 'sandbox_capability_denied' with
|
|
23
|
+
// details.requestedCapability === 'host.fetch'. Surfaced as `todo` so
|
|
24
|
+
// test reporters track the gap rather than reporting a vacuous PASS.
|
|
25
|
+
|
|
26
|
+
describe('sandbox-no-network-escape: behavioral (RFC 0035 §B)', () => {
|
|
27
|
+
it.todo('a misbehaving pack fetching without host.fetch in allowedHostCalls fails closed with sandbox_capability_denied');
|
|
28
|
+
});
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sandbox-timeout-cap — RFC 0035 §B invariant `node-pack-sandbox-timeout-cap`.
|
|
3
|
+
*
|
|
4
|
+
* Capability-gated on `capabilities.sandbox.supported: true` AND
|
|
5
|
+
* `capabilities.sandbox.wallClockLimitMs` advertised.
|
|
6
|
+
*
|
|
7
|
+
* Asserts (behavioral when host advertises): a pack invocation whose
|
|
8
|
+
* wall-clock execution exceeds `capabilities.sandbox.wallClockLimitMs`
|
|
9
|
+
* fails closed with `error.code: "sandbox_timeout"` per RFC 0035 §C. The
|
|
10
|
+
* host MUST advertise an integer ≥ 100 ms per the schema.
|
|
11
|
+
*
|
|
12
|
+
* @see RFCS/0035-sandbox-execution-contract.md §B + §C
|
|
13
|
+
* @see SECURITY/invariants.yaml node-pack-sandbox-timeout-cap
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it, expect } from 'vitest';
|
|
17
|
+
import { driver } from '../lib/driver.js';
|
|
18
|
+
|
|
19
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
20
|
+
|
|
21
|
+
interface D {
|
|
22
|
+
capabilities?: { sandbox?: { supported?: unknown; wallClockLimitMs?: unknown } };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function readSandbox(): Promise<{ supported: boolean; wallClockLimitMs?: number } | null> {
|
|
26
|
+
try {
|
|
27
|
+
const r = await driver.get('/.well-known/openwop');
|
|
28
|
+
if (r.status !== 200) return null;
|
|
29
|
+
const sb = (r.json as D).capabilities?.sandbox;
|
|
30
|
+
if (!sb || sb.supported !== true) return null;
|
|
31
|
+
return {
|
|
32
|
+
supported: true,
|
|
33
|
+
...(typeof sb.wallClockLimitMs === 'number' ? { wallClockLimitMs: sb.wallClockLimitMs } : {}),
|
|
34
|
+
};
|
|
35
|
+
} catch { return null; }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
describe.skipIf(HTTP_SKIP)('sandbox-timeout-cap: capability shape + behavioral (RFC 0035 §B)', () => {
|
|
39
|
+
it('wallClockLimitMs MUST be integer ≥ 100 ms when present (per schema)', async () => {
|
|
40
|
+
const sb = await readSandbox();
|
|
41
|
+
if (!sb) return;
|
|
42
|
+
if (sb.wallClockLimitMs === undefined) return; // optional
|
|
43
|
+
|
|
44
|
+
expect(
|
|
45
|
+
Number.isInteger(sb.wallClockLimitMs) && sb.wallClockLimitMs >= 100,
|
|
46
|
+
driver.describe(
|
|
47
|
+
'RFCS/0035-sandbox-execution-contract.md §A',
|
|
48
|
+
'wallClockLimitMs MUST be integer ≥ 100 ms',
|
|
49
|
+
),
|
|
50
|
+
).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
// Behavioral assertion lands when the misbehaving-timeout-cap typeId is
|
|
54
|
+
// available. Expected: error.code === 'sandbox_timeout';
|
|
55
|
+
// details.elapsedMs > wallClockLimitMs. Surfaced as `todo` so test
|
|
56
|
+
// reporters track the gap rather than reporting a vacuous PASS.
|
|
57
|
+
it.todo('a misbehaving pack exceeding wallClockLimitMs fails with sandbox_timeout');
|
|
58
|
+
});
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* search-bm25-roundtrip — RFC 0018 advertisement-shape verification + behavioral
|
|
2
|
+
* search-bm25-roundtrip — RFC 0018 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0018 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.searchIndex` block has
|
|
6
|
-
* `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0018 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.searchIndex` block has
|
|
6
|
+
* landed in `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
+
* advertisement shape against any host that boots the conformance suite, and
|
|
8
|
+
* exercises the behavioral surface through the `/v1/host/sample/test/surface`
|
|
9
|
+
* seam (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: index then query returns relevant documents.
|
|
12
12
|
*
|
|
@@ -408,12 +408,20 @@ function stripFencedCodeBlocks(markdown: string): string {
|
|
|
408
408
|
return markdown.replace(/```[\s\S]*?```/g, '');
|
|
409
409
|
}
|
|
410
410
|
|
|
411
|
+
function stripInlineCodeSpans(markdown: string): string {
|
|
412
|
+
// Strip double-backtick spans first so the inner segment of a span
|
|
413
|
+
// containing a literal backtick (``foo `bar` baz``) doesn't get
|
|
414
|
+
// mis-stripped by the single-backtick pass, leaving stray openers
|
|
415
|
+
// that could pair with later backticks elsewhere in the file.
|
|
416
|
+
return markdown.replace(/``[^`\n]+``/g, '').replace(/`[^`\n]*`/g, '');
|
|
417
|
+
}
|
|
418
|
+
|
|
411
419
|
function extractLocalMarkdownLinks(markdown: string): string[] {
|
|
412
420
|
const links: string[] = [];
|
|
413
421
|
const re = /!?\[[^\]\n]*\]\(([^)\n]+)\)/g;
|
|
414
422
|
let m: RegExpExecArray | null;
|
|
415
423
|
|
|
416
|
-
while ((m = re.exec(stripFencedCodeBlocks(markdown))) !== null) {
|
|
424
|
+
while ((m = re.exec(stripInlineCodeSpans(stripFencedCodeBlocks(markdown)))) !== null) {
|
|
417
425
|
let raw = (m[1] ?? '').trim();
|
|
418
426
|
raw = raw.replace(/\s+"[^"]*"$/, '').trim();
|
|
419
427
|
if (raw.startsWith('<') && raw.endsWith('>')) raw = raw.slice(1, -1);
|
|
@@ -444,6 +452,21 @@ describe('spec-corpus: JSON Schemas compile under Ajv2020', () => {
|
|
|
444
452
|
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
445
453
|
addFormats(ajv);
|
|
446
454
|
|
|
455
|
+
// Pre-register every schema with the Ajv instance so cross-file `$ref`s
|
|
456
|
+
// resolve regardless of compile order. Without this, a cross-ref from
|
|
457
|
+
// an alphabetically-earlier file (e.g. capabilities.schema.json) to a
|
|
458
|
+
// later one (e.g. prompt-kind.schema.json) fails with "can't resolve
|
|
459
|
+
// reference." `addSchema` only registers — it doesn't compile — so
|
|
460
|
+
// per-file compilation errors still surface in their own `it()` below.
|
|
461
|
+
for (const file of schemaFiles) {
|
|
462
|
+
try {
|
|
463
|
+
ajv.addSchema(readJson(join(SCHEMAS_DIR, file)) as Record<string, unknown>);
|
|
464
|
+
} catch {
|
|
465
|
+
// Bad schemas surface in the per-file `compile()` below; swallow
|
|
466
|
+
// here so registration order doesn't short-circuit reporting.
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
447
470
|
it('finds at least three schemas (workflow-definition, run-event, suspend-request)', () => {
|
|
448
471
|
expect(schemaFiles.length).toBeGreaterThanOrEqual(3);
|
|
449
472
|
expect(schemaFiles).toContain('workflow-definition.schema.json');
|
|
@@ -459,8 +482,9 @@ describe('spec-corpus: JSON Schemas compile under Ajv2020', () => {
|
|
|
459
482
|
`https://openwop.dev/spec/v1/${file}`,
|
|
460
483
|
);
|
|
461
484
|
expect(typeof schema['title']).toBe('string');
|
|
462
|
-
//
|
|
463
|
-
|
|
485
|
+
// `compile` uses the schemas registered by `addSchema` above to
|
|
486
|
+
// resolve cross-file `$ref`s — throws on structural issues.
|
|
487
|
+
const validate = ajv.getSchema(schema['$id'] as string) ?? ajv.compile(schema);
|
|
464
488
|
expect(typeof validate).toBe('function');
|
|
465
489
|
});
|
|
466
490
|
}
|
|
@@ -1240,9 +1264,10 @@ describe.skipIf(FIXTURES_DOC_PATH === null)('spec-corpus: fixtures.json catalog
|
|
|
1240
1264
|
// FIXTURES_DOC_PATH is non-null here — assertion narrows for TS.
|
|
1241
1265
|
const fixturesDocPath = FIXTURES_DOC_PATH as string;
|
|
1242
1266
|
const PACK_MANIFEST_FIXTURES_DIR = join(FIXTURES_DIR, 'pack-manifests');
|
|
1243
|
-
|
|
1244
|
-
//
|
|
1245
|
-
//
|
|
1267
|
+
const PROMPT_TEMPLATE_FIXTURES_DIR = join(FIXTURES_DIR, 'prompt-templates');
|
|
1268
|
+
// Top-level workflow fixtures + pack-manifest fixtures + prompt-
|
|
1269
|
+
// template fixtures from their respective sub-directories. All are
|
|
1270
|
+
// documented in fixtures.md so the regex scan below MUST cover them.
|
|
1246
1271
|
const fixtureJsonFiles = [
|
|
1247
1272
|
...readdirSync(FIXTURES_DIR)
|
|
1248
1273
|
.filter((f) => f.endsWith('.json'))
|
|
@@ -1250,6 +1275,9 @@ describe.skipIf(FIXTURES_DOC_PATH === null)('spec-corpus: fixtures.json catalog
|
|
|
1250
1275
|
...readdirSync(PACK_MANIFEST_FIXTURES_DIR)
|
|
1251
1276
|
.filter((f) => f.endsWith('.json'))
|
|
1252
1277
|
.map((f) => f.replace(/\.json$/, '')),
|
|
1278
|
+
...readdirSync(PROMPT_TEMPLATE_FIXTURES_DIR)
|
|
1279
|
+
.filter((f) => f.endsWith('.json'))
|
|
1280
|
+
.map((f) => f.replace(/\.json$/, '')),
|
|
1253
1281
|
].sort();
|
|
1254
1282
|
|
|
1255
1283
|
it('every fixture id mentioned in fixtures.md has a corresponding JSON', () => {
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* sql-transaction-atomicity — RFC 0018 advertisement-shape verification + behavioral
|
|
2
|
+
* sql-transaction-atomicity — RFC 0018 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0018 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.sql` block has landed in
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0018 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.sql` block has landed in
|
|
6
6
|
* `schemas/capabilities.schema.json`. This scenario asserts the advertisement
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* shape against any host that boots the conformance suite, and exercises the
|
|
8
|
+
* behavioral surface through the `/v1/host/sample/test/surface` seam
|
|
9
|
+
* (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: transactions MUST be atomic; partial failure rolls back.
|
|
12
12
|
*
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* stream-subscribe-from-beginning — RFC 0017 advertisement-shape verification + behavioral
|
|
2
|
+
* stream-subscribe-from-beginning — RFC 0017 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0017 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.queueBus` block has
|
|
6
|
-
* `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0017 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.queueBus` block has
|
|
6
|
+
* landed in `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
+
* advertisement shape against any host that boots the conformance suite, and
|
|
8
|
+
* exercises the behavioral surface through the `/v1/host/sample/test/surface`
|
|
9
|
+
* seam (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: Stream subscribers with fromBeginning=true receive records published before subscription.
|
|
12
12
|
*
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import { describe, it, expect } from 'vitest';
|
|
22
22
|
import { driver } from '../lib/driver.js';
|
|
23
|
-
import { pollUntilTerminal } from '../lib/polling.js';
|
|
23
|
+
import { pollUntilTerminal, pollUntilStatus } from '../lib/polling.js';
|
|
24
24
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
25
25
|
import { setHostCapability, resetHostCapabilities, isToggleAvailable } from '../lib/host-toggle.js';
|
|
26
26
|
|
|
@@ -122,9 +122,75 @@ describe.skipIf(SKIP)('subworkflow-input-mapping: parent → child variable seed
|
|
|
122
122
|
expect(v).not.toBe(null);
|
|
123
123
|
});
|
|
124
124
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
125
|
+
// HVMAP-2-no-midrun-propagation: `inputMapping` is a one-shot fold at
|
|
126
|
+
// child-dispatch time. Once the parent's mapping has projected
|
|
127
|
+
// `currentPrdId → receivedPrdId` and the child has been spawned, any
|
|
128
|
+
// mid-run mutation to the parent's `currentPrdId` MUST NOT propagate
|
|
129
|
+
// into the already-seeded child bag. The harness uses the sample-only
|
|
130
|
+
// test seam `POST /v1/host/sample/test/runs/:runId/variables` (gated
|
|
131
|
+
// on `OPENWOP_TEST_SEAM_ENABLED=true`; soft-skip when the seam is not
|
|
132
|
+
// exposed) to mutate the parent's variable bag WHILE the child is
|
|
133
|
+
// suspended on a `core.approvalGate`, then resolves the gate and reads
|
|
134
|
+
// the child's terminal `receivedPrdId` variable.
|
|
135
|
+
const MID_RUN_PARENT = 'conformance-subworkflow-mid-run-mutation';
|
|
136
|
+
const MID_RUN_CHILD = 'conformance-subworkflow-mid-run-mutation-child';
|
|
137
|
+
const CHILD_GATE_NODE = 'child-gate';
|
|
138
|
+
|
|
139
|
+
it('HVMAP-2-no-midrun-propagation: parent mid-run mutation MUST NOT propagate into the seeded child', async () => {
|
|
140
|
+
if (!isFixtureAdvertised(MID_RUN_PARENT) || !isFixtureAdvertised(MID_RUN_CHILD)) return; // fixture not seeded — soft-skip
|
|
141
|
+
if (!(await isToggleAvailable())) return; // sample test seam not exposed — soft-skip
|
|
142
|
+
|
|
143
|
+
const create = await driver.post('/v1/runs', { workflowId: MID_RUN_PARENT });
|
|
144
|
+
expect(create.status).toBe(201);
|
|
145
|
+
const parentRunId = (create.json as { runId: string }).runId;
|
|
146
|
+
|
|
147
|
+
// Wait for the parent to spawn the child and the child to reach
|
|
148
|
+
// `waiting-approval`. The parent's status mirrors the child's
|
|
149
|
+
// suspended kind via the dispatcher's parent-suspends-while-child-
|
|
150
|
+
// suspends contract (interrupt-profiles.md §openwop-interrupt-
|
|
151
|
+
// cascade-cancel).
|
|
152
|
+
await pollUntilStatus(parentRunId, 'waiting-approval', { timeoutMs: 15_000 });
|
|
153
|
+
|
|
154
|
+
// Find the child runId via the parent snapshot's `childRuns[]`
|
|
155
|
+
// projection (interrupt-profiles.md §openwop-interrupt-cascade-cancel).
|
|
156
|
+
const parentSnap = await driver.get(`/v1/runs/${encodeURIComponent(parentRunId)}`);
|
|
157
|
+
const parentJson = parentSnap.json as { childRuns?: Array<{ runId: string; status: string }> };
|
|
158
|
+
const childRunId = parentJson.childRuns?.[0]?.runId;
|
|
159
|
+
expect(childRunId, driver.describe(
|
|
160
|
+
'fixtures.md conformance-subworkflow-mid-run-mutation',
|
|
161
|
+
'parent snapshot MUST surface the spawned child runId via childRuns[]',
|
|
162
|
+
)).toBeDefined();
|
|
163
|
+
|
|
164
|
+
// Mutate the parent's `currentPrdId` WHILE the child is suspended.
|
|
165
|
+
// The mutation MUST NOT propagate per RFC 0022 §B (one-shot fold).
|
|
166
|
+
const mutate = await driver.post(
|
|
167
|
+
`/v1/host/sample/test/runs/${encodeURIComponent(parentRunId)}/variables`,
|
|
168
|
+
{ variables: { currentPrdId: 'mutated-id' } },
|
|
169
|
+
);
|
|
170
|
+
expect(mutate.status).toBe(200);
|
|
171
|
+
|
|
172
|
+
// Resolve the child's approval gate so it terminates.
|
|
173
|
+
const resolve = await driver.post(
|
|
174
|
+
`/v1/runs/${encodeURIComponent(childRunId!)}/interrupts/${encodeURIComponent(CHILD_GATE_NODE)}`,
|
|
175
|
+
{ resumeValue: { action: 'accept' } },
|
|
176
|
+
);
|
|
177
|
+
expect(resolve.status).toBeGreaterThanOrEqual(200);
|
|
178
|
+
expect(resolve.status).toBeLessThan(300);
|
|
179
|
+
|
|
180
|
+
const childTerminal = (await pollUntilTerminal(childRunId!)) as RunSnapshot;
|
|
181
|
+
expect(childTerminal.status).toBe('completed');
|
|
182
|
+
|
|
183
|
+
// The §B one-shot-fold assertion: the child's terminal
|
|
184
|
+
// `receivedPrdId` MUST still equal the dispatch-time fold value
|
|
185
|
+
// (`seeded-id`), NOT the post-mutation parent value (`mutated-id`).
|
|
186
|
+
expect(
|
|
187
|
+
childTerminal.variables?.receivedPrdId,
|
|
188
|
+
driver.describe(
|
|
189
|
+
'RFCS/0022-dispatch-input-output-mapping.md §B',
|
|
190
|
+
'mid-run parent mutation MUST NOT propagate; child receivedPrdId stays at dispatch-time fold',
|
|
191
|
+
),
|
|
192
|
+
).toBe('seeded-id');
|
|
193
|
+
});
|
|
128
194
|
|
|
129
195
|
});
|
|
130
196
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* table-cursor-pagination — RFC 0016 advertisement-shape verification + behavioral
|
|
2
|
+
* table-cursor-pagination — RFC 0016 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0016 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.tableStorage` block has
|
|
6
|
-
* `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0016 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.tableStorage` block has
|
|
6
|
+
* landed in `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
+
* advertisement shape against any host that boots the conformance suite, and
|
|
8
|
+
* exercises the behavioral surface through the `/v1/host/sample/test/surface`
|
|
9
|
+
* seam (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: query MUST support filter + cursor pagination.
|
|
12
12
|
*
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* table-schema-enforcement — RFC 0016 advertisement-shape verification + behavioral
|
|
2
|
+
* table-schema-enforcement — RFC 0016 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0016 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.tableStorage` block has
|
|
6
|
-
* `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0016 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.tableStorage` block has
|
|
6
|
+
* landed in `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
+
* advertisement shape against any host that boots the conformance suite, and
|
|
8
|
+
* exercises the behavioral surface through the `/v1/host/sample/test/surface`
|
|
9
|
+
* seam (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: Subsequent rows MUST conform to the schema established on first insert.
|
|
12
12
|
*
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* vector-knn-roundtrip — RFC 0018 advertisement-shape verification + behavioral
|
|
2
|
+
* vector-knn-roundtrip — RFC 0018 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0018 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.vectorStore` block has
|
|
6
|
-
* `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0018 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.vectorStore` block has
|
|
6
|
+
* landed in `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
+
* advertisement shape against any host that boots the conformance suite, and
|
|
8
|
+
* exercises the behavioral surface through the `/v1/host/sample/test/surface`
|
|
9
|
+
* seam (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: upsert then query returns the same vectors in top-k order.
|
|
12
12
|
*
|