@openwop/openwop-conformance 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -1
- package/README.md +3 -2
- package/api/asyncapi.yaml +8 -0
- package/api/openapi.yaml +371 -1
- package/coverage.md +25 -5
- package/fixtures/conformance-envelope-nl-to-format-engaged.json +41 -0
- package/fixtures/conformance-envelope-recovery-applied.json +39 -0
- package/fixtures/conformance-envelope-refusal.json +38 -0
- package/fixtures/conformance-envelope-retry-attempted.json +39 -0
- package/fixtures/conformance-envelope-retry-exhausted.json +38 -0
- package/fixtures/conformance-envelope-truncated.json +39 -0
- package/fixtures/conformance-envelope-truncation-cap-exhaustion.json +39 -0
- package/fixtures/conformance-model-capability-insufficient.json +25 -0
- package/fixtures/conformance-multi-agent-confidence-escalation.json +49 -0
- package/fixtures/conformance-multi-agent-handoff-child.json +27 -0
- package/fixtures/conformance-multi-agent-handoff.json +49 -0
- package/fixtures/conformance-prompt-all-four-kinds.json +39 -0
- package/fixtures/conformance-prompt-end-to-end.json +33 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation-child.json +31 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation.json +33 -0
- package/fixtures/openwop-smoke-cost-emit.json +37 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot-2.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-schema-hint.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-secret-redaction.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-trust-marker.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-system.json +15 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-user.json +15 -0
- package/fixtures.md +39 -0
- package/package.json +1 -1
- package/schemas/README.md +5 -0
- package/schemas/agent-manifest.schema.json +16 -0
- package/schemas/capabilities.schema.json +375 -1
- package/schemas/envelopes/clarification.request.schema.json +9 -0
- package/schemas/envelopes/error.schema.json +4 -0
- package/schemas/envelopes/schema.request.schema.json +4 -0
- package/schemas/envelopes/schema.response.schema.json +1 -1
- package/schemas/node-pack-manifest.schema.json +28 -0
- package/schemas/orchestrator-decision.schema.json +12 -0
- package/schemas/prompt-kind.schema.json +8 -0
- package/schemas/prompt-pack-manifest.schema.json +80 -0
- package/schemas/prompt-ref.schema.json +40 -0
- package/schemas/prompt-template.schema.json +149 -0
- package/schemas/registry-version-manifest.schema.json +5 -0
- package/schemas/run-ancestry-response.schema.json +54 -0
- package/schemas/run-event-payloads.schema.json +479 -11
- package/schemas/run-event.schema.json +15 -1
- package/schemas/run-snapshot.schema.json +3 -2
- package/schemas/workflow-definition.schema.json +19 -1
- package/src/lib/llm-cache-key-recipe.ts +68 -0
- package/src/scenarios/aiEnvelope.contractRefusal.test.ts +104 -13
- package/src/scenarios/aiEnvelope.correlationReplay.test.ts +32 -15
- package/src/scenarios/aiEnvelope.redaction.test.ts +6 -5
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +5 -5
- package/src/scenarios/aiEnvelope.trustBoundaryPropagation.test.ts +211 -12
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +7 -7
- package/src/scenarios/blob-presign-expiry.test.ts +7 -7
- package/src/scenarios/cache-ttl-expiry.test.ts +6 -6
- package/src/scenarios/cost-attribution.test.ts +124 -11
- package/src/scenarios/cross-engine-append-ordering.test.ts +99 -0
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +136 -0
- package/src/scenarios/cross-host-causation-shape.test.ts +117 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +60 -0
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +223 -0
- package/src/scenarios/envelope-nl-to-format-engaged.test.ts +152 -0
- package/src/scenarios/envelope-reasoning-secret-redaction.test.ts +343 -0
- package/src/scenarios/envelope-reasoning-shape.test.ts +190 -0
- package/src/scenarios/envelope-recovery-applied.test.ts +229 -0
- package/src/scenarios/envelope-refusal-shape.test.ts +289 -0
- package/src/scenarios/envelope-retry-attempted.test.ts +258 -0
- package/src/scenarios/envelope-retry-exhausted.test.ts +168 -0
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +229 -0
- package/src/scenarios/envelope-truncated.test.ts +136 -0
- package/src/scenarios/envelope-truncation-cap-exhaustion.test.ts +144 -0
- package/src/scenarios/envelope-variant-discriminator-static.test.ts +152 -0
- package/src/scenarios/fixtures-valid.test.ts +123 -15
- package/src/scenarios/kv-ttl-expiry.test.ts +7 -7
- package/src/scenarios/model-capability-insufficient.test.ts +221 -0
- package/src/scenarios/model-capability-substituted.test.ts +203 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +164 -0
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +167 -0
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +124 -0
- package/src/scenarios/multi-region-idempotency.test.ts +58 -0
- package/src/scenarios/node-module-required-capabilities-shape.test.ts +185 -0
- package/src/scenarios/prompt-all-four-kinds-events.test.ts +198 -0
- package/src/scenarios/prompt-composed-secret-redaction.test.ts +178 -0
- package/src/scenarios/prompt-composed-trust-marker.test.ts +165 -0
- package/src/scenarios/prompt-end-to-end-events.test.ts +202 -0
- package/src/scenarios/prompt-list-and-fetch.test.ts +207 -0
- package/src/scenarios/prompt-mutable-lifecycle.test.ts +216 -0
- package/src/scenarios/prompt-pack-install.test.ts +187 -0
- package/src/scenarios/prompt-render-deterministic.test.ts +240 -0
- package/src/scenarios/prompt-resolution-chain-agent-intrinsic.test.ts +140 -0
- package/src/scenarios/prompt-resolution-chain-fallback-cascade.test.ts +172 -0
- package/src/scenarios/prompt-resolution-chain-node-wins.test.ts +144 -0
- package/src/scenarios/prompt-template-shape.test.ts +359 -0
- package/src/scenarios/queue-ack-nack-dlq.test.ts +7 -7
- package/src/scenarios/queue-publish-consume-roundtrip.test.ts +7 -7
- package/src/scenarios/replay-divergence-at-refusal.test.ts +134 -0
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +197 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +1 -40
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +80 -0
- package/src/scenarios/sandbox-capability-gate-respected.test.ts +31 -0
- package/src/scenarios/sandbox-memory-cap.test.ts +61 -0
- package/src/scenarios/sandbox-no-cross-pack-mutation.test.ts +35 -0
- package/src/scenarios/sandbox-no-host-env-leak.test.ts +38 -0
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +91 -0
- package/src/scenarios/sandbox-no-host-process-escape.test.ts +30 -0
- package/src/scenarios/sandbox-no-network-escape.test.ts +49 -0
- package/src/scenarios/sandbox-timeout-cap.test.ts +61 -0
- package/src/scenarios/search-bm25-roundtrip.test.ts +7 -7
- package/src/scenarios/spec-corpus-validity.test.ts +34 -6
- package/src/scenarios/sql-transaction-atomicity.test.ts +6 -6
- package/src/scenarios/stream-subscribe-from-beginning.test.ts +7 -7
- package/src/scenarios/subworkflow-input-mapping.test.ts +70 -4
- package/src/scenarios/table-cursor-pagination.test.ts +7 -7
- package/src/scenarios/table-schema-enforcement.test.ts +7 -7
- package/src/scenarios/vector-knn-roundtrip.test.ts +7 -7
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-resolution-chain-fallback-cascade — RFC 0029 §A layer-3 + layer-4
|
|
3
|
+
* fallback cascade.
|
|
4
|
+
*
|
|
5
|
+
* Asserts:
|
|
6
|
+
* 1. When neither node nor agent yields a ref, the workflow's
|
|
7
|
+
* `defaults.promptRefs[kind]` wins (layer 3 — `workflow-defaults`).
|
|
8
|
+
* 2. When workflow defaults are also absent, host's
|
|
9
|
+
* `capabilities.prompts.defaults[kind]` wins (layer 4 —
|
|
10
|
+
* `host-defaults`).
|
|
11
|
+
* 3. When all four layers yield null, `resolved` is null and the
|
|
12
|
+
* emitted event's chain[] still lists every layer attempted with
|
|
13
|
+
* `applied: false`.
|
|
14
|
+
*
|
|
15
|
+
* Capability-gated: skips when the host doesn't advertise
|
|
16
|
+
* `capabilities.prompts.supported: true`.
|
|
17
|
+
*
|
|
18
|
+
* HTTP-driven: skips when no `OPENWOP_BASE_URL` is configured.
|
|
19
|
+
*
|
|
20
|
+
*
|
|
21
|
+
* Under `OPENWOP_REQUIRE_BEHAVIOR=true` the capability gate hardens
|
|
22
|
+
* from SKIP to FAIL — a host that advertises the gating capability
|
|
23
|
+
* but doesn't emit the asserted contract fails the scenario instead
|
|
24
|
+
* of silently skipping. See `conformance/coverage.md` §"Capability-
|
|
25
|
+
* gated scenarios."
|
|
26
|
+
*
|
|
27
|
+
* @see spec/v1/prompts.md §"Resolution chain (normative)" — Layers 3 + 4
|
|
28
|
+
* @see RFCS/0029-prompt-override-hierarchy.md §A
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { describe, it, expect } from 'vitest';
|
|
32
|
+
import { driver } from '../lib/driver.js';
|
|
33
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
34
|
+
|
|
35
|
+
interface DiscoveryDoc {
|
|
36
|
+
capabilities?: {
|
|
37
|
+
prompts?: {
|
|
38
|
+
supported?: unknown;
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
interface AgentPromptResolvedPayload {
|
|
44
|
+
nodeId: string;
|
|
45
|
+
kind: string;
|
|
46
|
+
agentId?: string;
|
|
47
|
+
chain: Array<{
|
|
48
|
+
layer: string;
|
|
49
|
+
source?: string;
|
|
50
|
+
applied: boolean;
|
|
51
|
+
reason?: string;
|
|
52
|
+
}>;
|
|
53
|
+
resolved: string | null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
57
|
+
const res = await driver.get('/.well-known/openwop');
|
|
58
|
+
if (res.status !== 200) return null;
|
|
59
|
+
return res.json as DiscoveryDoc;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function promptsSupported(d: DiscoveryDoc | null): boolean {
|
|
63
|
+
return d?.capabilities?.prompts?.supported === true;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
67
|
+
|
|
68
|
+
describe.skipIf(HTTP_SKIP)('prompt-resolution-chain-fallback-cascade: layers 3 + 4 fallback when node + agent yield null (RFC 0029 §A)', () => {
|
|
69
|
+
it('workflow defaults win over host defaults when both are set', async () => {
|
|
70
|
+
const d = await readDiscovery();
|
|
71
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
72
|
+
|
|
73
|
+
const res = await driver.post('/v1/host/sample/prompt/resolve', {
|
|
74
|
+
kind: 'system',
|
|
75
|
+
node: {
|
|
76
|
+
nodeId: 'writer',
|
|
77
|
+
config: {
|
|
78
|
+
// No agent binding, no layer-1 ref.
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
workflowDefaults: {
|
|
82
|
+
promptRefs: {
|
|
83
|
+
system: 'prompt:workflow-fallback@1.0.0',
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
hostDefaults: {
|
|
87
|
+
system: 'prompt:host-default@1.0.0',
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
if (res.status === 404) return;
|
|
91
|
+
expect(res.status).toBe(200);
|
|
92
|
+
|
|
93
|
+
const payload = res.json as AgentPromptResolvedPayload;
|
|
94
|
+
const applied = payload.chain.find((c) => c.applied);
|
|
95
|
+
expect(applied?.layer).toBe('workflow-defaults');
|
|
96
|
+
expect(
|
|
97
|
+
payload.resolved,
|
|
98
|
+
driver.describe(
|
|
99
|
+
'spec/v1/prompts.md §Resolution chain (normative) — Layer 3',
|
|
100
|
+
'workflow-defaults MUST win over host-defaults when both are set',
|
|
101
|
+
),
|
|
102
|
+
).toBe('prompt:workflow-fallback@1.0.0');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('host defaults win when workflow defaults are also absent', async () => {
|
|
106
|
+
const d = await readDiscovery();
|
|
107
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
108
|
+
|
|
109
|
+
const res = await driver.post('/v1/host/sample/prompt/resolve', {
|
|
110
|
+
kind: 'system',
|
|
111
|
+
node: { nodeId: 'writer', config: {} },
|
|
112
|
+
// workflowDefaults intentionally omitted.
|
|
113
|
+
hostDefaults: {
|
|
114
|
+
system: 'prompt:host-default@1.0.0',
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
if (res.status === 404) return;
|
|
118
|
+
expect(res.status).toBe(200);
|
|
119
|
+
|
|
120
|
+
const payload = res.json as AgentPromptResolvedPayload;
|
|
121
|
+
const applied = payload.chain.find((c) => c.applied);
|
|
122
|
+
expect(
|
|
123
|
+
applied?.layer,
|
|
124
|
+
driver.describe(
|
|
125
|
+
'spec/v1/prompts.md §Resolution chain (normative) — Layer 4',
|
|
126
|
+
'host-defaults MUST apply when layers 1-3 yield null',
|
|
127
|
+
),
|
|
128
|
+
).toBe('host-defaults');
|
|
129
|
+
expect(payload.resolved).toBe('prompt:host-default@1.0.0');
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('resolved is null and chain[] still lists every attempted layer when all four yield null', async () => {
|
|
133
|
+
const d = await readDiscovery();
|
|
134
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
135
|
+
|
|
136
|
+
const res = await driver.post('/v1/host/sample/prompt/resolve', {
|
|
137
|
+
kind: 'system',
|
|
138
|
+
node: { nodeId: 'writer', config: {} },
|
|
139
|
+
// workflowDefaults + hostDefaults intentionally omitted.
|
|
140
|
+
});
|
|
141
|
+
if (res.status === 404) return;
|
|
142
|
+
expect(res.status).toBe(200);
|
|
143
|
+
|
|
144
|
+
const payload = res.json as AgentPromptResolvedPayload;
|
|
145
|
+
expect(
|
|
146
|
+
payload.resolved,
|
|
147
|
+
driver.describe(
|
|
148
|
+
'spec/v1/prompts.md §Resolution chain (normative)',
|
|
149
|
+
'resolved MUST be null when all four layers yield null',
|
|
150
|
+
),
|
|
151
|
+
).toBe(null);
|
|
152
|
+
|
|
153
|
+
const appliedEntries = payload.chain.filter((c) => c.applied);
|
|
154
|
+
expect(
|
|
155
|
+
appliedEntries.length,
|
|
156
|
+
driver.describe(
|
|
157
|
+
'spec/v1/prompts.md §Resolution chain (normative)',
|
|
158
|
+
'zero applied entries when no layer yielded a candidate',
|
|
159
|
+
),
|
|
160
|
+
).toBe(0);
|
|
161
|
+
|
|
162
|
+
// Chain MUST still document each layer attempted so debuggers can
|
|
163
|
+
// see why the resolution returned null.
|
|
164
|
+
expect(
|
|
165
|
+
payload.chain.length,
|
|
166
|
+
driver.describe(
|
|
167
|
+
'spec/v1/prompts.md §Resolution chain (normative)',
|
|
168
|
+
'chain[] MUST list every layer attempted, even when none applied',
|
|
169
|
+
),
|
|
170
|
+
).toBeGreaterThan(0);
|
|
171
|
+
});
|
|
172
|
+
});
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-resolution-chain-node-wins — RFC 0029 §A layer-1 precedence.
|
|
3
|
+
*
|
|
4
|
+
* Asserts: when a workflow node carries `config.systemPromptRef` AND the
|
|
5
|
+
* node is bound to an agent whose `AgentManifest.promptOverrides.system`
|
|
6
|
+
* AND `AgentManifest.systemPromptRef` are both set, the layer-1 node-
|
|
7
|
+
* config ref wins. The emitted `agent.promptResolved.chain[0]` MUST be
|
|
8
|
+
* `layer: "node"` with `applied: true`, and `resolved` MUST equal the
|
|
9
|
+
* node's ref.
|
|
10
|
+
*
|
|
11
|
+
* Capability-gated: skips when the host doesn't advertise
|
|
12
|
+
* `capabilities.prompts.supported: true` (resolution is gated on Phase A).
|
|
13
|
+
*
|
|
14
|
+
* HTTP-driven: skips when no `OPENWOP_BASE_URL` is configured (the
|
|
15
|
+
* server-free subset of the gate can't exercise this — it requires a
|
|
16
|
+
* live reference-host resolution seam).
|
|
17
|
+
*
|
|
18
|
+
*
|
|
19
|
+
* Under `OPENWOP_REQUIRE_BEHAVIOR=true` the capability gate hardens
|
|
20
|
+
* from SKIP to FAIL — a host that advertises the gating capability
|
|
21
|
+
* but doesn't emit the asserted contract fails the scenario instead
|
|
22
|
+
* of silently skipping. See `conformance/coverage.md` §"Capability-
|
|
23
|
+
* gated scenarios."
|
|
24
|
+
*
|
|
25
|
+
* @see spec/v1/prompts.md §"Resolution chain (normative)" — Layer 1
|
|
26
|
+
* @see RFCS/0029-prompt-override-hierarchy.md §A
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { describe, it, expect } from 'vitest';
|
|
30
|
+
import { driver } from '../lib/driver.js';
|
|
31
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
32
|
+
|
|
33
|
+
interface DiscoveryDoc {
|
|
34
|
+
capabilities?: {
|
|
35
|
+
prompts?: {
|
|
36
|
+
supported?: unknown;
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
interface AgentPromptResolvedPayload {
|
|
42
|
+
nodeId: string;
|
|
43
|
+
kind: string;
|
|
44
|
+
agentId?: string;
|
|
45
|
+
chain: Array<{
|
|
46
|
+
layer: string;
|
|
47
|
+
source?: string;
|
|
48
|
+
applied: boolean;
|
|
49
|
+
reason?: string;
|
|
50
|
+
}>;
|
|
51
|
+
resolved: string | null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
55
|
+
const res = await driver.get('/.well-known/openwop');
|
|
56
|
+
if (res.status !== 200) return null;
|
|
57
|
+
return res.json as DiscoveryDoc;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function promptsSupported(d: DiscoveryDoc | null): boolean {
|
|
61
|
+
return d?.capabilities?.prompts?.supported === true;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
65
|
+
|
|
66
|
+
describe.skipIf(HTTP_SKIP)('prompt-resolution-chain-node-wins: layer-1 node-config supersedes lower layers (RFC 0029 §A)', () => {
|
|
67
|
+
it('node-level systemPromptRef wins over agent intrinsic + workflow defaults + host defaults', async () => {
|
|
68
|
+
const d = await readDiscovery();
|
|
69
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
70
|
+
|
|
71
|
+
// Driver test-seam endpoint: instructs the reference host to resolve
|
|
72
|
+
// a PromptRef for a (nodeId, kind) pair against a fixture inputs
|
|
73
|
+
// bundle that exercises every layer of the chain. Returns the
|
|
74
|
+
// emitted `agent.promptResolved` event payload synchronously so the
|
|
75
|
+
// scenario can assert without subscribing to the run event log.
|
|
76
|
+
const res = await driver.post('/v1/host/sample/prompt/resolve', {
|
|
77
|
+
kind: 'system',
|
|
78
|
+
node: {
|
|
79
|
+
nodeId: 'writer',
|
|
80
|
+
config: {
|
|
81
|
+
// Layer 1 explicit ref — should win.
|
|
82
|
+
systemPromptRef: 'prompt:experimental-writer@2.0.0',
|
|
83
|
+
agentId: 'vendor.acme.writer-agent',
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
agentManifest: {
|
|
87
|
+
agentId: 'vendor.acme.writer-agent',
|
|
88
|
+
// Layer 2 candidates (intrinsic + overrides) — should be
|
|
89
|
+
// recorded in the chain with applied: false.
|
|
90
|
+
systemPromptRef: 'prompts/intrinsic.md',
|
|
91
|
+
promptOverrides: {
|
|
92
|
+
system: 'prompt:editorial-house-style@1.0.0',
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
workflowDefaults: {
|
|
96
|
+
promptRefs: {
|
|
97
|
+
// Layer 3 candidate — should be applied: false.
|
|
98
|
+
system: 'prompt:workflow-default@1.0.0',
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
hostDefaults: {
|
|
102
|
+
// Layer 4 candidate — should be applied: false.
|
|
103
|
+
system: 'prompt:host-default@1.0.0',
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
if (res.status === 404) return; // host doesn't expose the seam
|
|
107
|
+
expect(res.status, 'resolve seam MUST return 200').toBe(200);
|
|
108
|
+
|
|
109
|
+
const payload = res.json as AgentPromptResolvedPayload;
|
|
110
|
+
|
|
111
|
+
expect(
|
|
112
|
+
payload.kind,
|
|
113
|
+
driver.describe(
|
|
114
|
+
'spec/v1/prompts.md §Resolution chain (normative)',
|
|
115
|
+
'agent.promptResolved.kind MUST match the requested kind',
|
|
116
|
+
),
|
|
117
|
+
).toBe('system');
|
|
118
|
+
|
|
119
|
+
expect(
|
|
120
|
+
payload.resolved,
|
|
121
|
+
driver.describe(
|
|
122
|
+
'spec/v1/prompts.md §Resolution chain (normative) — Layer 1',
|
|
123
|
+
'node-level systemPromptRef MUST win over agent intrinsic + overrides + workflow defaults + host defaults',
|
|
124
|
+
),
|
|
125
|
+
).toBe('prompt:experimental-writer@2.0.0');
|
|
126
|
+
|
|
127
|
+
// chain[] MUST list the layers attempted in precedence order. The
|
|
128
|
+
// node layer (first entry of the resolution chain) MUST carry
|
|
129
|
+
// applied: true; every other layer applied: false.
|
|
130
|
+
expect(Array.isArray(payload.chain), 'agent.promptResolved.chain MUST be an array').toBe(true);
|
|
131
|
+
expect(payload.chain.length).toBeGreaterThan(0);
|
|
132
|
+
|
|
133
|
+
const appliedEntries = payload.chain.filter((c) => c.applied);
|
|
134
|
+
expect(
|
|
135
|
+
appliedEntries.length,
|
|
136
|
+
driver.describe(
|
|
137
|
+
'spec/v1/prompts.md §Resolution chain (normative)',
|
|
138
|
+
'exactly one chain entry MUST carry applied: true',
|
|
139
|
+
),
|
|
140
|
+
).toBe(1);
|
|
141
|
+
expect(appliedEntries[0]?.layer).toBe('node');
|
|
142
|
+
expect(appliedEntries[0]?.source).toBe('prompt:experimental-writer@2.0.0');
|
|
143
|
+
});
|
|
144
|
+
});
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-template-shape — RFC 0027 §A + §B + §C wire-shape conformance.
|
|
3
|
+
*
|
|
4
|
+
* Asserts:
|
|
5
|
+
* 1. `schemas/prompt-kind.schema.json` is Ajv2020-compileable as a
|
|
6
|
+
* `type: string` enum with the four canonical values.
|
|
7
|
+
* 2. `schemas/prompt-template.schema.json` compiles AND its `kind`
|
|
8
|
+
* cross-ref to `prompt-kind.schema.json` resolves via Ajv's
|
|
9
|
+
* schema registry.
|
|
10
|
+
* 3. `schemas/prompt-ref.schema.json` compiles AND accepts both
|
|
11
|
+
* stringy and object forms; rejects malformed strings.
|
|
12
|
+
* 4. A positive PromptTemplate fixture round-trips; negative
|
|
13
|
+
* fixtures (missing required fields, bad templateId pattern,
|
|
14
|
+
* bad SemVer) reject.
|
|
15
|
+
* 5. `capabilities.prompts` block advertisement (when present)
|
|
16
|
+
* conforms to the optional shape per RFC 0027 §D.
|
|
17
|
+
*
|
|
18
|
+
* NOT capability-gated — schema-shape compilation always runs.
|
|
19
|
+
* Discovery-doc advertisement check soft-skips when no live host is
|
|
20
|
+
* configured.
|
|
21
|
+
*
|
|
22
|
+
* @see RFCS/0027-prompt-templates.md
|
|
23
|
+
* @see spec/v1/prompts.md
|
|
24
|
+
* @see schemas/prompt-template.schema.json
|
|
25
|
+
* @see schemas/prompt-ref.schema.json
|
|
26
|
+
* @see schemas/prompt-kind.schema.json
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { describe, it, expect } from 'vitest';
|
|
30
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
31
|
+
import addFormats from 'ajv-formats';
|
|
32
|
+
import { readFileSync } from 'node:fs';
|
|
33
|
+
import { join } from 'node:path';
|
|
34
|
+
import { driver } from '../lib/driver.js';
|
|
35
|
+
import { SCHEMAS_DIR } from '../lib/paths.js';
|
|
36
|
+
|
|
37
|
+
const PROMPT_KIND_VALUES = ['system', 'user', 'few-shot', 'schema-hint'] as const;
|
|
38
|
+
|
|
39
|
+
interface DiscoveryDoc {
|
|
40
|
+
capabilities?: {
|
|
41
|
+
prompts?: {
|
|
42
|
+
supported?: unknown;
|
|
43
|
+
templateKinds?: unknown;
|
|
44
|
+
variableSources?: unknown;
|
|
45
|
+
maxTemplateBytes?: unknown;
|
|
46
|
+
observability?: unknown;
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
52
|
+
const res = await driver.get('/.well-known/openwop');
|
|
53
|
+
if (res.status !== 200) return null;
|
|
54
|
+
return res.json as DiscoveryDoc;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function loadSchema(rel: string): Record<string, unknown> {
|
|
58
|
+
return JSON.parse(readFileSync(join(SCHEMAS_DIR, rel), 'utf8')) as Record<string, unknown>;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Pre-load the three RFC 0027 schemas into a shared Ajv instance so
|
|
62
|
+
// cross-schema `$ref`s (prompt-template → prompt-kind) resolve when
|
|
63
|
+
// validating. Mirrors the `agent-ref` / `agent-manifest` pre-load
|
|
64
|
+
// pattern in fixtures-valid.test.ts.
|
|
65
|
+
function makeAjv(): Ajv2020 {
|
|
66
|
+
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
67
|
+
addFormats(ajv);
|
|
68
|
+
// Parse prompt-kind once so the second alias key shares the same
|
|
69
|
+
// object reference — Ajv's `_checkUnique` allows duplicate registration
|
|
70
|
+
// of the same schema instance but throws when two distinct objects
|
|
71
|
+
// declare the same `$id` (see fixtures-valid.test.ts §"prompt-kind via
|
|
72
|
+
// ./ relative URI" for the canonical pattern).
|
|
73
|
+
const promptKindSchema = loadSchema('prompt-kind.schema.json');
|
|
74
|
+
ajv.addSchema(promptKindSchema, 'prompt-kind.schema.json');
|
|
75
|
+
ajv.addSchema(promptKindSchema, './prompt-kind.schema.json');
|
|
76
|
+
// Do NOT pre-register prompt-template / prompt-ref here. Each
|
|
77
|
+
// describe block calls `ajv.compile(loadSchema(...))` with a freshly
|
|
78
|
+
// parsed object; pre-registering causes `_checkUnique` to throw on
|
|
79
|
+
// the duplicate `$id`. prompt-kind stays pre-registered because
|
|
80
|
+
// prompt-template `$ref`s it relatively and needs it resolvable.
|
|
81
|
+
return ajv;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
85
|
+
|
|
86
|
+
describe('prompt-template-shape: schema compile (RFC 0027 §A)', () => {
|
|
87
|
+
const ajv = makeAjv();
|
|
88
|
+
|
|
89
|
+
it('prompt-kind.schema.json compiles and is a string enum of the four canonical kinds', () => {
|
|
90
|
+
const schema = loadSchema('prompt-kind.schema.json');
|
|
91
|
+
// Reuse the already-registered validator when present — Ajv refuses
|
|
92
|
+
// to re-`compile` a schema whose `$id` it already knows.
|
|
93
|
+
const validate = ajv.getSchema(schema['$id'] as string) ?? ajv.compile(schema);
|
|
94
|
+
expect(validate, 'RFC 0027 §A: prompt-kind.schema.json MUST compile').toBeTypeOf('function');
|
|
95
|
+
expect(schema.type, 'prompt-kind MUST be type: string').toBe('string');
|
|
96
|
+
expect(
|
|
97
|
+
schema.enum,
|
|
98
|
+
driver.describe(
|
|
99
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
100
|
+
'prompt-kind enum MUST contain exactly the four canonical values',
|
|
101
|
+
),
|
|
102
|
+
).toEqual(PROMPT_KIND_VALUES);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('prompt-template.schema.json compiles with cross-ref to prompt-kind', () => {
|
|
106
|
+
const schema = loadSchema('prompt-template.schema.json');
|
|
107
|
+
const validate = ajv.compile(schema);
|
|
108
|
+
expect(validate, 'RFC 0027 §A: prompt-template.schema.json MUST compile').toBeTypeOf('function');
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('prompt-ref.schema.json compiles', () => {
|
|
112
|
+
const schema = loadSchema('prompt-ref.schema.json');
|
|
113
|
+
const validate = ajv.compile(schema);
|
|
114
|
+
expect(validate, 'RFC 0027 §B: prompt-ref.schema.json MUST compile').toBeTypeOf('function');
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe('prompt-template-shape: PromptTemplate round-trip (RFC 0027 §A)', () => {
|
|
119
|
+
const ajv = makeAjv();
|
|
120
|
+
const validate = ajv.compile(loadSchema('prompt-template.schema.json'));
|
|
121
|
+
|
|
122
|
+
it('accepts a minimal positive PromptTemplate fixture', () => {
|
|
123
|
+
const positive = {
|
|
124
|
+
templateId: 'writer-system',
|
|
125
|
+
version: '1.0.0',
|
|
126
|
+
kind: 'system',
|
|
127
|
+
text: 'You are a careful editorial writer.',
|
|
128
|
+
};
|
|
129
|
+
const ok = validate(positive);
|
|
130
|
+
expect(
|
|
131
|
+
ok,
|
|
132
|
+
`RFC 0027 §A: minimal PromptTemplate MUST validate; errors: ${JSON.stringify(validate.errors)}`,
|
|
133
|
+
).toBe(true);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('accepts a PromptTemplate with typed variables + modelHints + meta', () => {
|
|
137
|
+
const positive = {
|
|
138
|
+
templateId: 'writer-user',
|
|
139
|
+
version: '1.2.3',
|
|
140
|
+
kind: 'user',
|
|
141
|
+
text: 'Write about: {{topic}}\nTone: {{tone}}',
|
|
142
|
+
name: 'Writer (user template)',
|
|
143
|
+
description: 'Two-variable user template.',
|
|
144
|
+
variables: [
|
|
145
|
+
{ name: 'topic', type: 'string', required: true, source: 'input' },
|
|
146
|
+
{ name: 'tone', type: 'string', required: false, source: 'input', defaultValue: 'neutral' },
|
|
147
|
+
],
|
|
148
|
+
modelHints: { modelClass: 'writing', temperature: 0.7 },
|
|
149
|
+
tags: ['editorial', 'writing'],
|
|
150
|
+
meta: {
|
|
151
|
+
author: 'openwop-conformance',
|
|
152
|
+
createdAt: '2026-05-20T10:00:00Z',
|
|
153
|
+
source: 'host',
|
|
154
|
+
},
|
|
155
|
+
};
|
|
156
|
+
const ok = validate(positive);
|
|
157
|
+
expect(
|
|
158
|
+
ok,
|
|
159
|
+
`RFC 0027 §A: full PromptTemplate MUST validate; errors: ${JSON.stringify(validate.errors)}`,
|
|
160
|
+
).toBe(true);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('rejects a PromptTemplate missing required `text`', () => {
|
|
164
|
+
const negative = {
|
|
165
|
+
templateId: 'writer-system',
|
|
166
|
+
version: '1.0.0',
|
|
167
|
+
kind: 'system',
|
|
168
|
+
// text omitted
|
|
169
|
+
};
|
|
170
|
+
expect(
|
|
171
|
+
validate(negative),
|
|
172
|
+
driver.describe(
|
|
173
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
174
|
+
'PromptTemplate MUST require text field',
|
|
175
|
+
),
|
|
176
|
+
).toBe(false);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('rejects a PromptTemplate with non-SemVer version', () => {
|
|
180
|
+
const negative = {
|
|
181
|
+
templateId: 'writer-system',
|
|
182
|
+
version: 'v1',
|
|
183
|
+
kind: 'system',
|
|
184
|
+
text: 'x',
|
|
185
|
+
};
|
|
186
|
+
expect(
|
|
187
|
+
validate(negative),
|
|
188
|
+
driver.describe(
|
|
189
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
190
|
+
'PromptTemplate.version MUST match SemVer 2.0.0 pattern',
|
|
191
|
+
),
|
|
192
|
+
).toBe(false);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('rejects a PromptTemplate with invalid templateId (uppercase letter)', () => {
|
|
196
|
+
const negative = {
|
|
197
|
+
templateId: 'Writer-System',
|
|
198
|
+
version: '1.0.0',
|
|
199
|
+
kind: 'system',
|
|
200
|
+
text: 'x',
|
|
201
|
+
};
|
|
202
|
+
expect(
|
|
203
|
+
validate(negative),
|
|
204
|
+
driver.describe(
|
|
205
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
206
|
+
'PromptTemplate.templateId MUST match ^[a-z0-9][a-z0-9._-]{0,127}$',
|
|
207
|
+
),
|
|
208
|
+
).toBe(false);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it('rejects a PromptTemplate with kind not in the prompt-kind enum', () => {
|
|
212
|
+
const negative = {
|
|
213
|
+
templateId: 'writer-system',
|
|
214
|
+
version: '1.0.0',
|
|
215
|
+
kind: 'made-up-kind',
|
|
216
|
+
text: 'x',
|
|
217
|
+
};
|
|
218
|
+
expect(
|
|
219
|
+
validate(negative),
|
|
220
|
+
driver.describe(
|
|
221
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
222
|
+
'PromptTemplate.kind MUST be one of the four canonical values',
|
|
223
|
+
),
|
|
224
|
+
).toBe(false);
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
it('rejects an unknown top-level property (additionalProperties:false)', () => {
|
|
228
|
+
const negative = {
|
|
229
|
+
templateId: 'writer-system',
|
|
230
|
+
version: '1.0.0',
|
|
231
|
+
kind: 'system',
|
|
232
|
+
text: 'x',
|
|
233
|
+
unknownExtra: 'should reject',
|
|
234
|
+
};
|
|
235
|
+
expect(
|
|
236
|
+
validate(negative),
|
|
237
|
+
driver.describe(
|
|
238
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
239
|
+
'PromptTemplate top-level additionalProperties:false MUST reject unknown fields',
|
|
240
|
+
),
|
|
241
|
+
).toBe(false);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it('rejects a PromptVariable with bad name pattern (dash)', () => {
|
|
245
|
+
const negative = {
|
|
246
|
+
templateId: 'writer-system',
|
|
247
|
+
version: '1.0.0',
|
|
248
|
+
kind: 'user',
|
|
249
|
+
text: 'x',
|
|
250
|
+
variables: [{ name: 'has-dash', type: 'string', required: true }],
|
|
251
|
+
};
|
|
252
|
+
expect(
|
|
253
|
+
validate(negative),
|
|
254
|
+
driver.describe(
|
|
255
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
256
|
+
'PromptVariable.name MUST match common-templating identifier pattern (no dashes)',
|
|
257
|
+
),
|
|
258
|
+
).toBe(false);
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
describe('prompt-template-shape: PromptRef round-trip (RFC 0027 §B)', () => {
|
|
263
|
+
const ajv = makeAjv();
|
|
264
|
+
const validate = ajv.compile(loadSchema('prompt-ref.schema.json'));
|
|
265
|
+
|
|
266
|
+
it('accepts stringy form without version', () => {
|
|
267
|
+
expect(validate('prompt:writer-system')).toBe(true);
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
it('accepts stringy form with version', () => {
|
|
271
|
+
expect(validate('prompt:writer-system@1.2.3')).toBe(true);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it('accepts stringy form with vendor-prefixed templateId', () => {
|
|
275
|
+
expect(validate('prompt:vendor.acme.writer.v2@2.0.0')).toBe(true);
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
it('accepts object form with templateId only', () => {
|
|
279
|
+
expect(validate({ templateId: 'writer-system' })).toBe(true);
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
it('accepts object form with libraryId, version, and variableOverrides', () => {
|
|
283
|
+
expect(
|
|
284
|
+
validate({
|
|
285
|
+
libraryId: 'vendor.acme.editorial-prompts',
|
|
286
|
+
templateId: 'writer-system',
|
|
287
|
+
version: '1.0.0',
|
|
288
|
+
variableOverrides: { tone: 'formal' },
|
|
289
|
+
}),
|
|
290
|
+
).toBe(true);
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
it('rejects stringy form without `prompt:` prefix', () => {
|
|
294
|
+
expect(
|
|
295
|
+
validate('writer-system'),
|
|
296
|
+
driver.describe('spec/v1/prompts.md §PromptRef', 'stringy PromptRef MUST start with prompt:'),
|
|
297
|
+
).toBe(false);
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
it('rejects stringy form with non-SemVer version', () => {
|
|
301
|
+
expect(
|
|
302
|
+
validate('prompt:writer-system@latest'),
|
|
303
|
+
driver.describe('spec/v1/prompts.md §PromptRef', 'stringy PromptRef version MUST be SemVer'),
|
|
304
|
+
).toBe(false);
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
it('rejects object form missing required templateId', () => {
|
|
308
|
+
expect(
|
|
309
|
+
validate({ version: '1.0.0' }),
|
|
310
|
+
driver.describe('spec/v1/prompts.md §PromptRef', 'object PromptRef MUST require templateId'),
|
|
311
|
+
).toBe(false);
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
it('rejects object form with unknown additional property', () => {
|
|
315
|
+
expect(
|
|
316
|
+
validate({ templateId: 'writer-system', unknownExtra: true }),
|
|
317
|
+
driver.describe(
|
|
318
|
+
'spec/v1/prompts.md §PromptRef',
|
|
319
|
+
'object PromptRef additionalProperties:false MUST reject unknown fields',
|
|
320
|
+
),
|
|
321
|
+
).toBe(false);
|
|
322
|
+
});
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
describe.skipIf(HTTP_SKIP)('prompt-template-shape: capabilities.prompts advertisement (RFC 0027 §D)', () => {
|
|
326
|
+
it('capabilities.prompts (when present) carries the optional shape per RFC 0027 §D', async () => {
|
|
327
|
+
const d = await readDiscovery();
|
|
328
|
+
if (d === null) return;
|
|
329
|
+
const prompts = d.capabilities?.prompts;
|
|
330
|
+
if (prompts === undefined) return; // optional block; host MAY omit
|
|
331
|
+
expect(
|
|
332
|
+
typeof prompts.supported,
|
|
333
|
+
driver.describe(
|
|
334
|
+
'spec/v1/prompts.md §Capability advertisement',
|
|
335
|
+
'capabilities.prompts.supported MUST be boolean when block is advertised',
|
|
336
|
+
),
|
|
337
|
+
).toBe('boolean');
|
|
338
|
+
if (prompts.templateKinds !== undefined) {
|
|
339
|
+
expect(Array.isArray(prompts.templateKinds), 'templateKinds MUST be an array').toBe(true);
|
|
340
|
+
for (const k of prompts.templateKinds as unknown[]) {
|
|
341
|
+
expect((PROMPT_KIND_VALUES as readonly string[]).includes(String(k))).toBe(true);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
if (prompts.variableSources !== undefined) {
|
|
345
|
+
expect(Array.isArray(prompts.variableSources), 'variableSources MUST be an array').toBe(true);
|
|
346
|
+
for (const s of prompts.variableSources as unknown[]) {
|
|
347
|
+
expect(['input', 'variable', 'secret', 'context']).toContain(String(s));
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (prompts.observability !== undefined) {
|
|
351
|
+
expect(['off', 'hashed', 'full']).toContain(String(prompts.observability));
|
|
352
|
+
}
|
|
353
|
+
if (prompts.maxTemplateBytes !== undefined) {
|
|
354
|
+
expect(typeof prompts.maxTemplateBytes, 'maxTemplateBytes MUST be integer').toBe('number');
|
|
355
|
+
expect((prompts.maxTemplateBytes as number) > 0).toBe(true);
|
|
356
|
+
expect((prompts.maxTemplateBytes as number) <= 65536).toBe(true);
|
|
357
|
+
}
|
|
358
|
+
});
|
|
359
|
+
});
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* queue-ack-nack-dlq — RFC 0017 advertisement-shape verification + behavioral
|
|
2
|
+
* queue-ack-nack-dlq — RFC 0017 advertisement-shape verification + behavioral roundtrip.
|
|
3
3
|
*
|
|
4
|
-
* Status: ACTIVE (advertisement-shape). RFC 0017 promoted to
|
|
5
|
-
* 2026-05-17. The matching `capabilities.queueBus` block has
|
|
6
|
-
* `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
-
* shape against any host that boots the conformance suite, and
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Status: ACTIVE (advertisement-shape + behavioral). RFC 0017 promoted to
|
|
5
|
+
* `Active` 2026-05-17. The matching `capabilities.queueBus` block has
|
|
6
|
+
* landed in `schemas/capabilities.schema.json`. This scenario asserts the
|
|
7
|
+
* advertisement shape against any host that boots the conformance suite, and
|
|
8
|
+
* exercises the behavioral surface through the `/v1/host/sample/test/surface`
|
|
9
|
+
* seam (soft-skip with HTTP 404 on hosts that don't expose it).
|
|
10
10
|
*
|
|
11
11
|
* Summary: nack returns for redelivery; deadLetter routes to the configured DLQ.
|
|
12
12
|
*
|