@openwop/openwop-conformance 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -1
- package/README.md +3 -2
- package/api/asyncapi.yaml +8 -0
- package/api/openapi.yaml +371 -1
- package/coverage.md +25 -5
- package/fixtures/conformance-envelope-nl-to-format-engaged.json +41 -0
- package/fixtures/conformance-envelope-recovery-applied.json +39 -0
- package/fixtures/conformance-envelope-refusal.json +38 -0
- package/fixtures/conformance-envelope-retry-attempted.json +39 -0
- package/fixtures/conformance-envelope-retry-exhausted.json +38 -0
- package/fixtures/conformance-envelope-truncated.json +39 -0
- package/fixtures/conformance-envelope-truncation-cap-exhaustion.json +39 -0
- package/fixtures/conformance-model-capability-insufficient.json +25 -0
- package/fixtures/conformance-multi-agent-confidence-escalation.json +49 -0
- package/fixtures/conformance-multi-agent-handoff-child.json +27 -0
- package/fixtures/conformance-multi-agent-handoff.json +49 -0
- package/fixtures/conformance-prompt-all-four-kinds.json +39 -0
- package/fixtures/conformance-prompt-end-to-end.json +33 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation-child.json +31 -0
- package/fixtures/conformance-subworkflow-mid-run-mutation.json +33 -0
- package/fixtures/openwop-smoke-cost-emit.json +37 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot-2.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-few-shot.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-schema-hint.json +14 -0
- package/fixtures/prompt-templates/conformance-prompt-secret-redaction.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-trust-marker.json +23 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-system.json +15 -0
- package/fixtures/prompt-templates/conformance-prompt-writer-user.json +15 -0
- package/fixtures.md +39 -0
- package/package.json +1 -1
- package/schemas/README.md +5 -0
- package/schemas/agent-manifest.schema.json +16 -0
- package/schemas/capabilities.schema.json +375 -1
- package/schemas/envelopes/clarification.request.schema.json +9 -0
- package/schemas/envelopes/error.schema.json +4 -0
- package/schemas/envelopes/schema.request.schema.json +4 -0
- package/schemas/envelopes/schema.response.schema.json +1 -1
- package/schemas/node-pack-manifest.schema.json +28 -0
- package/schemas/orchestrator-decision.schema.json +12 -0
- package/schemas/prompt-kind.schema.json +8 -0
- package/schemas/prompt-pack-manifest.schema.json +80 -0
- package/schemas/prompt-ref.schema.json +40 -0
- package/schemas/prompt-template.schema.json +149 -0
- package/schemas/registry-version-manifest.schema.json +5 -0
- package/schemas/run-ancestry-response.schema.json +54 -0
- package/schemas/run-event-payloads.schema.json +479 -11
- package/schemas/run-event.schema.json +15 -1
- package/schemas/run-snapshot.schema.json +3 -2
- package/schemas/workflow-definition.schema.json +19 -1
- package/src/lib/llm-cache-key-recipe.ts +68 -0
- package/src/scenarios/aiEnvelope.contractRefusal.test.ts +104 -13
- package/src/scenarios/aiEnvelope.correlationReplay.test.ts +32 -15
- package/src/scenarios/aiEnvelope.redaction.test.ts +6 -5
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +5 -5
- package/src/scenarios/aiEnvelope.trustBoundaryPropagation.test.ts +211 -12
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +7 -7
- package/src/scenarios/blob-presign-expiry.test.ts +7 -7
- package/src/scenarios/cache-ttl-expiry.test.ts +6 -6
- package/src/scenarios/cost-attribution.test.ts +124 -11
- package/src/scenarios/cross-engine-append-ordering.test.ts +99 -0
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +136 -0
- package/src/scenarios/cross-host-causation-shape.test.ts +117 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +60 -0
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +223 -0
- package/src/scenarios/envelope-nl-to-format-engaged.test.ts +152 -0
- package/src/scenarios/envelope-reasoning-secret-redaction.test.ts +343 -0
- package/src/scenarios/envelope-reasoning-shape.test.ts +190 -0
- package/src/scenarios/envelope-recovery-applied.test.ts +229 -0
- package/src/scenarios/envelope-refusal-shape.test.ts +289 -0
- package/src/scenarios/envelope-retry-attempted.test.ts +258 -0
- package/src/scenarios/envelope-retry-exhausted.test.ts +168 -0
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +229 -0
- package/src/scenarios/envelope-truncated.test.ts +136 -0
- package/src/scenarios/envelope-truncation-cap-exhaustion.test.ts +144 -0
- package/src/scenarios/envelope-variant-discriminator-static.test.ts +152 -0
- package/src/scenarios/fixtures-valid.test.ts +123 -15
- package/src/scenarios/kv-ttl-expiry.test.ts +7 -7
- package/src/scenarios/model-capability-insufficient.test.ts +221 -0
- package/src/scenarios/model-capability-substituted.test.ts +203 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +164 -0
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +167 -0
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +124 -0
- package/src/scenarios/multi-region-idempotency.test.ts +58 -0
- package/src/scenarios/node-module-required-capabilities-shape.test.ts +185 -0
- package/src/scenarios/prompt-all-four-kinds-events.test.ts +198 -0
- package/src/scenarios/prompt-composed-secret-redaction.test.ts +178 -0
- package/src/scenarios/prompt-composed-trust-marker.test.ts +165 -0
- package/src/scenarios/prompt-end-to-end-events.test.ts +202 -0
- package/src/scenarios/prompt-list-and-fetch.test.ts +207 -0
- package/src/scenarios/prompt-mutable-lifecycle.test.ts +216 -0
- package/src/scenarios/prompt-pack-install.test.ts +187 -0
- package/src/scenarios/prompt-render-deterministic.test.ts +240 -0
- package/src/scenarios/prompt-resolution-chain-agent-intrinsic.test.ts +140 -0
- package/src/scenarios/prompt-resolution-chain-fallback-cascade.test.ts +172 -0
- package/src/scenarios/prompt-resolution-chain-node-wins.test.ts +144 -0
- package/src/scenarios/prompt-template-shape.test.ts +359 -0
- package/src/scenarios/queue-ack-nack-dlq.test.ts +7 -7
- package/src/scenarios/queue-publish-consume-roundtrip.test.ts +7 -7
- package/src/scenarios/replay-divergence-at-refusal.test.ts +134 -0
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +197 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +1 -40
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +80 -0
- package/src/scenarios/sandbox-capability-gate-respected.test.ts +31 -0
- package/src/scenarios/sandbox-memory-cap.test.ts +61 -0
- package/src/scenarios/sandbox-no-cross-pack-mutation.test.ts +35 -0
- package/src/scenarios/sandbox-no-host-env-leak.test.ts +38 -0
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +91 -0
- package/src/scenarios/sandbox-no-host-process-escape.test.ts +30 -0
- package/src/scenarios/sandbox-no-network-escape.test.ts +49 -0
- package/src/scenarios/sandbox-timeout-cap.test.ts +61 -0
- package/src/scenarios/search-bm25-roundtrip.test.ts +7 -7
- package/src/scenarios/spec-corpus-validity.test.ts +34 -6
- package/src/scenarios/sql-transaction-atomicity.test.ts +6 -6
- package/src/scenarios/stream-subscribe-from-beginning.test.ts +7 -7
- package/src/scenarios/subworkflow-input-mapping.test.ts +70 -4
- package/src/scenarios/table-cursor-pagination.test.ts +7 -7
- package/src/scenarios/table-schema-enforcement.test.ts +7 -7
- package/src/scenarios/vector-knn-roundtrip.test.ts +7 -7
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-end-to-end-events — RFC 0027 + RFC 0029 end-to-end emission.
|
|
3
|
+
*
|
|
4
|
+
* Asserts: when a workflow node carries `config.systemPromptRef` and
|
|
5
|
+
* the host advertises `capabilities.prompts.supported: true`,
|
|
6
|
+
* dispatching the run MUST cause the host to emit (in this order)
|
|
7
|
+
* one `agent.promptResolved` event (per RFC 0029 §A — the resolution
|
|
8
|
+
* chain trace with `applied: true` on the winning layer) followed by
|
|
9
|
+
* one `prompt.composed` event (per RFC 0027 §E — the composed body
|
|
10
|
+
* + sha256 hash + per-variable hashes), and the run MUST reach
|
|
11
|
+
* terminal `completed` carrying the composed body's mock-AI
|
|
12
|
+
* completion.
|
|
13
|
+
*
|
|
14
|
+
* This is the integration regression pin: prior conformance scenarios
|
|
15
|
+
* exercised composition + resolution via the host-extension test
|
|
16
|
+
* seams (`/v1/host/sample/prompt/{compose,resolve}`); this scenario
|
|
17
|
+
* exercises the SAME pipeline through real workflow-engine dispatch.
|
|
18
|
+
* If the executor stops wiring the prompt-library helpers into node
|
|
19
|
+
* execution, this scenario fails first.
|
|
20
|
+
*
|
|
21
|
+
* Capability-gated: skips when the host doesn't advertise
|
|
22
|
+
* `capabilities.prompts.supported: true`. Under
|
|
23
|
+
* `OPENWOP_REQUIRE_BEHAVIOR=true`, the gate hardens from SKIP to
|
|
24
|
+
* FAIL via `behaviorGate('prompts-supported', ...)`.
|
|
25
|
+
*
|
|
26
|
+
* @see spec/v1/prompts.md §"Composition + observability"
|
|
27
|
+
* @see spec/v1/prompts.md §"Resolution chain (normative)"
|
|
28
|
+
* @see RFCS/0027-prompt-templates.md §E
|
|
29
|
+
* @see RFCS/0029-prompt-override-hierarchy.md §A
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { describe, it, expect } from 'vitest';
|
|
33
|
+
import { driver } from '../lib/driver.js';
|
|
34
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
35
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
36
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
37
|
+
|
|
38
|
+
const WORKFLOW_ID = 'conformance-prompt-end-to-end';
|
|
39
|
+
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
40
|
+
|
|
41
|
+
interface DiscoveryDoc {
|
|
42
|
+
capabilities?: {
|
|
43
|
+
prompts?: { supported?: unknown };
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface RunEventDoc {
|
|
48
|
+
eventId: string;
|
|
49
|
+
runId: string;
|
|
50
|
+
type: string;
|
|
51
|
+
payload: unknown;
|
|
52
|
+
sequence: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
interface PollEventsResponse {
|
|
56
|
+
events: RunEventDoc[];
|
|
57
|
+
isComplete?: boolean;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
61
|
+
const res = await driver.get('/.well-known/openwop');
|
|
62
|
+
if (res.status !== 200) return null;
|
|
63
|
+
return res.json as DiscoveryDoc;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function promptsSupported(d: DiscoveryDoc | null): boolean {
|
|
67
|
+
return d?.capabilities?.prompts?.supported === true;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Drain the run's event log via polling. The fixture is tiny so all
|
|
71
|
+
* events fit in one page; conformance hosts may paginate via the
|
|
72
|
+
* optional `nextSequence` but our local sample doesn't. */
|
|
73
|
+
async function readAllEvents(runId: string): Promise<RunEventDoc[]> {
|
|
74
|
+
const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll?lastSequence=0`);
|
|
75
|
+
if (res.status !== 200) return [];
|
|
76
|
+
const body = res.json as PollEventsResponse;
|
|
77
|
+
return body.events ?? [];
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
81
|
+
|
|
82
|
+
describe.skipIf(SKIP_NO_FIXTURE || HTTP_SKIP)('prompt-end-to-end-events: real dispatch emits agent.promptResolved + prompt.composed (RFC 0027/0029)', () => {
|
|
83
|
+
it('emits agent.promptResolved with chain[].applied: true for layer "node" when systemPromptRef is set on node.config', async () => {
|
|
84
|
+
const d = await readDiscovery();
|
|
85
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
86
|
+
|
|
87
|
+
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
88
|
+
expect(
|
|
89
|
+
create.status,
|
|
90
|
+
driver.describe(
|
|
91
|
+
'spec/v1/rest-endpoints.md',
|
|
92
|
+
'POST /v1/runs MUST return 201 on accepted creation',
|
|
93
|
+
),
|
|
94
|
+
).toBe(201);
|
|
95
|
+
const { runId } = create.json as { runId: string };
|
|
96
|
+
|
|
97
|
+
const terminal = await pollUntilTerminal(runId);
|
|
98
|
+
expect(
|
|
99
|
+
terminal.status,
|
|
100
|
+
driver.describe(
|
|
101
|
+
'fixtures.md conformance-prompt-end-to-end §Terminal status',
|
|
102
|
+
'fixture MUST reach terminal `completed`',
|
|
103
|
+
),
|
|
104
|
+
).toBe('completed');
|
|
105
|
+
|
|
106
|
+
const events = await readAllEvents(runId);
|
|
107
|
+
const resolved = events.find((e) => e.type === 'agent.promptResolved');
|
|
108
|
+
expect(
|
|
109
|
+
resolved,
|
|
110
|
+
driver.describe(
|
|
111
|
+
'spec/v1/prompts.md §"Resolution chain (normative)"',
|
|
112
|
+
'host MUST emit `agent.promptResolved` when a node carries a `*PromptRef` and prompts.supported is advertised',
|
|
113
|
+
),
|
|
114
|
+
).toBeDefined();
|
|
115
|
+
|
|
116
|
+
const payload = resolved!.payload as {
|
|
117
|
+
nodeId?: string;
|
|
118
|
+
kind?: string;
|
|
119
|
+
chain?: Array<{ layer?: string; applied?: boolean; source?: string }>;
|
|
120
|
+
resolved?: string | null;
|
|
121
|
+
};
|
|
122
|
+
expect(payload.nodeId, 'agent.promptResolved.nodeId MUST be set').toBe('writer');
|
|
123
|
+
expect(payload.kind, 'agent.promptResolved.kind MUST match the resolved kind').toBe('system');
|
|
124
|
+
const applied = (payload.chain ?? []).find((c) => c.applied === true);
|
|
125
|
+
expect(
|
|
126
|
+
applied?.layer,
|
|
127
|
+
driver.describe(
|
|
128
|
+
'spec/v1/prompts.md §"Resolution chain (normative)" — Layer 1',
|
|
129
|
+
'node-config ref MUST win when no higher-precedence run-configurable layer is configured',
|
|
130
|
+
),
|
|
131
|
+
).toBe('node');
|
|
132
|
+
expect(payload.resolved).toBe('prompt:conformance.prompt.writer-system@1.0.0');
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('emits prompt.composed with sha256:<hex64> hash + non-empty composed body for system-kind template', async () => {
|
|
136
|
+
const d = await readDiscovery();
|
|
137
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
138
|
+
|
|
139
|
+
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
140
|
+
if (create.status !== 201) return;
|
|
141
|
+
const { runId } = create.json as { runId: string };
|
|
142
|
+
await pollUntilTerminal(runId);
|
|
143
|
+
const events = await readAllEvents(runId);
|
|
144
|
+
|
|
145
|
+
const composed = events.find((e) => e.type === 'prompt.composed');
|
|
146
|
+
expect(
|
|
147
|
+
composed,
|
|
148
|
+
driver.describe(
|
|
149
|
+
'spec/v1/prompts.md §"Composition + observability"',
|
|
150
|
+
'host MUST emit `prompt.composed` after `agent.promptResolved` when a ref resolves and observability !== off',
|
|
151
|
+
),
|
|
152
|
+
).toBeDefined();
|
|
153
|
+
|
|
154
|
+
const payload = composed!.payload as {
|
|
155
|
+
nodeId?: string;
|
|
156
|
+
kind?: string;
|
|
157
|
+
hash?: string;
|
|
158
|
+
composed?: string;
|
|
159
|
+
systemPrompt?: string;
|
|
160
|
+
contentTrust?: string;
|
|
161
|
+
};
|
|
162
|
+
expect(
|
|
163
|
+
payload.hash && /^sha256:[0-9a-f]{64}$/.test(payload.hash),
|
|
164
|
+
driver.describe(
|
|
165
|
+
'schemas/run-event-payloads.schema.json §promptComposed.hash',
|
|
166
|
+
'hash MUST match `^sha256:[0-9a-f]{64}$`',
|
|
167
|
+
),
|
|
168
|
+
).toBe(true);
|
|
169
|
+
expect(payload.kind, 'system-only kind composition').toBe('system-only');
|
|
170
|
+
expect(
|
|
171
|
+
typeof payload.composed === 'string' && payload.composed.length > 0,
|
|
172
|
+
driver.describe(
|
|
173
|
+
'spec/v1/prompts.md §"Composition + observability"',
|
|
174
|
+
'composed body MUST be non-empty for system-kind under observability: full',
|
|
175
|
+
),
|
|
176
|
+
).toBe(true);
|
|
177
|
+
// systemPrompt should mirror composed for system-kind templates.
|
|
178
|
+
expect(payload.systemPrompt).toBe(payload.composed);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
it('emits agent.promptResolved before prompt.composed (causal ordering)', async () => {
|
|
182
|
+
const d = await readDiscovery();
|
|
183
|
+
if (!behaviorGate('prompts-supported', promptsSupported(d))) return;
|
|
184
|
+
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
185
|
+
if (create.status !== 201) return;
|
|
186
|
+
const { runId } = create.json as { runId: string };
|
|
187
|
+
await pollUntilTerminal(runId);
|
|
188
|
+
const events = await readAllEvents(runId);
|
|
189
|
+
|
|
190
|
+
const resolvedIdx = events.findIndex((e) => e.type === 'agent.promptResolved');
|
|
191
|
+
const composedIdx = events.findIndex((e) => e.type === 'prompt.composed');
|
|
192
|
+
expect(resolvedIdx, 'agent.promptResolved MUST appear in the event log').toBeGreaterThanOrEqual(0);
|
|
193
|
+
expect(composedIdx, 'prompt.composed MUST appear in the event log').toBeGreaterThanOrEqual(0);
|
|
194
|
+
expect(
|
|
195
|
+
resolvedIdx,
|
|
196
|
+
driver.describe(
|
|
197
|
+
'spec/v1/prompts.md §"Resolution chain (normative)"',
|
|
198
|
+
'agent.promptResolved MUST emit BEFORE the corresponding prompt.composed (resolution precedes composition)',
|
|
199
|
+
),
|
|
200
|
+
).toBeLessThan(composedIdx);
|
|
201
|
+
});
|
|
202
|
+
});
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-list-and-fetch — RFC 0028 §A `listPromptTemplates` +
|
|
3
|
+
* `getPromptTemplate` shape contract.
|
|
4
|
+
*
|
|
5
|
+
* Asserts:
|
|
6
|
+
* 1. `GET /v1/prompts` returns `{ items: PromptTemplate[],
|
|
7
|
+
* nextCursor?: string }`.
|
|
8
|
+
* 2. Each item validates against `prompt-template.schema.json`.
|
|
9
|
+
* 3. Filters (`?kind`, `?source`) narrow the result set without
|
|
10
|
+
* breaking the envelope.
|
|
11
|
+
* 4. `GET /v1/prompts/{templateId}` returns a single template,
|
|
12
|
+
* sets an `ETag` header, and honors `If-None-Match` with 304.
|
|
13
|
+
* 5. Unknown templateId returns 404 with the canonical
|
|
14
|
+
* ErrorEnvelope shape.
|
|
15
|
+
*
|
|
16
|
+
* Capability-gated: skips when the host doesn't advertise
|
|
17
|
+
* `capabilities.prompts.endpointsSupported: true`.
|
|
18
|
+
*
|
|
19
|
+
* Under `OPENWOP_REQUIRE_BEHAVIOR=true`, the capability gate hardens
|
|
20
|
+
* from SKIP to FAIL when the host advertises `endpointsSupported`
|
|
21
|
+
* but doesn't serve the routes.
|
|
22
|
+
*
|
|
23
|
+
* HTTP-driven: skips when no `OPENWOP_BASE_URL` is configured.
|
|
24
|
+
*
|
|
25
|
+
* @see spec/v1/prompts.md §"Discovery & distribution"
|
|
26
|
+
* @see RFCS/0028-prompt-library-endpoints.md §A
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { describe, it, expect } from 'vitest';
|
|
30
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
31
|
+
import addFormats from 'ajv-formats';
|
|
32
|
+
import { readFileSync } from 'node:fs';
|
|
33
|
+
import { join } from 'node:path';
|
|
34
|
+
import { driver } from '../lib/driver.js';
|
|
35
|
+
import { SCHEMAS_DIR } from '../lib/paths.js';
|
|
36
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
37
|
+
|
|
38
|
+
interface DiscoveryDoc {
|
|
39
|
+
capabilities?: {
|
|
40
|
+
prompts?: {
|
|
41
|
+
supported?: unknown;
|
|
42
|
+
endpointsSupported?: unknown;
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface PromptTemplate {
|
|
48
|
+
templateId: string;
|
|
49
|
+
version: string;
|
|
50
|
+
kind: 'system' | 'user' | 'few-shot' | 'schema-hint';
|
|
51
|
+
text: string;
|
|
52
|
+
name?: string;
|
|
53
|
+
meta?: { source?: 'host' | 'pack' | 'user' };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface ListResponse {
|
|
57
|
+
items: PromptTemplate[];
|
|
58
|
+
nextCursor?: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
62
|
+
const res = await driver.get('/.well-known/openwop');
|
|
63
|
+
if (res.status !== 200) return null;
|
|
64
|
+
return res.json as DiscoveryDoc;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function endpointsSupported(d: DiscoveryDoc | null): boolean {
|
|
68
|
+
return d?.capabilities?.prompts?.endpointsSupported === true;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
72
|
+
|
|
73
|
+
describe.skipIf(HTTP_SKIP)('prompt-list-and-fetch: REST surface shape (RFC 0028 §A)', () => {
|
|
74
|
+
// Pre-load schemas so cross-ref validation works against responses.
|
|
75
|
+
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
76
|
+
addFormats(ajv);
|
|
77
|
+
const promptKindSchema = JSON.parse(readFileSync(join(SCHEMAS_DIR, 'prompt-kind.schema.json'), 'utf8'));
|
|
78
|
+
ajv.addSchema(promptKindSchema, 'prompt-kind.schema.json');
|
|
79
|
+
ajv.addSchema(promptKindSchema, './prompt-kind.schema.json');
|
|
80
|
+
const promptTemplateSchema = JSON.parse(readFileSync(join(SCHEMAS_DIR, 'prompt-template.schema.json'), 'utf8'));
|
|
81
|
+
const validate = ajv.compile(promptTemplateSchema);
|
|
82
|
+
|
|
83
|
+
it('GET /v1/prompts returns { items: PromptTemplate[], nextCursor? } when endpointsSupported is true', async () => {
|
|
84
|
+
const d = await readDiscovery();
|
|
85
|
+
if (!behaviorGate('prompts-endpoints', endpointsSupported(d))) return;
|
|
86
|
+
|
|
87
|
+
const res = await driver.get('/v1/prompts');
|
|
88
|
+
expect(res.status, driver.describe('spec/v1/prompts.md §Discovery & distribution', 'GET /v1/prompts MUST return 200 when endpointsSupported: true')).toBe(200);
|
|
89
|
+
const body = res.json as ListResponse;
|
|
90
|
+
expect(
|
|
91
|
+
Array.isArray(body.items),
|
|
92
|
+
driver.describe('spec/v1/prompts.md §Discovery & distribution', 'response MUST contain an `items` array'),
|
|
93
|
+
).toBe(true);
|
|
94
|
+
for (const item of body.items) {
|
|
95
|
+
const ok = validate(item);
|
|
96
|
+
expect(
|
|
97
|
+
ok,
|
|
98
|
+
driver.describe(
|
|
99
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
100
|
+
`every list item MUST validate against prompt-template.schema.json; errors: ${JSON.stringify(validate.errors)}`,
|
|
101
|
+
),
|
|
102
|
+
).toBe(true);
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('GET /v1/prompts?source=host narrows to host-built-in templates', async () => {
|
|
107
|
+
const d = await readDiscovery();
|
|
108
|
+
if (!behaviorGate('prompts-endpoints', endpointsSupported(d))) return;
|
|
109
|
+
const res = await driver.get('/v1/prompts?source=host');
|
|
110
|
+
expect(res.status).toBe(200);
|
|
111
|
+
const body = res.json as ListResponse;
|
|
112
|
+
for (const item of body.items) {
|
|
113
|
+
expect(
|
|
114
|
+
item.meta?.source,
|
|
115
|
+
driver.describe(
|
|
116
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
117
|
+
'source filter MUST narrow to templates whose meta.source matches',
|
|
118
|
+
),
|
|
119
|
+
).toBe('host');
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('GET /v1/prompts?kind=system narrows to system-kind templates', async () => {
|
|
124
|
+
const d = await readDiscovery();
|
|
125
|
+
if (!behaviorGate('prompts-endpoints', endpointsSupported(d))) return;
|
|
126
|
+
const res = await driver.get('/v1/prompts?kind=system');
|
|
127
|
+
expect(res.status).toBe(200);
|
|
128
|
+
const body = res.json as ListResponse;
|
|
129
|
+
for (const item of body.items) {
|
|
130
|
+
expect(item.kind).toBe('system');
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it('GET /v1/prompts/{templateId} returns the template + ETag header for a known fixture', async () => {
|
|
135
|
+
const d = await readDiscovery();
|
|
136
|
+
if (!behaviorGate('prompts-endpoints', endpointsSupported(d))) return;
|
|
137
|
+
|
|
138
|
+
// List first to discover a known templateId we can fetch.
|
|
139
|
+
const list = await driver.get('/v1/prompts?source=host&limit=1');
|
|
140
|
+
if (list.status !== 200) return;
|
|
141
|
+
const body = list.json as ListResponse;
|
|
142
|
+
if (body.items.length === 0) return; // host advertises endpoints but ships no fixtures — tolerable
|
|
143
|
+
const known = body.items[0]!;
|
|
144
|
+
|
|
145
|
+
const fetched = await driver.get(`/v1/prompts/${encodeURIComponent(known.templateId)}`);
|
|
146
|
+
expect(fetched.status).toBe(200);
|
|
147
|
+
const tpl = fetched.json as PromptTemplate;
|
|
148
|
+
expect(tpl.templateId).toBe(known.templateId);
|
|
149
|
+
|
|
150
|
+
// Headers.get() is case-insensitive per the Fetch spec, so one call
|
|
151
|
+
// covers both "etag" and "ETag" wire spellings.
|
|
152
|
+
const etag = fetched.headers?.get('etag');
|
|
153
|
+
expect(
|
|
154
|
+
typeof etag === 'string' && etag.length > 0,
|
|
155
|
+
driver.describe(
|
|
156
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
157
|
+
'GET /v1/prompts/{templateId} SHOULD set an ETag header (RFC 0028 §A cache semantics)',
|
|
158
|
+
),
|
|
159
|
+
).toBe(true);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it('GET /v1/prompts/{templateId} with If-None-Match returns 304 when ETag matches', async () => {
|
|
163
|
+
const d = await readDiscovery();
|
|
164
|
+
if (!behaviorGate('prompts-endpoints', endpointsSupported(d))) return;
|
|
165
|
+
const list = await driver.get('/v1/prompts?source=host&limit=1');
|
|
166
|
+
if (list.status !== 200) return;
|
|
167
|
+
const body = list.json as ListResponse;
|
|
168
|
+
if (body.items.length === 0) return;
|
|
169
|
+
const known = body.items[0]!;
|
|
170
|
+
|
|
171
|
+
const first = await driver.get(`/v1/prompts/${encodeURIComponent(known.templateId)}`);
|
|
172
|
+
if (first.status !== 200) return;
|
|
173
|
+
const etag = first.headers?.get('etag') ?? undefined;
|
|
174
|
+
if (!etag) return; // ETag is SHOULD, not MUST — soft-skip when absent
|
|
175
|
+
|
|
176
|
+
const second = await driver.get(`/v1/prompts/${encodeURIComponent(known.templateId)}`, {
|
|
177
|
+
headers: { 'If-None-Match': etag },
|
|
178
|
+
});
|
|
179
|
+
expect(
|
|
180
|
+
second.status,
|
|
181
|
+
driver.describe(
|
|
182
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
183
|
+
'conditional revalidation MUST return 304 when ETag matches',
|
|
184
|
+
),
|
|
185
|
+
).toBe(304);
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
it('GET /v1/prompts/unknown-template returns 404 with ErrorEnvelope', async () => {
|
|
189
|
+
const d = await readDiscovery();
|
|
190
|
+
if (!behaviorGate('prompts-endpoints', endpointsSupported(d))) return;
|
|
191
|
+
const res = await driver.get('/v1/prompts/conformance-unknown-template-deadbeef');
|
|
192
|
+
expect(res.status).toBe(404);
|
|
193
|
+
// Canonical ErrorEnvelope per `schemas/error-envelope.schema.json`:
|
|
194
|
+
// FLAT `{ error: <code-string>, message: <human> }`. NOT the nested
|
|
195
|
+
// `{ error: { code, message } }` shape — the schema's
|
|
196
|
+
// `additionalProperties: false` rules that out.
|
|
197
|
+
const body = res.json as { error?: unknown; message?: unknown };
|
|
198
|
+
expect(
|
|
199
|
+
typeof body.error,
|
|
200
|
+
driver.describe(
|
|
201
|
+
'schemas/error-envelope.schema.json',
|
|
202
|
+
'404 response MUST carry canonical ErrorEnvelope: `error` is a machine-readable code STRING (flat shape per the schema, not nested)',
|
|
203
|
+
),
|
|
204
|
+
).toBe('string');
|
|
205
|
+
expect(typeof body.message).toBe('string');
|
|
206
|
+
});
|
|
207
|
+
});
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-mutable-lifecycle — RFC 0028 §A `create` / `update` /
|
|
3
|
+
* `delete` round-trip for user-source templates.
|
|
4
|
+
*
|
|
5
|
+
* Asserts the full mutating lifecycle:
|
|
6
|
+
* 1. POST /v1/prompts creates a user-source template; returns 201
|
|
7
|
+
* with a Location header.
|
|
8
|
+
* 2. GET /v1/prompts/{templateId} returns the newly-created
|
|
9
|
+
* template with `meta.source: "user"`.
|
|
10
|
+
* 3. POST /v1/prompts with the same (templateId, version) returns
|
|
11
|
+
* 409 (duplicate).
|
|
12
|
+
* 4. PUT /v1/prompts/{templateId} with a strictly-greater SemVer
|
|
13
|
+
* replaces the template; the stored version reflects the bump.
|
|
14
|
+
* 5. PUT /v1/prompts/{templateId} with a non-monotonic SemVer
|
|
15
|
+
* returns 409.
|
|
16
|
+
* 6. DELETE /v1/prompts/{templateId} returns 204; subsequent GET
|
|
17
|
+
* returns 404.
|
|
18
|
+
* 7. DELETE on a host-built-in (meta.source: "host") template
|
|
19
|
+
* returns 403.
|
|
20
|
+
*
|
|
21
|
+
* Capability-gated: skips when the host doesn't advertise BOTH
|
|
22
|
+
* `capabilities.prompts.endpointsSupported: true` AND
|
|
23
|
+
* `capabilities.prompts.mutableLibrary: true`.
|
|
24
|
+
*
|
|
25
|
+
* HTTP-driven: skips when no `OPENWOP_BASE_URL` is configured.
|
|
26
|
+
*
|
|
27
|
+
* Under `OPENWOP_REQUIRE_BEHAVIOR=true`, the capability gate hardens
|
|
28
|
+
* from SKIP to FAIL when the host advertises mutableLibrary but
|
|
29
|
+
* fails to round-trip the lifecycle.
|
|
30
|
+
*
|
|
31
|
+
* @see spec/v1/prompts.md §"Discovery & distribution"
|
|
32
|
+
* @see RFCS/0028-prompt-library-endpoints.md §A
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { describe, it, expect } from 'vitest';
|
|
36
|
+
import { driver } from '../lib/driver.js';
|
|
37
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
38
|
+
|
|
39
|
+
interface DiscoveryDoc {
|
|
40
|
+
capabilities?: {
|
|
41
|
+
prompts?: {
|
|
42
|
+
endpointsSupported?: unknown;
|
|
43
|
+
mutableLibrary?: unknown;
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface PromptTemplate {
|
|
49
|
+
templateId: string;
|
|
50
|
+
version: string;
|
|
51
|
+
kind: 'system' | 'user' | 'few-shot' | 'schema-hint';
|
|
52
|
+
text: string;
|
|
53
|
+
meta?: { source?: 'host' | 'pack' | 'user' };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
57
|
+
const res = await driver.get('/.well-known/openwop');
|
|
58
|
+
if (res.status !== 200) return null;
|
|
59
|
+
return res.json as DiscoveryDoc;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function mutableSupport(d: DiscoveryDoc | null): boolean {
|
|
63
|
+
const p = d?.capabilities?.prompts;
|
|
64
|
+
return p?.endpointsSupported === true && p?.mutableLibrary === true;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
68
|
+
|
|
69
|
+
// Stable per-suite-run templateId so retries don't accumulate state.
|
|
70
|
+
const TEMPLATE_ID = `conformance.user.lifecycle-${Math.random().toString(36).slice(2, 10)}`;
|
|
71
|
+
|
|
72
|
+
describe.skipIf(HTTP_SKIP)('prompt-mutable-lifecycle: user-source create/update/delete round-trip (RFC 0028 §A)', () => {
|
|
73
|
+
it('POST /v1/prompts creates a user-source template (201 + Location)', async () => {
|
|
74
|
+
const d = await readDiscovery();
|
|
75
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
76
|
+
const body: PromptTemplate = {
|
|
77
|
+
templateId: TEMPLATE_ID,
|
|
78
|
+
version: '1.0.0',
|
|
79
|
+
kind: 'system',
|
|
80
|
+
text: 'You are a conformance probe. {{tone}}',
|
|
81
|
+
};
|
|
82
|
+
const res = await driver.post('/v1/prompts', body);
|
|
83
|
+
expect(
|
|
84
|
+
res.status,
|
|
85
|
+
driver.describe(
|
|
86
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
87
|
+
'POST /v1/prompts MUST return 201 on successful user-source create',
|
|
88
|
+
),
|
|
89
|
+
).toBe(201);
|
|
90
|
+
const location = res.headers?.get?.('location');
|
|
91
|
+
expect(
|
|
92
|
+
typeof location === 'string' && location.includes(TEMPLATE_ID),
|
|
93
|
+
driver.describe(
|
|
94
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
95
|
+
'201 response MUST set a Location header referencing the new templateId',
|
|
96
|
+
),
|
|
97
|
+
).toBe(true);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('GET /v1/prompts/{templateId} returns the new template with meta.source: "user"', async () => {
|
|
101
|
+
const d = await readDiscovery();
|
|
102
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
103
|
+
const res = await driver.get(`/v1/prompts/${encodeURIComponent(TEMPLATE_ID)}`);
|
|
104
|
+
expect(res.status).toBe(200);
|
|
105
|
+
const tpl = res.json as PromptTemplate;
|
|
106
|
+
expect(tpl.templateId).toBe(TEMPLATE_ID);
|
|
107
|
+
expect(
|
|
108
|
+
tpl.meta?.source,
|
|
109
|
+
driver.describe(
|
|
110
|
+
'spec/v1/prompts.md §PromptTemplate',
|
|
111
|
+
'host MUST stamp meta.source: "user" on POST-created templates',
|
|
112
|
+
),
|
|
113
|
+
).toBe('user');
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('POST /v1/prompts with same (templateId, version) returns 409', async () => {
|
|
117
|
+
const d = await readDiscovery();
|
|
118
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
119
|
+
const body: PromptTemplate = {
|
|
120
|
+
templateId: TEMPLATE_ID,
|
|
121
|
+
version: '1.0.0',
|
|
122
|
+
kind: 'system',
|
|
123
|
+
text: 'duplicate',
|
|
124
|
+
};
|
|
125
|
+
const res = await driver.post('/v1/prompts', body);
|
|
126
|
+
expect(
|
|
127
|
+
res.status,
|
|
128
|
+
driver.describe(
|
|
129
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
130
|
+
'POST /v1/prompts MUST return 409 on (templateId, version) duplicate',
|
|
131
|
+
),
|
|
132
|
+
).toBe(409);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('PUT /v1/prompts/{templateId} with strictly-greater SemVer replaces the template', async () => {
|
|
136
|
+
const d = await readDiscovery();
|
|
137
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
138
|
+
const body: PromptTemplate = {
|
|
139
|
+
templateId: TEMPLATE_ID,
|
|
140
|
+
version: '1.1.0',
|
|
141
|
+
kind: 'system',
|
|
142
|
+
text: 'You are a conformance probe v1.1. {{tone}}',
|
|
143
|
+
};
|
|
144
|
+
const res = await driver.put(`/v1/prompts/${encodeURIComponent(TEMPLATE_ID)}`, body);
|
|
145
|
+
expect(
|
|
146
|
+
res.status,
|
|
147
|
+
driver.describe(
|
|
148
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
149
|
+
'PUT /v1/prompts/{templateId} MUST return 200 on monotonic-SemVer update',
|
|
150
|
+
),
|
|
151
|
+
).toBe(200);
|
|
152
|
+
// Latest fetch reflects the bumped version.
|
|
153
|
+
const fetched = await driver.get(`/v1/prompts/${encodeURIComponent(TEMPLATE_ID)}`);
|
|
154
|
+
expect((fetched.json as PromptTemplate).version).toBe('1.1.0');
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it('PUT /v1/prompts/{templateId} with non-monotonic SemVer returns 409', async () => {
|
|
158
|
+
const d = await readDiscovery();
|
|
159
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
160
|
+
const body: PromptTemplate = {
|
|
161
|
+
templateId: TEMPLATE_ID,
|
|
162
|
+
version: '0.9.0',
|
|
163
|
+
kind: 'system',
|
|
164
|
+
text: 'cannot replay',
|
|
165
|
+
};
|
|
166
|
+
const res = await driver.put(`/v1/prompts/${encodeURIComponent(TEMPLATE_ID)}`, body);
|
|
167
|
+
expect(
|
|
168
|
+
res.status,
|
|
169
|
+
driver.describe(
|
|
170
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
171
|
+
'PUT MUST return 409 when submitted version does not exceed stored',
|
|
172
|
+
),
|
|
173
|
+
).toBe(409);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('DELETE /v1/prompts/{templateId} returns 204 and subsequent GET returns 404', async () => {
|
|
177
|
+
const d = await readDiscovery();
|
|
178
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
179
|
+
const del = await driver.delete(`/v1/prompts/${encodeURIComponent(TEMPLATE_ID)}`);
|
|
180
|
+
expect(
|
|
181
|
+
del.status,
|
|
182
|
+
driver.describe(
|
|
183
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
184
|
+
'DELETE /v1/prompts/{templateId} MUST return 204 on successful delete',
|
|
185
|
+
),
|
|
186
|
+
).toBe(204);
|
|
187
|
+
const after = await driver.get(`/v1/prompts/${encodeURIComponent(TEMPLATE_ID)}`);
|
|
188
|
+
expect(
|
|
189
|
+
after.status,
|
|
190
|
+
driver.describe(
|
|
191
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
192
|
+
'GET after DELETE MUST return 404',
|
|
193
|
+
),
|
|
194
|
+
).toBe(404);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it('DELETE on a host-built-in template returns 403', async () => {
|
|
198
|
+
const d = await readDiscovery();
|
|
199
|
+
if (!behaviorGate('prompts-mutable', mutableSupport(d))) return;
|
|
200
|
+
// Find a host-built-in to probe; the conformance-fixture set
|
|
201
|
+
// is the standard source for this test.
|
|
202
|
+
const list = await driver.get('/v1/prompts?source=host&limit=1');
|
|
203
|
+
if (list.status !== 200) return;
|
|
204
|
+
const body = list.json as { items: PromptTemplate[] };
|
|
205
|
+
if (body.items.length === 0) return;
|
|
206
|
+
const hostTemplate = body.items[0]!;
|
|
207
|
+
const res = await driver.delete(`/v1/prompts/${encodeURIComponent(hostTemplate.templateId)}`);
|
|
208
|
+
expect(
|
|
209
|
+
res.status,
|
|
210
|
+
driver.describe(
|
|
211
|
+
'spec/v1/prompts.md §Discovery & distribution',
|
|
212
|
+
'DELETE on a host-built-in template MUST return 403 (read-only)',
|
|
213
|
+
),
|
|
214
|
+
).toBe(403);
|
|
215
|
+
});
|
|
216
|
+
});
|