@openwop/openwop-conformance 1.106.1 → 1.124.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/coverage.md +6 -0
- package/dist/cli.js +89 -45
- package/dist/lib/certification-bundle-verify.js +274 -0
- package/dist/lib/profiles.js +102 -19
- package/dist/lib/requirement-ledger.js +182 -0
- package/dist/lib/requirement-registry.js +110 -0
- package/dist/lib/scenario-disposition.js +249 -0
- package/dist/lib/soft-skip.js +82 -0
- package/fixtures/conformance-a2a-task-roundtrip.json +1 -1
- package/fixtures.md +1 -1
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/schemas/README.md +1 -0
- package/schemas/capabilities.schema.json +2 -2
- package/schemas/compensation-policy.schema.json +64 -0
- package/schemas/run-snapshot.schema.json +5 -0
- package/schemas/workflow-chain-pack-manifest.schema.json +125 -0
- package/schemas/workflow-definition.schema.json +4 -0
- package/src/cli.ts +99 -45
- package/src/lib/a2a-fake-peer.ts +290 -110
- package/src/lib/certification-bundle-verify.ts +356 -0
- package/src/lib/error-envelope.ts +77 -0
- package/src/lib/llm-cache-key-recipe.ts +6 -5
- package/src/lib/mcp-fake-server.ts +235 -72
- package/src/lib/profiles.ts +132 -19
- package/src/lib/requirement-ledger.ts +98 -1
- package/src/lib/requirement-registry.ts +47 -4
- package/src/lib/scenario-disposition.ts +291 -0
- package/src/lib/soft-skip.ts +89 -0
- package/src/lib/workflow-chain-expansion.ts +189 -0
- package/src/scenarios/a2a-1-0-agent-card.test.ts +179 -0
- package/src/scenarios/a2a-1-0-task-roundtrip.test.ts +109 -0
- package/src/scenarios/a2a-card-runtime-consistency.test.ts +167 -0
- package/src/scenarios/a2a-peer-authority.test.ts +99 -0
- package/src/scenarios/a2a-task-roundtrip.test.ts +18 -6
- package/src/scenarios/agent-loop-iteration-monotonic.test.ts +3 -2
- package/src/scenarios/agent-loop-stateful-resume.test.ts +3 -2
- package/src/scenarios/agent-loop-version5-shape.test.ts +2 -1
- package/src/scenarios/agent-loop-workspace-snapshot.test.ts +4 -3
- package/src/scenarios/approval-gate-flow.test.ts +5 -4
- package/src/scenarios/artifact-type-store-without-render.test.ts +2 -1
- package/src/scenarios/auth-scim-profile.test.ts +5 -4
- package/src/scenarios/authorization-fail-closed.test.ts +4 -3
- package/src/scenarios/certification-bundle-non-vacuous.test.ts +249 -0
- package/src/scenarios/certification-bundle-redaction.test.ts +182 -0
- package/src/scenarios/certification-floor-enforcement.test.ts +30 -6
- package/src/scenarios/chain-compensation-expansion.test.ts +243 -0
- package/src/scenarios/commitment-fired.test.ts +3 -2
- package/src/scenarios/compensation-behavior.test.ts +63 -0
- package/src/scenarios/compensation-profile.test.ts +144 -1
- package/src/scenarios/conformance-advertised-seam-required.test.ts +118 -0
- package/src/scenarios/conformance-execution-witness.test.ts +104 -0
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +3 -2
- package/src/scenarios/core-manifest-and-extension-registry.test.ts +26 -0
- package/src/scenarios/credential-payload-redaction.test.ts +4 -3
- package/src/scenarios/credentials-capability-shape.test.ts +4 -3
- package/src/scenarios/cross-engine-append-behavior.test.ts +5 -4
- package/src/scenarios/cross-engine-append-ordering.test.ts +3 -2
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +5 -4
- package/src/scenarios/cross-host-causation-shape.test.ts +3 -2
- package/src/scenarios/cross-workspace-isolation.test.ts +4 -3
- package/src/scenarios/data-residency-admission.test.ts +6 -6
- package/src/scenarios/deadletter-retry-exhaustion.test.ts +5 -4
- package/src/scenarios/distillation-index-roundtrip.test.ts +3 -2
- package/src/scenarios/distillation-secret-carryforward.test.ts +3 -2
- package/src/scenarios/distillation-shape.test.ts +2 -1
- package/src/scenarios/distillation-stable-archive.test.ts +4 -3
- package/src/scenarios/distillation-token-budget.test.ts +4 -3
- package/src/scenarios/envelope-recovery-applied.test.ts +3 -2
- package/src/scenarios/envelope-refusal-shape.test.ts +4 -3
- package/src/scenarios/error-envelope-canonical-shape.test.ts +64 -0
- package/src/scenarios/feedback-correction-redaction.test.ts +4 -3
- package/src/scenarios/feedback-cross-tenant-isolation.test.ts +4 -3
- package/src/scenarios/feedback-fork-not-copied.test.ts +6 -5
- package/src/scenarios/feedback-on-terminal-run.test.ts +4 -3
- package/src/scenarios/feedback-record-and-list.test.ts +4 -3
- package/src/scenarios/feedback-unsupported-501.test.ts +3 -2
- package/src/scenarios/fixtures-valid.test.ts +16 -18
- package/src/scenarios/fs-path-traversal.test.ts +3 -2
- package/src/scenarios/heartbeat-capability-shape.test.ts +2 -1
- package/src/scenarios/heartbeat-fires-once-per-tick.test.ts +4 -3
- package/src/scenarios/heartbeat-idempotent-no-spam.test.ts +6 -5
- package/src/scenarios/heartbeat-runtime-bound.test.ts +4 -3
- package/src/scenarios/mcp-2026-07-28-discover.test.ts +207 -0
- package/src/scenarios/mcp-cache-tenant-scope.test.ts +66 -0
- package/src/scenarios/mcp-current-auth-boundary.test.ts +55 -0
- package/src/scenarios/mcp-extension-opacity.test.ts +79 -0
- package/src/scenarios/mcp-mrtr-roundtrip.test.ts +140 -0
- package/src/scenarios/mcp-stateless-request.test.ts +77 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +4 -3
- package/src/scenarios/mcp-toolcall-redaction.test.ts +2 -1
- package/src/scenarios/memory-attribution-emits-on-write.test.ts +6 -5
- package/src/scenarios/memory-attribution-no-content.test.ts +4 -3
- package/src/scenarios/memory-attribution-replay-stable.test.ts +6 -5
- package/src/scenarios/memory-attribution-tenant-scoped.test.ts +4 -3
- package/src/scenarios/memory-compaction-event-emitted.test.ts +3 -2
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +6 -5
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +4 -3
- package/src/scenarios/memory-consolidation-idempotent.test.ts +4 -3
- package/src/scenarios/memory-injection-budget.test.ts +4 -3
- package/src/scenarios/metric-emission.test.ts +4 -3
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +5 -4
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +4 -3
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +5 -4
- package/src/scenarios/multi-party-conversation-behavioral.test.ts +3 -2
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +7 -6
- package/src/scenarios/multi-region-idempotency.test.ts +4 -3
- package/src/scenarios/normative-example-extraction.test.ts +242 -0
- package/src/scenarios/oauth-authorization-code-roundtrip.test.ts +4 -3
- package/src/scenarios/oauth-capability-shape.test.ts +4 -3
- package/src/scenarios/oauth-connector-redaction.test.ts +4 -3
- package/src/scenarios/openapi-asyncapi-sdk-parity.test.ts +309 -0
- package/src/scenarios/org-position-no-authority-escalation.test.ts +2 -1
- package/src/scenarios/otel-emission-grpc.test.ts +4 -3
- package/src/scenarios/otel-emission.test.ts +4 -3
- package/src/scenarios/otel-trace-propagation-subworkflow.test.ts +5 -4
- package/src/scenarios/otel-trace-propagation.test.ts +4 -3
- package/src/scenarios/pack-registry-isolation.test.ts +4 -3
- package/src/scenarios/pack-registry-publish.test.ts +65 -3
- package/src/scenarios/pack-registry.test.ts +22 -0
- package/src/scenarios/profile-discovery-core-alias.test.ts +137 -0
- package/src/scenarios/profileDerivation.test.ts +2 -1
- package/src/scenarios/provider-usage.test.ts +2 -2
- package/src/scenarios/rate-limit-envelope.test.ts +2 -1
- package/src/scenarios/registry-public.test.ts +6 -5
- package/src/scenarios/replay-divergence-at-refusal.test.ts +9 -8
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +19 -13
- package/src/scenarios/replay-llm-cache-key.test.ts +120 -47
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +5 -4
- package/src/scenarios/replay-side-effect-suppression.test.ts +20 -1
- package/src/scenarios/requirement-ledger.test.ts +29 -6
- package/src/scenarios/runner-ledger.test.ts +279 -0
- package/src/scenarios/runtime-requires-install-gate.test.ts +6 -5
- package/src/scenarios/sandbox-memory-cap.test.ts +3 -2
- package/src/scenarios/sandbox-mvp-behavior.test.ts +11 -10
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +2 -1
- package/src/scenarios/sandbox-timeout-cap.test.ts +3 -2
- package/src/scenarios/stream-text-fixture.test.ts +5 -4
- package/src/scenarios/subrun-approval-fail-closed.test.ts +3 -2
- package/src/scenarios/subrun-approval-gate.test.ts +4 -3
- package/src/scenarios/subrun-attestation-shape.test.ts +2 -1
- package/src/scenarios/subrun-checksum-stable.test.ts +4 -3
- package/src/scenarios/table-schema-enforcement.test.ts +2 -2
- package/src/scenarios/tool-hooks-authorization-fail-closed.test.ts +3 -2
- package/src/scenarios/tool-hooks-content-free.test.ts +3 -2
- package/src/scenarios/tool-hooks-rate-limit.test.ts +3 -2
- package/src/scenarios/tool-hooks-secret-redaction.test.ts +3 -2
- package/src/scenarios/tool-hooks-shape.test.ts +2 -1
- package/src/scenarios/trigger-bridge-delivery.test.ts +3 -2
- package/src/scenarios/version-fold.test.ts +5 -4
- package/src/scenarios/voice-streamref-tenant-bound.test.ts +2 -1
- package/src/scenarios/voice-transcription-streaming.test.ts +2 -1
- package/src/scenarios/voice-transcription-unadvertised.test.ts +2 -1
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +5 -4
- package/src/scenarios/wasm-pack-invoke-completed.test.ts +3 -2
- package/src/scenarios/wasm-pack-invoke-suspended.test.ts +3 -2
- package/src/scenarios/wasm-pack-load.test.ts +4 -3
- package/src/scenarios/wasm-pack-memory-cap.test.ts +4 -3
- package/src/scenarios/wasm-pack-replay-determinism.test.ts +3 -2
- package/src/scenarios/webhook-signed-delivery.test.ts +4 -3
- package/src/scenarios/workflow-primary-output-annotation.test.ts +13 -23
- package/src/scenarios/workload-identity-behavior.test.ts +7 -4
- package/src/scenarios/workspace-cross-tenant-isolation-blackbox.test.ts +3 -2
- package/src/scenarios/workspace-cross-tenant-isolation.test.ts +3 -2
- package/src/setup.ts +95 -1
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0155 §A — `openwop-discovery-core` is the canonical name of the discovery
|
|
3
|
+
* predicate; `openwop-core` is its deprecated alias for all of v1 and MUST derive
|
|
4
|
+
* exactly when the canonical name derives.
|
|
5
|
+
*
|
|
6
|
+
* Why a scenario for a rename: the predicate is a discovery-payload check and
|
|
7
|
+
* says nothing about whether a run can be started, suspended, or replayed. A
|
|
8
|
+
* published certification bundle claimed `openwop-core` while failing six
|
|
9
|
+
* `interrupt-*` scenarios (RFC 0155 §A's motivating defect) — possible only
|
|
10
|
+
* because the name did not say what it measured. The rename makes the claim
|
|
11
|
+
* vocabulary unambiguous; this file makes the alias rule checkable:
|
|
12
|
+
*
|
|
13
|
+
* - both names are in the closed catalog;
|
|
14
|
+
* - for ANY payload, the alias derives iff the canonical name derives, and
|
|
15
|
+
* `deriveProfiles` emits both or neither (never the alias alone);
|
|
16
|
+
* - `profiles.md` states the rename, the alias rule, and the claim
|
|
17
|
+
* vocabulary (an unqualified claim means `openwop-core-standard`);
|
|
18
|
+
* - (gated) a live host's derived set contains both or neither.
|
|
19
|
+
*
|
|
20
|
+
* Server-free except the last leg. Reads `profiles.md` from `V1_DIR` when the
|
|
21
|
+
* repo layout is present; skips that leg from the published tarball.
|
|
22
|
+
*
|
|
23
|
+
* @see spec/v1/profiles.md §"openwop-discovery-core" + §"Claim vocabulary"
|
|
24
|
+
* @see RFCS/0155-core-profile-and-extension-discipline.md §A
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { describe, it, expect } from 'vitest';
|
|
28
|
+
import { readFileSync } from 'node:fs';
|
|
29
|
+
import { join } from 'node:path';
|
|
30
|
+
import { driver } from '../lib/driver.js';
|
|
31
|
+
import { PROFILE_NAMES, deriveProfiles, hasProfile, type DiscoveryPayload } from '../lib/profiles.js';
|
|
32
|
+
import { V1_DIR } from '../lib/paths.js';
|
|
33
|
+
|
|
34
|
+
const CANONICAL = 'openwop-discovery-core';
|
|
35
|
+
const ALIAS = 'openwop-core';
|
|
36
|
+
|
|
37
|
+
const CORE: DiscoveryPayload = {
|
|
38
|
+
protocolVersion: '1.0', // MAJOR.MINOR — RFC 0149 §C grammar, not semver
|
|
39
|
+
supportedEnvelopes: [],
|
|
40
|
+
schemaVersions: { workflow: 1 },
|
|
41
|
+
limits: { clarificationRounds: 0, schemaRounds: 0, envelopesPerTurn: 0 },
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** Payloads on both sides of the predicate, so the equivalence leg is not vacuous. */
|
|
45
|
+
const PAYLOADS: ReadonlyArray<readonly [string, DiscoveryPayload]> = [
|
|
46
|
+
['minimal core', CORE],
|
|
47
|
+
['rich', { ...CORE, supportedEnvelopes: ['clarification.request'], secrets: { supported: true } }],
|
|
48
|
+
['empty object', {}],
|
|
49
|
+
['wrong major', { ...CORE, protocolVersion: '2.0' }],
|
|
50
|
+
['semver not grammar', { ...CORE, protocolVersion: '1.0.0' }],
|
|
51
|
+
['missing limits', { protocolVersion: '1.0', supportedEnvelopes: [], schemaVersions: {} }],
|
|
52
|
+
['negative limit', { ...CORE, limits: { clarificationRounds: -1, schemaRounds: 0, envelopesPerTurn: 0 } }],
|
|
53
|
+
['envelopes not array', { ...CORE, supportedEnvelopes: 'clarification.request' as unknown as string[] }],
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
describe('RFC 0155 §A — openwop-discovery-core is canonical, openwop-core is its deprecated alias', () => {
|
|
57
|
+
it('both names are in the closed catalog, canonical first', () => {
|
|
58
|
+
expect(PROFILE_NAMES).toContain(CANONICAL);
|
|
59
|
+
expect(PROFILE_NAMES).toContain(ALIAS);
|
|
60
|
+
expect(
|
|
61
|
+
PROFILE_NAMES.indexOf(CANONICAL),
|
|
62
|
+
'the canonical name precedes its alias so derived sets read canonical-first',
|
|
63
|
+
).toBeLessThan(PROFILE_NAMES.indexOf(ALIAS));
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('the alias derives exactly when the canonical name derives — for payloads on both sides', () => {
|
|
67
|
+
let trueCount = 0;
|
|
68
|
+
let falseCount = 0;
|
|
69
|
+
for (const [label, c] of PAYLOADS) {
|
|
70
|
+
const canonical = hasProfile(c, CANONICAL);
|
|
71
|
+
const alias = hasProfile(c, ALIAS);
|
|
72
|
+
expect(alias, `${label}: hasProfile(alias) MUST equal hasProfile(canonical)`).toBe(canonical);
|
|
73
|
+
const derived = deriveProfiles(c);
|
|
74
|
+
expect(
|
|
75
|
+
derived.includes(ALIAS),
|
|
76
|
+
`${label}: deriveProfiles MUST emit the alias iff it emits the canonical name (never the alias alone)`,
|
|
77
|
+
).toBe(derived.includes(CANONICAL));
|
|
78
|
+
if (canonical) trueCount++;
|
|
79
|
+
else falseCount++;
|
|
80
|
+
}
|
|
81
|
+
// Non-vacuity: an equivalence over payloads that all derive (or none do)
|
|
82
|
+
// proves nothing about the other side.
|
|
83
|
+
expect(trueCount, 'at least one payload MUST derive the predicate').toBeGreaterThan(0);
|
|
84
|
+
expect(falseCount, 'at least one payload MUST fail the predicate').toBeGreaterThan(0);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('a derived set that carries the alias carries the canonical name immediately before it', () => {
|
|
88
|
+
const derived = deriveProfiles(CORE);
|
|
89
|
+
const i = derived.indexOf(CANONICAL);
|
|
90
|
+
expect(i).toBeGreaterThanOrEqual(0);
|
|
91
|
+
expect(derived[i + 1], 'alias rides directly beside its canonical name').toBe(ALIAS);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
describe.skipIf(V1_DIR === null)('RFC 0155 §A — profiles.md states the rename and the claim vocabulary', () => {
|
|
96
|
+
const doc = () => readFileSync(join(V1_DIR as string, 'profiles.md'), 'utf8');
|
|
97
|
+
|
|
98
|
+
it('names the canonical profile and marks openwop-core as a deprecated alias', () => {
|
|
99
|
+
const md = doc();
|
|
100
|
+
expect(md).toMatch(/###\s+`openwop-discovery-core`\s+\(canonical\)/);
|
|
101
|
+
expect(md).toMatch(/alias `openwop-core` \(deprecated\)/);
|
|
102
|
+
expect(md, 'the alias MUST be defined as deriving exactly when the canonical name derives').toMatch(
|
|
103
|
+
/derives \*\*exactly when\*\* `openwop-discovery-core` derives/,
|
|
104
|
+
);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('states that an unqualified conformance claim means openwop-core-standard', () => {
|
|
108
|
+
const md = doc();
|
|
109
|
+
expect(md).toMatch(/unqualified\*\* "OpenWOP conformant".*\*\*MUST\*\* mean \*\*`openwop-core-standard`\*\*/s);
|
|
110
|
+
expect(md, 'discovery-only claims MUST say openwop-discovery-core').toMatch(
|
|
111
|
+
/discovery-only\*\* claim \*\*MUST\*\* say \*\*`openwop-discovery-core`\*\*/,
|
|
112
|
+
);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('the reference derivation lists the alias beside the canonical name, never alone', () => {
|
|
116
|
+
const md = doc();
|
|
117
|
+
const canonicalLine = md.indexOf("'openwop-discovery-core' if openwop-discovery-core(c)");
|
|
118
|
+
const aliasLine = md.indexOf("'openwop-core' if openwop-discovery-core(c)");
|
|
119
|
+
expect(canonicalLine).toBeGreaterThan(0);
|
|
120
|
+
expect(aliasLine).toBeGreaterThan(canonicalLine);
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
describe.skipIf(!process.env.OPENWOP_BASE_URL)('RFC 0155 §A — a live host derives both names or neither', () => {
|
|
125
|
+
it('the discovery payload derives the alias iff it derives the canonical name', async () => {
|
|
126
|
+
const res = await driver.get('/.well-known/openwop');
|
|
127
|
+
if (res.status !== 200) return; // discovery is witnessed elsewhere; this leg is about the alias rule
|
|
128
|
+
const derived = deriveProfiles(res.json as DiscoveryPayload);
|
|
129
|
+
expect(
|
|
130
|
+
derived.includes(ALIAS),
|
|
131
|
+
driver.describe(
|
|
132
|
+
'profiles.md §"openwop-discovery-core"',
|
|
133
|
+
'the deprecated alias MUST derive exactly when the canonical name derives — a host is never `openwop-core` without being `openwop-discovery-core`',
|
|
134
|
+
),
|
|
135
|
+
).toBe(derived.includes(CANONICAL));
|
|
136
|
+
});
|
|
137
|
+
});
|
|
@@ -272,7 +272,8 @@ describe('profiles: deriveProfiles produces the full set', () => {
|
|
|
272
272
|
};
|
|
273
273
|
const result = deriveProfiles(rich);
|
|
274
274
|
const expected: ProfileName[] = [
|
|
275
|
-
'openwop-core',
|
|
275
|
+
'openwop-discovery-core', // RFC 0155 §A canonical name
|
|
276
|
+
'openwop-core', // deprecated alias — always beside the canonical name, never alone
|
|
276
277
|
'openwop-interrupts',
|
|
277
278
|
'openwop-stream-sse',
|
|
278
279
|
'openwop-stream-poll',
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import { describe, it, expect } from 'vitest';
|
|
23
|
+
import { readErrorCode } from '../lib/error-envelope.js';
|
|
23
24
|
import Ajv2020 from 'ajv/dist/2020.js';
|
|
24
25
|
import { readFileSync } from 'node:fs';
|
|
25
26
|
import { join } from 'node:path';
|
|
@@ -179,8 +180,7 @@ describe('provider-usage: event presence via emit-seam + event-log query (RFC 00
|
|
|
179
180
|
res.status,
|
|
180
181
|
driver.describe('SECURITY/invariants.yaml provider-usage-no-credential-leak', 'payload with credentialRef-shaped content MUST be refused'),
|
|
181
182
|
).toBe(400);
|
|
182
|
-
|
|
183
|
-
expect(body.error?.code).toBe('provider_usage_credential_leak');
|
|
183
|
+
expect(readErrorCode(res.json)).toBe('provider_usage_credential_leak');
|
|
184
184
|
await resetTestSeam();
|
|
185
185
|
});
|
|
186
186
|
});
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import { describe, it, expect } from 'vitest';
|
|
24
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
24
25
|
import { driver } from '../lib/driver.js';
|
|
25
26
|
|
|
26
27
|
const ALLOWED_SCOPES = new Set(['tenant', 'route', 'global', 'key']);
|
|
@@ -43,7 +44,7 @@ describe('rate-limit-envelope: 429 conforms to canonical shape', () => {
|
|
|
43
44
|
console.warn(
|
|
44
45
|
'[rate-limit-envelope] no 429 observed within burst; skipping shape assertions',
|
|
45
46
|
);
|
|
46
|
-
return;
|
|
47
|
+
return softSkip('blocked', 'precondition not met — `!last || last.status !== 429` returned early (seam, prior step, or fixture unavailable)');
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
const retryAfter = last.headers.get('retry-after');
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import { describe, it, expect } from 'vitest';
|
|
29
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
29
30
|
import { createHash, createPublicKey, verify as cryptoVerify } from 'node:crypto';
|
|
30
31
|
|
|
31
32
|
const REGISTRY_BASE = 'https://packs.openwop.dev';
|
|
@@ -54,7 +55,7 @@ describe('registry-public: packs.openwop.dev discovery document', () => {
|
|
|
54
55
|
console.warn(
|
|
55
56
|
'[registry-public] skipped — set OPENWOP_TEST_PUBLIC_REGISTRY=true to enable',
|
|
56
57
|
);
|
|
57
|
-
return;
|
|
58
|
+
return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!ENABLED` returned early');
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
const res = await get('/.well-known/openwop-registry');
|
|
@@ -90,7 +91,7 @@ describe('registry-public: packs.openwop.dev discovery document', () => {
|
|
|
90
91
|
|
|
91
92
|
describe('registry-public: packs.openwop.dev index', () => {
|
|
92
93
|
it('GET /v1/index.json returns a non-empty pack list with valid name + version shapes', async () => {
|
|
93
|
-
if (!ENABLED) return;
|
|
94
|
+
if (!ENABLED) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!ENABLED` returned early');
|
|
94
95
|
|
|
95
96
|
const res = await get('/v1/index.json');
|
|
96
97
|
expect(res.status).toBe(200);
|
|
@@ -119,7 +120,7 @@ describe('registry-public: spec-canonical pack manifests resolve', () => {
|
|
|
119
120
|
|
|
120
121
|
for (const { name, version } of KNOWN_PACKS) {
|
|
121
122
|
it(`GET /v1/packs/${name}/-/${version}.json returns a valid manifest`, async () => {
|
|
122
|
-
if (!ENABLED) return;
|
|
123
|
+
if (!ENABLED) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!ENABLED` returned early');
|
|
123
124
|
|
|
124
125
|
const res = await get(`/v1/packs/${name}/-/${version}.json`);
|
|
125
126
|
expect(res.status).toBe(200);
|
|
@@ -162,7 +163,7 @@ describe('registry-public: tarball + signature + Ed25519 verify roundtrip', () =
|
|
|
162
163
|
}
|
|
163
164
|
|
|
164
165
|
it(`tarball + sig + public key all retrievable, SRI matches, Ed25519 verifies for ${PACK_NAME}@${PACK_VERSION}`, async () => {
|
|
165
|
-
if (!ENABLED) return;
|
|
166
|
+
if (!ENABLED) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!ENABLED` returned early');
|
|
166
167
|
|
|
167
168
|
// 1. Manifest (JSON).
|
|
168
169
|
const manifestRes = await get(`/v1/packs/${PACK_NAME}/-/${PACK_VERSION}.json`);
|
|
@@ -213,7 +214,7 @@ describe('registry-public: tarball + signature + Ed25519 verify roundtrip', () =
|
|
|
213
214
|
// `method=ed25519` — the simpler path. Extending to `manual` would
|
|
214
215
|
// require the tarball extractor from registry/scripts/verify-
|
|
215
216
|
// signatures.mjs which is intentionally out of scope here.
|
|
216
|
-
if (method !== 'ed25519') return;
|
|
217
|
+
if (method !== 'ed25519') return softSkip('blocked', 'precondition not met — `method !== \'ed25519\'` returned early (seam, prior step, or fixture unavailable)');
|
|
217
218
|
|
|
218
219
|
const verified = cryptoVerify(null, tarball.bytes, publicKey, sig.bytes);
|
|
219
220
|
expect(verified, `Ed25519 signature over ${PACK_NAME}@${PACK_VERSION}.tgz MUST verify against ${manifest.signing!.keyId}`)
|
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
*/
|
|
46
46
|
|
|
47
47
|
import { describe, it, expect } from 'vitest';
|
|
48
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
48
49
|
import { driver } from '../lib/driver.js';
|
|
49
50
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
50
51
|
|
|
@@ -78,12 +79,12 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: advertisement shape (R
|
|
|
78
79
|
const d = await readDiscovery();
|
|
79
80
|
if (d === null) {
|
|
80
81
|
ctx.skip();
|
|
81
|
-
return;
|
|
82
|
+
return softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
82
83
|
}
|
|
83
84
|
const rd = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.replayDeterminism;
|
|
84
85
|
if (rd === undefined) {
|
|
85
86
|
ctx.skip(); // optional advertisement — host hasn't opted in
|
|
86
|
-
return;
|
|
87
|
+
return softSkip('blocked', 'precondition not met — `rd === undefined` returned early (seam, prior step, or fixture unavailable)');
|
|
87
88
|
}
|
|
88
89
|
|
|
89
90
|
expect(
|
|
@@ -180,7 +181,7 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
180
181
|
}
|
|
181
182
|
|
|
182
183
|
it('Phase 4 host MUST emit replay.divergedAtRefusal + fail with replay_diverged_at_refusal when original=valid + replay=refusal', async (ctx) => {
|
|
183
|
-
if (!(await gateOnPhase4(ctx))) return;
|
|
184
|
+
if (!(await gateOnPhase4(ctx))) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await gateOnPhase4(ctx))` returned early');
|
|
184
185
|
|
|
185
186
|
const NODE_ID = 'structured-call';
|
|
186
187
|
// Original program: valid envelope. Replay program (set after the
|
|
@@ -192,7 +193,7 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
192
193
|
]);
|
|
193
194
|
if (programStatus === 404) {
|
|
194
195
|
ctx.skip(); // mock-AI program seam not exposed — soft-skip
|
|
195
|
-
return;
|
|
196
|
+
return softSkip('blocked', 'precondition not met — `programStatus === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
196
197
|
}
|
|
197
198
|
expect(programStatus).toBe(200);
|
|
198
199
|
|
|
@@ -201,7 +202,7 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
201
202
|
});
|
|
202
203
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
203
204
|
ctx.skip(); // fixture not advertised
|
|
204
|
-
return;
|
|
205
|
+
return softSkip('blocked', 'precondition not met — `createRes.status === 404 || createRes.status === 422` returned early (seam, prior step, or fixture unavailable)');
|
|
205
206
|
}
|
|
206
207
|
expect(createRes.status).toBe(201);
|
|
207
208
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -264,7 +265,7 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
264
265
|
});
|
|
265
266
|
|
|
266
267
|
it('Phase 4 host MUST emit replay.divergedAtRefusal + fail with replay_diverged_at_refusal when original=refusal + replay=valid (symmetric case)', async (ctx) => {
|
|
267
|
-
if (!(await gateOnPhase4(ctx))) return;
|
|
268
|
+
if (!(await gateOnPhase4(ctx))) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await gateOnPhase4(ctx))` returned early');
|
|
268
269
|
|
|
269
270
|
const NODE_ID = 'structured-call';
|
|
270
271
|
// Symmetric: original=refusal, replay=valid.
|
|
@@ -273,7 +274,7 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
273
274
|
]);
|
|
274
275
|
if (programStatus === 404) {
|
|
275
276
|
ctx.skip();
|
|
276
|
-
return;
|
|
277
|
+
return softSkip('blocked', 'precondition not met — `programStatus === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
277
278
|
}
|
|
278
279
|
expect(programStatus).toBe(200);
|
|
279
280
|
|
|
@@ -282,7 +283,7 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
282
283
|
});
|
|
283
284
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
284
285
|
ctx.skip();
|
|
285
|
-
return;
|
|
286
|
+
return softSkip('blocked', 'precondition not met — `createRes.status === 404 || createRes.status === 422` returned early (seam, prior step, or fixture unavailable)');
|
|
286
287
|
}
|
|
287
288
|
expect(createRes.status).toBe(201);
|
|
288
289
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -20,14 +20,19 @@
|
|
|
20
20
|
* portability requirement at the single-host boundary).
|
|
21
21
|
*
|
|
22
22
|
* 2. The emitted key is reproducible offline: locally recomputed
|
|
23
|
-
* SHA-256-over-RFC-8785-JCS over the canonical
|
|
23
|
+
* SHA-256-over-RFC-8785-JCS over the canonical **v2** semantic request
|
|
24
|
+
* (RFC 0150 §C, `recipe: "openwop-semantic-request-v2"`) MUST equal
|
|
24
25
|
* the host's emission. This is the load-bearing claim — without
|
|
25
26
|
* it, the recipe is private host state masquerading as a content-
|
|
26
27
|
* addressable hash.
|
|
27
28
|
*
|
|
28
|
-
* 3. (Negative) Permuting any
|
|
29
|
-
* `
|
|
30
|
-
*
|
|
29
|
+
* 3. (Negative) Permuting any transport-only field (`stream`, `metadata`,
|
|
30
|
+
* `user`, request IDs, trace context, tenant id, timeout) MUST NOT
|
|
31
|
+
* shift the key. (`max_tokens`/`stop`/`seed` were in this list until
|
|
32
|
+
* suite 1.109.0; RFC 0150 §C moved them INTO the recipe because each
|
|
33
|
+
* changes the completion — see `replay-llm-cache-key.test.ts` for the
|
|
34
|
+
* sensitivity legs. Asserting their exclusion here was the suite
|
|
35
|
+
* contradicting `semantic-digest-v2.test.ts` in the same package.) This is the security boundary: hosts
|
|
31
36
|
* that mix non-recipe state into the key leak that state across
|
|
32
37
|
* the cache boundary, defeating the portability claim and (via
|
|
33
38
|
* the SR-1 sibling invariant) potentially leaking BYOK plaintexts
|
|
@@ -35,7 +40,8 @@
|
|
|
35
40
|
*
|
|
36
41
|
* 4. (Gated on Phase 4 advertisement.) The host's discovery doc MUST
|
|
37
42
|
* advertise `replayDeterminism.llmCacheKeyRecipe` matching the
|
|
38
|
-
* recipe it honors — `spec-rfc-0041` for the canonical recipe,
|
|
43
|
+
* recipe it honors — `spec-rfc-0041` for the canonical recipe (which,
|
|
44
|
+
* since RFC 0150 §C replaced §A/§B in place, IS the v2 recipe),
|
|
39
45
|
* `x-host-<host>-<recipe-name>` for vendor variants per
|
|
40
46
|
* `host-extensions.md` §"Canonical prefixes".
|
|
41
47
|
*
|
|
@@ -46,13 +52,13 @@
|
|
|
46
52
|
*
|
|
47
53
|
* @see RFCS/0041-multi-agent-replay-under-nondeterminism.md §E
|
|
48
54
|
* @see SECURITY/invariants.yaml §replay-llm-cache-key-portable
|
|
49
|
-
* @see spec/v1/replay.md §"LLM cache-key recipe" §A + §B + §D
|
|
55
|
+
* @see spec/v1/replay.md §"LLM cache-key recipe" §A + §B + §D (v2, RFC 0150 §C)
|
|
50
56
|
* @see conformance/src/scenarios/replay-llm-cache-key.test.ts (the sibling behavioral suite)
|
|
51
57
|
*/
|
|
52
58
|
|
|
53
59
|
import { describe, it, expect } from 'vitest';
|
|
54
60
|
import { driver } from '../lib/driver.js';
|
|
55
|
-
import {
|
|
61
|
+
import { semanticRequestDigestV2, callCacheKeySeam as callSeam } from '../lib/llm-cache-key-recipe.js';
|
|
56
62
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
57
63
|
|
|
58
64
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
@@ -102,7 +108,7 @@ describe.skipIf(HTTP_SKIP)('replay-llm-cache-key-portable: intra-host reproducib
|
|
|
102
108
|
'SECURITY/invariants.yaml §replay-llm-cache-key-portable + replay.md §B',
|
|
103
109
|
'host cache key MUST be reproducible offline from the recipe alone — no host-internal state',
|
|
104
110
|
),
|
|
105
|
-
).toBe(
|
|
111
|
+
).toBe(semanticRequestDigestV2(input));
|
|
106
112
|
});
|
|
107
113
|
|
|
108
114
|
it('two identical probes MUST yield byte-identical keys (intra-host determinism)', async (ctx) => {
|
|
@@ -129,7 +135,7 @@ describe.skipIf(HTTP_SKIP)('replay-llm-cache-key-portable: intra-host reproducib
|
|
|
129
135
|
});
|
|
130
136
|
|
|
131
137
|
describe.skipIf(HTTP_SKIP)('replay-llm-cache-key-portable: non-recipe-field invariance (RFC 0041 §E security boundary)', () => {
|
|
132
|
-
it('
|
|
138
|
+
it('transport-only fields (request ID, trace context, tenant ID) MUST NOT influence the cache key', async (ctx) => {
|
|
133
139
|
const base = {
|
|
134
140
|
provider: 'openai',
|
|
135
141
|
model: 'gpt-4',
|
|
@@ -146,20 +152,20 @@ describe.skipIf(HTTP_SKIP)('replay-llm-cache-key-portable: non-recipe-field inva
|
|
|
146
152
|
// would expose tenant/request state through cache-collision behavior.
|
|
147
153
|
const polluted = {
|
|
148
154
|
...base,
|
|
149
|
-
max_tokens: 1000,
|
|
150
|
-
stop: ['STOP'],
|
|
151
155
|
stream: true,
|
|
152
|
-
seed: 42,
|
|
153
156
|
metadata: { tenantId: 'tenant-A', traceparent: '00-deadbeef-cafe-01' },
|
|
154
157
|
user: 'user-42',
|
|
155
158
|
'x-request-id': 'req-abc-123',
|
|
159
|
+
tenantId: 'tenant-A',
|
|
160
|
+
timeoutMs: 30_000,
|
|
161
|
+
runId: 'run_123',
|
|
156
162
|
};
|
|
157
163
|
const pollutedResult = await callSeam(polluted);
|
|
158
164
|
expect(
|
|
159
165
|
pollutedResult.cacheKey,
|
|
160
166
|
driver.describe(
|
|
161
167
|
'SECURITY/invariants.yaml §replay-llm-cache-key-portable + replay.md §A',
|
|
162
|
-
'
|
|
168
|
+
'transport-only fields (request id, trace context, tenant id, run id, timeout) MUST NOT influence the cache key — leaking them defeats the portability invariant',
|
|
163
169
|
),
|
|
164
170
|
).toBe(baseResult.cacheKey);
|
|
165
171
|
});
|
|
@@ -1,30 +1,38 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* LLM cache-key recipe — `replay.md §"LLM cache-key recipe"` §A + §B
|
|
2
|
+
* LLM cache-key recipe — `replay.md §"LLM cache-key recipe"` §A + §B, at the
|
|
3
|
+
* RFC 0150 §C **v2** semantic request.
|
|
3
4
|
*
|
|
4
|
-
* Verifies that an OpenWOP host computes the LLM cache key per the
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* Verifies that an OpenWOP host computes the LLM cache key per the normative
|
|
6
|
+
* recipe: SHA-256 over RFC 8785 JCS-canonicalized JSON of the v2 canonical
|
|
7
|
+
* object — `{ recipe: "openwop-semantic-request-v2", provider, model,
|
|
8
|
+
* request: { messages, tools?, temperature?, topP?, topK?, maxOutputTokens?,
|
|
9
|
+
* seed?, stop?, responseFormat?, safetySettings? }, providerOptions? }`.
|
|
8
10
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* **Why this file was rewritten (2026-08-16).** Until suite 1.109.0 it
|
|
12
|
+
* recomputed the expected key with the *v1* projection and asserted that
|
|
13
|
+
* `max_tokens / stop / seed` MUST NOT influence the key — while
|
|
14
|
+
* `semantic-digest-v2.test.ts`, in the same package since `730ff3de`, asserted
|
|
15
|
+
* that excluding them "is wrong … a wrong hit, not a miss". A host that
|
|
16
|
+
* implemented the current recipe (openwop-app, ADR 0549 P3) went red here for
|
|
17
|
+
* being right, and the pinned suite contradicted itself. RFC 0150 §C replaced
|
|
18
|
+
* v1 in place as a safety-fix; this file now follows it. `stream`, `metadata`,
|
|
19
|
+
* `user`, request IDs, and trace context remain excluded — they are transport
|
|
20
|
+
* or bookkeeping and cannot change the completion.
|
|
14
21
|
*
|
|
15
|
-
* The
|
|
16
|
-
*
|
|
17
|
-
*
|
|
22
|
+
* The single-host assertions drive the seam at
|
|
23
|
+
* `POST /v1/host/sample/test/llm-cache-key` (`host-sample-test-seams.md` §4)
|
|
24
|
+
* and recompute the expected key locally, asserting equality. The cross-host
|
|
25
|
+
* assertion requires `OPENWOP_BASE_URL_B` (operator-supplied).
|
|
18
26
|
*
|
|
19
|
-
* @see spec/v1/replay.md §"LLM cache-key recipe"
|
|
27
|
+
* @see spec/v1/replay.md §"LLM cache-key recipe" (v2), RFC 0150 §C
|
|
20
28
|
*/
|
|
21
29
|
|
|
22
30
|
import { describe, it, expect } from 'vitest';
|
|
23
31
|
import { driver } from '../lib/driver.js';
|
|
24
|
-
import {
|
|
32
|
+
import { semanticRequestDigestV2, callCacheKeySeam as callSeam } from '../lib/llm-cache-key-recipe.js';
|
|
25
33
|
|
|
26
|
-
describe('replay-llm-cache-key: SHA-256-over-JCS recipe (replay.md §B)', () => {
|
|
27
|
-
it('host cache key MUST equal locally-recomputed
|
|
34
|
+
describe('replay-llm-cache-key: SHA-256-over-JCS recipe, v2 (replay.md §B, RFC 0150 §C)', () => {
|
|
35
|
+
it('host cache key MUST equal the locally-recomputed v2 semantic-request digest', async () => {
|
|
28
36
|
const input = {
|
|
29
37
|
provider: 'anthropic',
|
|
30
38
|
model: 'claude-3-5-sonnet-20240620',
|
|
@@ -39,8 +47,13 @@ describe('replay-llm-cache-key: SHA-256-over-JCS recipe (replay.md §B)', () =>
|
|
|
39
47
|
expect(result.status).toBe(200);
|
|
40
48
|
expect(
|
|
41
49
|
result.cacheKey,
|
|
42
|
-
driver.describe(
|
|
43
|
-
|
|
50
|
+
driver.describe(
|
|
51
|
+
'replay.md §B (RFC 0150 §C)',
|
|
52
|
+
'host cache key MUST be lowercase-hex SHA-256 of the JCS-canonical v2 semantic request ' +
|
|
53
|
+
'(stamped `recipe: "openwop-semantic-request-v2"`); a v1 key here is a host that has ' +
|
|
54
|
+
'not adopted the safety-fix and would return wrong hits for requests differing in seed/stop/max tokens',
|
|
55
|
+
),
|
|
56
|
+
).toBe(semanticRequestDigestV2(input));
|
|
44
57
|
});
|
|
45
58
|
|
|
46
59
|
it('cache key MUST be 64 lowercase-hex characters (SHA-256 output shape)', async () => {
|
|
@@ -54,8 +67,8 @@ describe('replay-llm-cache-key: SHA-256-over-JCS recipe (replay.md §B)', () =>
|
|
|
54
67
|
});
|
|
55
68
|
});
|
|
56
69
|
|
|
57
|
-
describe('replay-llm-cache-key:
|
|
58
|
-
it('
|
|
70
|
+
describe('replay-llm-cache-key: transport-only fields are EXCLUDED (replay.md §A, v2)', () => {
|
|
71
|
+
it('stream / metadata / user / request ids / trace context MUST NOT influence the cache key', async () => {
|
|
59
72
|
const base = {
|
|
60
73
|
provider: 'openai',
|
|
61
74
|
model: 'gpt-4',
|
|
@@ -65,22 +78,26 @@ describe('replay-llm-cache-key: non-recipe fields are EXCLUDED (replay.md §A)',
|
|
|
65
78
|
const baseResult = await callSeam(base);
|
|
66
79
|
if (baseResult.status === 404) return;
|
|
67
80
|
|
|
68
|
-
//
|
|
81
|
+
// Transport / bookkeeping only. None of these can change what the model
|
|
82
|
+
// returns, so none may change the key. (`max_tokens`, `stop`, and `seed`
|
|
83
|
+
// used to be in this list — they CAN change the completion, which is why
|
|
84
|
+
// RFC 0150 §C moved them into the recipe; see the next describe.)
|
|
69
85
|
const noisy = {
|
|
70
86
|
...base,
|
|
71
|
-
max_tokens: 1000,
|
|
72
|
-
stop: ['STOP'],
|
|
73
87
|
stream: true,
|
|
74
|
-
seed: 42,
|
|
75
88
|
metadata: { traceId: 'abcd' },
|
|
76
89
|
user: 'unit-test-user',
|
|
90
|
+
'x-request-id': 'req-123',
|
|
91
|
+
traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01',
|
|
92
|
+
tenantId: 'tenant-b',
|
|
93
|
+
timeoutMs: 30_000,
|
|
77
94
|
};
|
|
78
95
|
const noisyResult = await callSeam(noisy);
|
|
79
96
|
expect(
|
|
80
97
|
noisyResult.cacheKey,
|
|
81
98
|
driver.describe(
|
|
82
|
-
'replay.md §A',
|
|
83
|
-
'cache key MUST be invariant under
|
|
99
|
+
'replay.md §A (v2)',
|
|
100
|
+
'cache key MUST be invariant under transport-only fields (stream, metadata, user, request ids, trace context, tenant, timeout)',
|
|
84
101
|
),
|
|
85
102
|
).toBe(baseResult.cacheKey);
|
|
86
103
|
});
|
|
@@ -103,35 +120,91 @@ describe('replay-llm-cache-key: non-recipe fields are EXCLUDED (replay.md §A)',
|
|
|
103
120
|
});
|
|
104
121
|
});
|
|
105
122
|
|
|
123
|
+
describe('replay-llm-cache-key: outcome-affecting fields are INCLUDED (RFC 0150 §C — the v2 safety-fix)', () => {
|
|
124
|
+
const base = {
|
|
125
|
+
provider: 'openai',
|
|
126
|
+
model: 'gpt-4',
|
|
127
|
+
messages: [{ role: 'user', content: 'v2-sensitivity-probe' }],
|
|
128
|
+
temperature: 0.2,
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
// Each of these fields changes the completion. v1 excluded them, which keyed
|
|
132
|
+
// two requests that produce DIFFERENT text to the SAME entry — a wrong hit,
|
|
133
|
+
// not a miss. A host still keying under v1 passes the exclusion leg above and
|
|
134
|
+
// fails here, which is the point: the two legs together distinguish "drops
|
|
135
|
+
// transport fields" from "drops the wrong fields".
|
|
136
|
+
it.each([
|
|
137
|
+
['seed', { seed: 42 }, { seed: 43 }],
|
|
138
|
+
['stop', { stop: ['STOP'] }, { stop: ['HALT'] }],
|
|
139
|
+
['maxOutputTokens', { maxOutputTokens: 100 }, { maxOutputTokens: 1000 }],
|
|
140
|
+
])('%s DOES change the cache key', async (_field, a, b) => {
|
|
141
|
+
const ra = await callSeam({ ...base, ...a });
|
|
142
|
+
if (ra.status === 404) return;
|
|
143
|
+
const rb = await callSeam({ ...base, ...b });
|
|
144
|
+
expect(
|
|
145
|
+
ra.cacheKey === rb.cacheKey,
|
|
146
|
+
driver.describe(
|
|
147
|
+
'replay.md §A (RFC 0150 §C)',
|
|
148
|
+
`\`${_field}\` decides the completion, so two requests differing only in it MUST NOT share a cache key — ` +
|
|
149
|
+
'sharing one returns a response the caller never asked for, deterministically',
|
|
150
|
+
),
|
|
151
|
+
).toBe(false);
|
|
152
|
+
// And each side MUST equal the locally-recomputed v2 digest, so a host cannot
|
|
153
|
+
// pass this leg by salting the key with something arbitrary.
|
|
154
|
+
expect(ra.cacheKey, driver.describe('replay.md §B (v2)', 'key MUST equal the v2 digest')).toBe(
|
|
155
|
+
semanticRequestDigestV2({ ...base, ...a }),
|
|
156
|
+
);
|
|
157
|
+
expect(rb.cacheKey, driver.describe('replay.md §B (v2)', 'key MUST equal the v2 digest')).toBe(
|
|
158
|
+
semanticRequestDigestV2({ ...base, ...b }),
|
|
159
|
+
);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it('providerOptions are carried into the key, not dropped', async () => {
|
|
163
|
+
const withOpts = { ...base, providerOptions: { 'vendor.openai.logitBias': { '50256': -100 } } };
|
|
164
|
+
const withOtherOpts = { ...base, providerOptions: { 'vendor.openai.logitBias': { '50256': 100 } } };
|
|
165
|
+
const r1 = await callSeam(withOpts);
|
|
166
|
+
if (r1.status === 404) return;
|
|
167
|
+
const r2 = await callSeam(withOtherOpts);
|
|
168
|
+
expect(
|
|
169
|
+
r1.cacheKey === r2.cacheKey,
|
|
170
|
+
driver.describe(
|
|
171
|
+
'replay.md §B step 1 (RFC 0150 §C)',
|
|
172
|
+
'an unknown provider option MUST be carried into `providerOptions` before hashing — a dropped option ' +
|
|
173
|
+
'that alters output is indistinguishable from one that was never set',
|
|
174
|
+
),
|
|
175
|
+
).toBe(false);
|
|
176
|
+
expect(r1.cacheKey, driver.describe('replay.md §B (v2)', 'key MUST equal the v2 digest')).toBe(
|
|
177
|
+
semanticRequestDigestV2(withOpts),
|
|
178
|
+
);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
|
|
106
182
|
describe('replay-llm-cache-key: cross-host parity (replay.md §D)', () => {
|
|
107
183
|
it('two hosts compute the same cache key for the same input (when OPENWOP_BASE_URL_B is configured)', async () => {
|
|
108
|
-
const otherBaseUrl = process.env
|
|
109
|
-
if (!otherBaseUrl
|
|
184
|
+
const otherBaseUrl = process.env['OPENWOP_BASE_URL_B'];
|
|
185
|
+
if (!otherBaseUrl) return; // operator-supplied second host not configured
|
|
110
186
|
const input = {
|
|
111
|
-
provider: '
|
|
112
|
-
model: '
|
|
113
|
-
messages: [
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
],
|
|
117
|
-
temperature: 0.5,
|
|
187
|
+
provider: 'openai',
|
|
188
|
+
model: 'gpt-4',
|
|
189
|
+
messages: [{ role: 'user', content: 'cross-host parity probe' }],
|
|
190
|
+
temperature: 0.3,
|
|
191
|
+
seed: 7,
|
|
118
192
|
};
|
|
119
|
-
const
|
|
120
|
-
if (
|
|
121
|
-
const otherApiKey = process.env.OPENWOP_API_KEY_B ?? process.env.OPENWOP_API_KEY ?? '';
|
|
122
|
-
// Issue the second probe directly via fetch since the driver is bound to
|
|
123
|
-
// OPENWOP_BASE_URL. Authorization mirrors the suite's default.
|
|
193
|
+
const resA = await callSeam(input);
|
|
194
|
+
if (resA.status === 404) return;
|
|
124
195
|
const resB = await fetch(`${otherBaseUrl.replace(/\/$/, '')}/v1/host/sample/test/llm-cache-key`, {
|
|
125
196
|
method: 'POST',
|
|
126
|
-
headers: { '
|
|
197
|
+
headers: { 'content-type': 'application/json' },
|
|
127
198
|
body: JSON.stringify(input),
|
|
128
199
|
});
|
|
129
|
-
if (resB.status === 404) return;
|
|
130
|
-
|
|
131
|
-
const b = (await resB.json()) as { cacheKey?: string };
|
|
200
|
+
if (resB.status === 404) return;
|
|
201
|
+
const jsonB = (await resB.json()) as { cacheKey?: string };
|
|
132
202
|
expect(
|
|
133
|
-
|
|
203
|
+
jsonB.cacheKey,
|
|
134
204
|
driver.describe('replay.md §D', 'two compliant hosts MUST compute byte-identical cache keys for the same recipe input'),
|
|
135
|
-
).toBe(
|
|
205
|
+
).toBe(resA.cacheKey);
|
|
206
|
+
expect(resA.cacheKey, driver.describe('replay.md §B (v2)', 'and both MUST equal the v2 digest')).toBe(
|
|
207
|
+
semanticRequestDigestV2(input),
|
|
208
|
+
);
|
|
136
209
|
});
|
|
137
210
|
});
|