@openwop/openwop-conformance 1.72.2 → 1.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +2 -2
- package/api/asyncapi.yaml +58 -0
- package/api/openapi.yaml +4 -1
- package/dist/cli.js +107 -1
- package/dist/lib/profiles.js +70 -4
- package/package.json +2 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/schemas/README.md +2 -0
- package/schemas/capabilities.schema.json +2054 -572
- package/schemas/certification-bundle-v2.schema.json +108 -0
- package/schemas/run-event-payloads.schema.json +3411 -857
- package/schemas/run-event.schema.json +31 -9
- package/schemas/workflow-definition.schema.json +492 -130
- package/schemas/workload-identity.schema.json +73 -0
- package/src/cli.ts +119 -1
- package/src/lib/a2a-fake-peer.ts +20 -0
- package/src/lib/behavior-gate.ts +42 -7
- package/src/lib/llm-cache-key-recipe.ts +51 -0
- package/src/lib/mcp-fake-server.ts +20 -0
- package/src/lib/profiles.ts +95 -4
- package/src/lib/requirement-ledger.ts +138 -0
- package/src/lib/requirement-registry.ts +62 -0
- package/src/scenarios/a2a-version-negotiation.test.ts +159 -0
- package/src/scenarios/capability-example-root-layout.test.ts +113 -0
- package/src/scenarios/certification-bundle-v2.test.ts +157 -0
- package/src/scenarios/certification-floor-enforcement.test.ts +115 -0
- package/src/scenarios/compensation-behavior.test.ts +164 -0
- package/src/scenarios/compensation-profile.test.ts +175 -0
- package/src/scenarios/contract-provenance.test.ts +209 -0
- package/src/scenarios/core-manifest-and-extension-registry.test.ts +198 -0
- package/src/scenarios/discovery-canonical-family-no-shadow.test.ts +219 -0
- package/src/scenarios/effect-identity-composition.test.ts +129 -0
- package/src/scenarios/effect-identity-cross-scope.test.ts +82 -0
- package/src/scenarios/mcp-version-negotiation.test.ts +159 -0
- package/src/scenarios/multi-region-effect-vocabulary.test.ts +175 -0
- package/src/scenarios/multi-region-idempotency.test.ts +17 -7
- package/src/scenarios/openapi-resolved-paths.test.ts +127 -0
- package/src/scenarios/protocol-version-grammar.test.ts +119 -0
- package/src/scenarios/requirement-ledger.test.ts +162 -0
- package/src/scenarios/rfc-0147-self-audit.test.ts +104 -0
- package/src/scenarios/rfc-lifecycle-coherence.test.ts +137 -0
- package/src/scenarios/semantic-digest-v2.test.ts +128 -0
- package/src/scenarios/semantic-digest-vectors.test.ts +140 -0
- package/src/scenarios/spec-corpus-validity.test.ts +22 -8
- package/src/scenarios/strict-behavior-gate.test.ts +120 -0
- package/src/scenarios/versioned-composition-profiles.test.ts +183 -0
- package/src/scenarios/workload-identity-behavior.test.ts +188 -0
- package/src/scenarios/workload-identity-profile.test.ts +175 -0
- package/vectors/semantic-request-digest-v2.json +236 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0148 §A — the requirement execution ledger, and the properties that make
|
|
3
|
+
* it non-vacuous.
|
|
4
|
+
*
|
|
5
|
+
* §A's operative sentence is negative: "A plain test return, caught exception
|
|
6
|
+
* converted to a return, or empty assertion body **MUST NOT** produce
|
|
7
|
+
* `executed-pass`." A ledger that merely *offers* five dispositions does not
|
|
8
|
+
* deliver that — it delivers it only if **absence** resolves to something other
|
|
9
|
+
* than a pass, because every vacuity found in this corpus so far reached
|
|
10
|
+
* `pass` by not running rather than by running wrong:
|
|
11
|
+
*
|
|
12
|
+
* - `floorProven = missingFloor.length === 0 && prefixOk` over an undefined
|
|
13
|
+
* floor — `[].every(...)` is `true`;
|
|
14
|
+
* - a gated subtest that 404'd, soft-skipped, and left the file green;
|
|
15
|
+
* - a scenario whose assertions never executed but whose file still counted.
|
|
16
|
+
*
|
|
17
|
+
* So these legs test the *default*, not the happy path. A ledger where an
|
|
18
|
+
* unrecorded requirement reads as `executed-pass` would pass every
|
|
19
|
+
* disposition-vocabulary check and still be the bug.
|
|
20
|
+
*
|
|
21
|
+
* Server-free and always-on.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { describe, it, expect, beforeEach } from 'vitest';
|
|
25
|
+
import {
|
|
26
|
+
DISPOSITIONS,
|
|
27
|
+
CERTIFIABLE,
|
|
28
|
+
recordRequirement,
|
|
29
|
+
dispositionOf,
|
|
30
|
+
entryOf,
|
|
31
|
+
snapshot,
|
|
32
|
+
resetLedger,
|
|
33
|
+
verifyProfileRequirements,
|
|
34
|
+
} from '../lib/requirement-ledger.js';
|
|
35
|
+
import { allRequirements, requirementsFor } from '../lib/requirement-registry.js';
|
|
36
|
+
|
|
37
|
+
describe('RFC 0148 §A — requirement execution ledger', () => {
|
|
38
|
+
beforeEach(() => resetLedger());
|
|
39
|
+
|
|
40
|
+
it('the disposition vocabulary is exactly the five §A names', () => {
|
|
41
|
+
expect([...DISPOSITIONS].sort()).toEqual(
|
|
42
|
+
['blocked', 'executed-fail', 'executed-pass', 'inapplicable', 'skipped'].sort(),
|
|
43
|
+
);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('an unrecorded requirement resolves to blocked, NOT to a pass', () => {
|
|
47
|
+
// The load-bearing leg. A scenario that returned early, threw and swallowed,
|
|
48
|
+
// or was never written leaves no entry — and the honest reading of no entry
|
|
49
|
+
// is "not exercised". Every vacuity in this corpus reached `pass` this way.
|
|
50
|
+
expect(
|
|
51
|
+
dispositionOf('openwop.never.recorded'),
|
|
52
|
+
'RFC 0148 §A: silence is evidence of nothing. An absent disposition MUST NOT read as executed-pass.',
|
|
53
|
+
).toBe('blocked');
|
|
54
|
+
expect(entryOf('openwop.never.recorded').detail).toMatch(/not exercised/);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('blocked is not certifiable, while skipped and inapplicable are', () => {
|
|
58
|
+
// "We could not check" and "we checked and it holds" are the two states this
|
|
59
|
+
// whole program exists to stop conflating.
|
|
60
|
+
expect(CERTIFIABLE).not.toContain('blocked');
|
|
61
|
+
expect(CERTIFIABLE).not.toContain('executed-fail');
|
|
62
|
+
expect([...CERTIFIABLE].sort()).toEqual(['executed-pass', 'inapplicable', 'skipped']);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('a profile with no requirements is not certifiable', () => {
|
|
66
|
+
// This is the `[].every(...)` shape itself. An empty requirement list must
|
|
67
|
+
// not vacuously certify; callers distinguish "no floor by design" from "no
|
|
68
|
+
// floor written yet" BEFORE reaching here.
|
|
69
|
+
const verdict = verifyProfileRequirements('openwop-example', []);
|
|
70
|
+
expect(
|
|
71
|
+
verdict.certifiable,
|
|
72
|
+
'RFC 0148 §C / gap G6: an empty requirement set is exactly the vacuity that started this program.',
|
|
73
|
+
).toBe(false);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('one blocked requirement invalidates the whole profile', () => {
|
|
77
|
+
recordRequirement('openwop.a', 'executed-pass');
|
|
78
|
+
recordRequirement('openwop.b', 'skipped', 'profile not advertised; operator opted out');
|
|
79
|
+
const verdict = verifyProfileRequirements('openwop-example', ['openwop.a', 'openwop.b', 'openwop.c']);
|
|
80
|
+
expect(verdict.certifiable, 'RFC 0148 §A: a blocked requirement invalidates the claim').toBe(false);
|
|
81
|
+
expect(verdict.blocking.map((b) => b.requirementId)).toEqual(['openwop.c']);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('a fully-exercised profile certifies', () => {
|
|
85
|
+
recordRequirement('openwop.a', 'executed-pass');
|
|
86
|
+
recordRequirement('openwop.b', 'inapplicable', 'host advertises no streaming surface');
|
|
87
|
+
expect(verifyProfileRequirements('openwop-example', ['openwop.a', 'openwop.b']).certifiable).toBe(true);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('a non-pass disposition without a reason is rejected', () => {
|
|
91
|
+
// A `blocked` with no explanation is an outcome nobody can act on, and it is
|
|
92
|
+
// the shape a lazily-instrumented scenario would emit by default.
|
|
93
|
+
expect(() => recordRequirement('openwop.a', 'blocked')).toThrow(/without a reason/);
|
|
94
|
+
expect(() => recordRequirement('openwop.b', 'skipped', ' ')).toThrow(/without a reason/);
|
|
95
|
+
// A pass needs no prose — the assertion itself is the evidence.
|
|
96
|
+
expect(() => recordRequirement('openwop.c', 'executed-pass')).not.toThrow();
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('contradictory dispositions for one requirement throw rather than last-write-wins', () => {
|
|
100
|
+
// Silent overwrite would let a later soft-skip bury an earlier real failure
|
|
101
|
+
// — the same conflation running the other direction.
|
|
102
|
+
recordRequirement('openwop.a', 'executed-fail', 'assertion failed against target');
|
|
103
|
+
expect(() => recordRequirement('openwop.a', 'executed-pass')).toThrow(/already recorded/);
|
|
104
|
+
expect(dispositionOf('openwop.a')).toBe('executed-fail');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('re-recording the same disposition is idempotent', () => {
|
|
108
|
+
recordRequirement('openwop.a', 'executed-pass');
|
|
109
|
+
recordRequirement('openwop.a', 'executed-pass');
|
|
110
|
+
expect(snapshot()).toHaveLength(1);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe('RFC 0148 §A — the registry binds to the certification floor', () => {
|
|
115
|
+
beforeEach(() => resetLedger());
|
|
116
|
+
|
|
117
|
+
it('every profile with a runtime floor yields requirement IDs', () => {
|
|
118
|
+
// Guard: an empty registry would make the legs below vacuous.
|
|
119
|
+
expect(allRequirements().length, 'the floor MUST produce requirement IDs').toBeGreaterThan(10);
|
|
120
|
+
expect(allRequirements()).toContain('openwop.floor.runs-lifecycle');
|
|
121
|
+
expect(allRequirements()).toContain('openwop.floor.any.interrupt-');
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('an unwritten floor returns null, not an empty list', () => {
|
|
125
|
+
// `openwop-replay-fork` is deliberately unspecified — `profiles.md` gives it
|
|
126
|
+
// a discovery-conditional floor a flat list cannot express (gap G7). Null
|
|
127
|
+
// forces the caller to decide; an empty array would silently certify.
|
|
128
|
+
expect(
|
|
129
|
+
requirementsFor('openwop-replay-fork'),
|
|
130
|
+
'RFC 0148 §C: unspecified MUST be distinguishable from empty-by-design',
|
|
131
|
+
).toBeNull();
|
|
132
|
+
expect(requirementsFor('openwop-does-not-exist')).toBeNull();
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('a discovery-only profile returns an empty list, which does not certify', () => {
|
|
136
|
+
// `openwop-core` has no runtime floor BY DESIGN — the predicate is the whole
|
|
137
|
+
// claim. That is a decision on record, and it is still not a runtime pass.
|
|
138
|
+
expect(requirementsFor('openwop-core')).toEqual([]);
|
|
139
|
+
expect(
|
|
140
|
+
verifyProfileRequirements('openwop-core', requirementsFor('openwop-core') ?? []).certifiable,
|
|
141
|
+
'RFC 0155 §A: an `openwop-core` badge is a statement about a document, not a running system',
|
|
142
|
+
).toBe(false);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('the real core-standard floor does not certify until every requirement is exercised', () => {
|
|
146
|
+
const ids = requirementsFor('openwop-core-standard');
|
|
147
|
+
expect(ids).not.toBeNull();
|
|
148
|
+
const requirements = ids as readonly string[];
|
|
149
|
+
// Nothing recorded yet: every requirement is blocked, so the profile fails.
|
|
150
|
+
expect(verifyProfileRequirements('openwop-core-standard', requirements).blocking).toHaveLength(
|
|
151
|
+
requirements.length,
|
|
152
|
+
);
|
|
153
|
+
// Record all but one — still not certifiable. Partial evidence is not evidence.
|
|
154
|
+
for (const id of requirements.slice(1)) recordRequirement(id, 'executed-pass');
|
|
155
|
+
const partial = verifyProfileRequirements('openwop-core-standard', requirements);
|
|
156
|
+
expect(partial.certifiable).toBe(false);
|
|
157
|
+
expect(partial.blocking.map((b) => b.requirementId)).toEqual([requirements[0]]);
|
|
158
|
+
// Record the last one and it certifies.
|
|
159
|
+
recordRequirement(requirements[0] as string, 'executed-pass');
|
|
160
|
+
expect(verifyProfileRequirements('openwop-core-standard', requirements).certifiable).toBe(true);
|
|
161
|
+
});
|
|
162
|
+
});
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0147 §A.10 — the program audits itself, and silence is not compliance.
|
|
3
|
+
*
|
|
4
|
+
* §A states ten invariants that "apply to every workstream". §A.10 forbids using
|
|
5
|
+
* the RFC's existence or partial implementation as evidence its gaps are closed.
|
|
6
|
+
* A program that audits everything except itself has the same defect it was
|
|
7
|
+
* written to fix, one level up.
|
|
8
|
+
*
|
|
9
|
+
* This gate does NOT assert the program is compliant. It asserts that every §A
|
|
10
|
+
* invariant carries an **explicit disposition** — including the ones recorded as
|
|
11
|
+
* VIOLATED. That is the RFC 0148 §A design applied to governance: an invariant
|
|
12
|
+
* with no row is uncovered, not satisfied, and a self-audit that quietly omits
|
|
13
|
+
* its uncomfortable rows is worth less than none because it looks like coverage.
|
|
14
|
+
*
|
|
15
|
+
* Two rows are currently VIOLATED and the gate is green, which is the intended
|
|
16
|
+
* behavior. Making the gate fail on a violation would create pressure to delete
|
|
17
|
+
* the row rather than fix the program — the failure mode RFC 0149 §D measured
|
|
18
|
+
* when it declined to ship a gate that fires 69 times on its first run.
|
|
19
|
+
*
|
|
20
|
+
* Server-free.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { describe, it, expect } from 'vitest';
|
|
24
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
25
|
+
import { resolve, dirname, join } from 'node:path';
|
|
26
|
+
import { fileURLToPath } from 'node:url';
|
|
27
|
+
import { V1_DIR } from '../lib/paths.js';
|
|
28
|
+
|
|
29
|
+
const AUDIT = V1_DIR === null
|
|
30
|
+
? null
|
|
31
|
+
: join(resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'), 'docs', 'RFC-0147-SELF-AUDIT.md');
|
|
32
|
+
|
|
33
|
+
/** The ten §A invariants, by their heading anchor. */
|
|
34
|
+
const INVARIANTS = [
|
|
35
|
+
'A.1',
|
|
36
|
+
'A.2',
|
|
37
|
+
'A.3',
|
|
38
|
+
'A.4',
|
|
39
|
+
'A.5',
|
|
40
|
+
'A.6',
|
|
41
|
+
'A.7',
|
|
42
|
+
'A.8',
|
|
43
|
+
'A.9',
|
|
44
|
+
'A.10',
|
|
45
|
+
] as const;
|
|
46
|
+
|
|
47
|
+
const DISPOSITIONS = /satisfied|VIOLATED|not satisfied|partially satisfied/;
|
|
48
|
+
|
|
49
|
+
const AUDIT_PRESENT = AUDIT !== null && existsSync(AUDIT);
|
|
50
|
+
|
|
51
|
+
describe.skipIf(!AUDIT_PRESENT)('RFC 0147 §A.10 — program self-audit', () => {
|
|
52
|
+
// `describe.skipIf` still EXECUTES this factory — it decides afterwards not to
|
|
53
|
+
// run the tests it collected. So a read here throws at collection time even
|
|
54
|
+
// when the suite is destined to skip, and one file's collection error takes
|
|
55
|
+
// the whole file down rather than skipping it. The audit lives under `docs/`,
|
|
56
|
+
// which is not bundled, so this is exactly the published-layout path.
|
|
57
|
+
const doc = AUDIT_PRESENT ? readFileSync(AUDIT as string, 'utf8') : '';
|
|
58
|
+
|
|
59
|
+
it('the audit exists and is substantive', () => {
|
|
60
|
+
// Guard: a stub file would make every leg below vacuous, and this gate's
|
|
61
|
+
// whole purpose is that an omitted row is visible.
|
|
62
|
+
expect(doc.length, 'the self-audit MUST be substantive').toBeGreaterThan(2000);
|
|
63
|
+
expect(doc).toMatch(/RFC 0147/);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('every §A invariant carries an explicit disposition', () => {
|
|
67
|
+
const missing = INVARIANTS.filter((id) => {
|
|
68
|
+
const heading = new RegExp(`^## ${id.replace('.', '\\.')} — `, 'm');
|
|
69
|
+
const at = heading.exec(doc);
|
|
70
|
+
if (at === null) return true;
|
|
71
|
+
// The disposition must appear in that section, not merely somewhere.
|
|
72
|
+
const section = doc.slice(at.index, doc.indexOf('\n## ', at.index + 1) + 1 || undefined);
|
|
73
|
+
return !DISPOSITIONS.test(section);
|
|
74
|
+
});
|
|
75
|
+
expect(
|
|
76
|
+
missing,
|
|
77
|
+
'RFC 0147 §A.10: an invariant with no recorded disposition is UNCOVERED, not satisfied. ' +
|
|
78
|
+
'A self-audit that omits its uncomfortable rows is worth less than none, because it ' +
|
|
79
|
+
'looks like coverage.',
|
|
80
|
+
).toEqual([]);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('the audit records the violations it found rather than only the wins', () => {
|
|
84
|
+
// The load-bearing leg. An audit that reported nothing adverse would be
|
|
85
|
+
// indistinguishable from one nobody ran — and this program has two known
|
|
86
|
+
// violations, so a clean sheet here means the file stopped being honest.
|
|
87
|
+
expect(
|
|
88
|
+
/VIOLATED/.test(doc),
|
|
89
|
+
'RFC 0147 §A.5 and §A.6 are currently violated: four RFCs reached `Accepted` with no ' +
|
|
90
|
+
'evidence, and five high-risk RFCs had their comment windows waived by the exact ' +
|
|
91
|
+
'bootstrap mechanism §A.6 says must not shorten them. If those rows have been removed ' +
|
|
92
|
+
'rather than resolved, this leg is the thing that notices.',
|
|
93
|
+
).toBe(true);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it('the audit does not claim the exit criteria are met', () => {
|
|
97
|
+
// §A.10, applied to the audit itself.
|
|
98
|
+
expect(
|
|
99
|
+
/exit criteria in RFC 0147 are not met|exit criteria .{0,40}not met/i.test(doc),
|
|
100
|
+
'RFC 0147 §A.10: the program MUST NOT be read as closed. Three remaining gates — external ' +
|
|
101
|
+
'audit, second maintainer, Tier-3 host — cannot be closed by work in this repository.',
|
|
102
|
+
).toBe(true);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0149 §D — an `Accepted` RFC's unticked acceptance items must say why.
|
|
3
|
+
*
|
|
4
|
+
* §D asks the corpus generator to fail when an `Accepted` RFC "retains an
|
|
5
|
+
* unresolved acceptance blocker not explicitly carried to a register/known-limit".
|
|
6
|
+
* The obvious signal is the `- [ ]` boxes under §"Acceptance criteria", and the
|
|
7
|
+
* obvious gate — every box ticked before `Accepted` — was measured and rejected.
|
|
8
|
+
*
|
|
9
|
+
* `docs/RFC-LIFECYCLE-COHERENCE.md` recorded the distribution: of 141 `Accepted`
|
|
10
|
+
* RFCs, 42% ticked every box, 25% ticked none, 24% ticked some. A blanket gate
|
|
11
|
+
* fails 69 RFCs on its first run, mostly for an authoring convention, and a gate
|
|
12
|
+
* that fires 69 times on its first run gets disabled rather than fixed.
|
|
13
|
+
*
|
|
14
|
+
* The first triage hypothesis was also wrong, and correcting it produced the rule
|
|
15
|
+
* this gate actually enforces. The partially-ticked RFCs are not a blocker
|
|
16
|
+
* backlog: reading `0027`/`0028`/`0029`/`0040`/`0041` shows every trailing item
|
|
17
|
+
* deliberately unticked AND annotated with why — "(Will land alongside the first
|
|
18
|
+
* non-steward advertisement.)", "(Path-to-Accepted.)", "(Follow-up — … not
|
|
19
|
+
* normative gate-blockers.)". That inline annotation IS §D's "explicitly
|
|
20
|
+
* carried", just carried in a parenthetical rather than a register row.
|
|
21
|
+
*
|
|
22
|
+
* So the signal is **annotated vs bare, not ticked vs unticked**. An unticked
|
|
23
|
+
* item with no explanation is indistinguishable from one nobody checked; an
|
|
24
|
+
* unticked item that states its external gate is a decision on record. Ticking
|
|
25
|
+
* nothing stays legal — the RFC did not use the mechanism — but leaving a box
|
|
26
|
+
* unticked and unexplained does not.
|
|
27
|
+
*
|
|
28
|
+
* Applies from `LIFECYCLE_RULE_RFC` forward. Earlier RFCs are the dated record
|
|
29
|
+
* of a period when the convention did not exist, and rewriting them would make
|
|
30
|
+
* the record lie; the boundary is asserted here rather than assumed, so a NEW
|
|
31
|
+
* RFC cannot inherit the exemption. Same carve-out shape as RFC 0149 §B's
|
|
32
|
+
* root-layout lint.
|
|
33
|
+
*
|
|
34
|
+
* Server-free. `RFCS/` is repository-only, so this self-skips under the
|
|
35
|
+
* published tarball layout.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { describe, it, expect } from 'vitest';
|
|
39
|
+
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
|
40
|
+
import { join, resolve as pathResolve } from 'node:path';
|
|
41
|
+
import { V1_DIR } from '../lib/paths.js';
|
|
42
|
+
|
|
43
|
+
/** RFC 0149 §D takes effect with the RFC 0147 program's own cohort. */
|
|
44
|
+
const LIFECYCLE_RULE_RFC = 147;
|
|
45
|
+
|
|
46
|
+
const RFCS_DIR = V1_DIR === null ? null : pathResolve(V1_DIR, '..', '..', 'RFCS');
|
|
47
|
+
|
|
48
|
+
interface BareItem {
|
|
49
|
+
readonly rfc: number;
|
|
50
|
+
readonly file: string;
|
|
51
|
+
readonly line: number;
|
|
52
|
+
readonly text: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* An item is "carried" when it states a reason: a parenthetical, or a pointer to
|
|
57
|
+
* a gap register / known-limit / follow-up RFC. Anything else is bare.
|
|
58
|
+
*/
|
|
59
|
+
function isAnnotated(text: string): boolean {
|
|
60
|
+
if (/\([^)]{12,}\)/.test(text)) return true;
|
|
61
|
+
return /\b(gap register|known.limit|register row|carried|deferred|blocked on|gated on)\b/i.test(text);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function statusOf(doc: string): string | null {
|
|
65
|
+
const m = /^\|\s*\*\*Status\*\*\s*\|\s*`([^`]+)`/m.exec(doc);
|
|
66
|
+
return m === null ? null : m[1]!;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Unticked acceptance items lacking a stated reason, for `Accepted` RFCs at or after the cutoff. */
|
|
70
|
+
function bareItems(dir: string): BareItem[] {
|
|
71
|
+
const found: BareItem[] = [];
|
|
72
|
+
for (const name of readdirSync(dir).filter((f) => /^\d{4}-.*\.md$/.test(f)).sort()) {
|
|
73
|
+
const rfc = Number.parseInt(name.slice(0, 4), 10);
|
|
74
|
+
if (!Number.isFinite(rfc) || rfc < LIFECYCLE_RULE_RFC) continue;
|
|
75
|
+
const doc = readFileSync(join(dir, name), 'utf8');
|
|
76
|
+
if (statusOf(doc) !== 'Accepted') continue;
|
|
77
|
+
const lines = doc.split('\n');
|
|
78
|
+
let inAcceptance = false;
|
|
79
|
+
for (let i = 0; i < lines.length; i++) {
|
|
80
|
+
const line = lines[i]!;
|
|
81
|
+
if (/^##\s/.test(line)) inAcceptance = /^##\s+Acceptance criteria\s*$/.test(line);
|
|
82
|
+
if (!inAcceptance) continue;
|
|
83
|
+
if (!/^\s*-\s*\[ \]\s/.test(line)) continue;
|
|
84
|
+
const text = line.replace(/^\s*-\s*\[ \]\s*/, '');
|
|
85
|
+
if (!isAnnotated(text)) found.push({ rfc, file: name, line: i + 1, text });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return found;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
describe.skipIf(RFCS_DIR === null || !existsSync(RFCS_DIR))('RFC 0149 §D — lifecycle coherence', () => {
|
|
92
|
+
const dir = RFCS_DIR as string;
|
|
93
|
+
|
|
94
|
+
it('the scan reaches Accepted RFCs in the governed range', () => {
|
|
95
|
+
// Guard: a status regex that matched nothing, or a heading regex that never
|
|
96
|
+
// entered the section, would make the assertion below vacuously true. That
|
|
97
|
+
// is the failure RFC 0148 exists to close, and it is especially easy here
|
|
98
|
+
// because the gate's PASSING state and its BROKEN state look identical.
|
|
99
|
+
const governed = readdirSync(dir)
|
|
100
|
+
.filter((f) => /^\d{4}-.*\.md$/.test(f))
|
|
101
|
+
.filter((f) => Number.parseInt(f.slice(0, 4), 10) >= LIFECYCLE_RULE_RFC)
|
|
102
|
+
.filter((f) => statusOf(readFileSync(join(dir, f), 'utf8')) === 'Accepted');
|
|
103
|
+
expect(
|
|
104
|
+
governed.length,
|
|
105
|
+
`at least one Accepted RFC numbered >= ${LIFECYCLE_RULE_RFC} MUST exist for this gate to mean anything`,
|
|
106
|
+
).toBeGreaterThan(0);
|
|
107
|
+
|
|
108
|
+
const withBoxes = governed.filter((f) =>
|
|
109
|
+
/^##\s+Acceptance criteria\s*$/m.test(readFileSync(join(dir, f), 'utf8')),
|
|
110
|
+
);
|
|
111
|
+
expect(withBoxes.length, 'the acceptance-criteria heading MUST be found').toBeGreaterThan(0);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('every unticked acceptance item states why it is unticked', () => {
|
|
115
|
+
const bare = bareItems(dir).map((b) => `RFCS/${b.file}:${b.line} — ${b.text}`);
|
|
116
|
+
expect(
|
|
117
|
+
bare,
|
|
118
|
+
'RFC 0149 §D: an unticked acceptance item on an `Accepted` RFC MUST carry its reason — an ' +
|
|
119
|
+
'external gate, a follow-up note, or a register / known-limit pointer. Unticked-and-' +
|
|
120
|
+
'unexplained is indistinguishable from unchecked, which is what makes the checkbox ' +
|
|
121
|
+
'signal unusable. Tick it, annotate it, or carry it to a register row.\n ' +
|
|
122
|
+
bare.join('\n '),
|
|
123
|
+
).toEqual([]);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('the rule binds the cohort that proposed it', () => {
|
|
127
|
+
// RFC 0149 is itself in the governed range. A gate whose author exempted
|
|
128
|
+
// their own RFC would be the shape RFC 0147 §A.10 forbids — citing a
|
|
129
|
+
// program's status as evidence its gaps are closed.
|
|
130
|
+
const self = readFileSync(join(dir, '0149-machine-contract-and-version-reconciliation.md'), 'utf8');
|
|
131
|
+
expect(statusOf(self), 'RFC 0149 is `Accepted`').toBe('Accepted');
|
|
132
|
+
expect(
|
|
133
|
+
149 >= LIFECYCLE_RULE_RFC,
|
|
134
|
+
'RFC 0149 MUST fall inside the range its own §D governs',
|
|
135
|
+
).toBe(true);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0150 §C — the semantic request digest, and three defects in `replay.md`.
|
|
3
|
+
*
|
|
4
|
+
* **1. The exclusion list forbids exactly what §C requires.** `replay.md` §A
|
|
5
|
+
* says "Fields NOT in this set MUST NOT influence the cache key — including but
|
|
6
|
+
* not limited to: `max_tokens`, `stop`, … `seed`". §C says the digest "MUST
|
|
7
|
+
* cover the complete semantic provider request … maximum output bound, stop
|
|
8
|
+
* conditions, seed". Those are direct opposites, and §A's side is wrong: a
|
|
9
|
+
* request with `stop: ["END"]` and one without produce different completions, so
|
|
10
|
+
* a cache keyed identically for both **returns the wrong response** — not a
|
|
11
|
+
* miss, a wrong hit. Same for `seed`, whose entire purpose is to change output,
|
|
12
|
+
* and for the output bound, which decides whether a response is truncated.
|
|
13
|
+
*
|
|
14
|
+
* **2. It prescribes a normalization JCS does not perform.** Step 2 says
|
|
15
|
+
* canonicalize "via RFC 8785 JCS", then tells hosts without JCS to emit "UTF-8
|
|
16
|
+
* NFC for all strings". JCS does **not** apply NFC. So the two paths the same
|
|
17
|
+
* sentence offers produce **different bytes for the same input** whenever a
|
|
18
|
+
* string is not already NFC — which is the portability property §D claims as a
|
|
19
|
+
* normative invariant. §C: "Implementations MUST NOT add Unicode normalization
|
|
20
|
+
* outside JCS."
|
|
21
|
+
*
|
|
22
|
+
* **3. It cites a formula that no longer exists.** §C quotes the Layer-2 id as
|
|
23
|
+
* `sha256(runId ':' nodeId ':' attempt ':' providerKey)` — the composition RFC
|
|
24
|
+
* 0150 §B retired as a safety-fix, because `attempt` in the preimage guaranteed
|
|
25
|
+
* a duplicate effect on every retry. `idempotency.md` is v1.4; `replay.md` was
|
|
26
|
+
* left quoting v1.1. This one is a cross-document staleness the §B change itself
|
|
27
|
+
* introduced and did not catch.
|
|
28
|
+
*
|
|
29
|
+
* Server-free; reads the corpus. `spec/v1/` is repository-only, so it self-skips
|
|
30
|
+
* under the published tarball layout.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { describe, it, expect } from 'vitest';
|
|
34
|
+
import { readFileSync } from 'node:fs';
|
|
35
|
+
import { join } from 'node:path';
|
|
36
|
+
import { V1_DIR } from '../lib/paths.js';
|
|
37
|
+
|
|
38
|
+
/** Fields whose presence changes the model's output, so they change its identity. */
|
|
39
|
+
const OUTCOME_AFFECTING = ['seed', 'stop', 'maxOutputTokens'];
|
|
40
|
+
|
|
41
|
+
describe.skipIf(V1_DIR === null)('RFC 0150 §C — semantic request digest v2', () => {
|
|
42
|
+
const doc = V1_DIR === null ? '' : readFileSync(join(V1_DIR as string, 'replay.md'), 'utf8');
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Normative text only. Blockquote lines are excluded because this document
|
|
46
|
+
* deliberately QUOTES the rules it retired, so a reader can see what changed
|
|
47
|
+
* and why — and a naive substring search cannot tell a live rule from a
|
|
48
|
+
* quoted-and-retired one. Same carve-out shape as RFC 0149 §B's lint, which
|
|
49
|
+
* exempts pre-0073 RFCs as the dated record of what was proposed.
|
|
50
|
+
*
|
|
51
|
+
* The exemption is narrow on purpose: it covers `>` blocks, not the body, so
|
|
52
|
+
* a retired rule cannot be resurrected into normative voice unnoticed.
|
|
53
|
+
*/
|
|
54
|
+
const normative = doc
|
|
55
|
+
.split('\n')
|
|
56
|
+
.filter((l) => !/^\s*>/.test(l))
|
|
57
|
+
.join('\n');
|
|
58
|
+
const plain = normative.replace(/[`*_]/g, '').replace(/\s+/g, ' ');
|
|
59
|
+
/** Full text including quotes — used only where a quote is the thing under test. */
|
|
60
|
+
const everything = doc.replace(/[`*_]/g, '').replace(/\s+/g, ' ');
|
|
61
|
+
|
|
62
|
+
it('the digest recipe section is found at all', () => {
|
|
63
|
+
// Guard: an empty read makes every leg below vacuously true.
|
|
64
|
+
expect(doc.length, 'replay.md MUST be readable').toBeGreaterThan(1000);
|
|
65
|
+
expect(plain, 'the cache-key recipe MUST exist').toMatch(/cache key/i);
|
|
66
|
+
// The blockquote filter must not swallow the document.
|
|
67
|
+
expect(
|
|
68
|
+
plain.length / everything.length,
|
|
69
|
+
'the normative-text filter MUST retain most of the document, or every leg below is vacuous',
|
|
70
|
+
).toBeGreaterThan(0.7);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('outcome-affecting fields are covered, not excluded', () => {
|
|
74
|
+
for (const field of OUTCOME_AFFECTING) {
|
|
75
|
+
expect(
|
|
76
|
+
plain.includes(field),
|
|
77
|
+
`RFC 0150 §C: the digest MUST cover \`${field}\`. Excluding it means two requests that ` +
|
|
78
|
+
'produce different completions share a cache key — a wrong hit, not a miss.',
|
|
79
|
+
).toBe(true);
|
|
80
|
+
}
|
|
81
|
+
// The v1 exclusion sentence named them as MUST NOT influence. It must be gone.
|
|
82
|
+
expect(
|
|
83
|
+
/Fields NOT in this set MUST NOT influence the cache key/.test(plain),
|
|
84
|
+
'RFC 0150 §C: the v1 exclusion list forbade the very fields §C requires. It cannot survive ' +
|
|
85
|
+
'alongside the v2 recipe — a reader following it would build the colliding digest.',
|
|
86
|
+
).toBe(false);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('no Unicode normalization is prescribed outside JCS', () => {
|
|
90
|
+
expect(
|
|
91
|
+
/UTF-8 NFC for all strings/.test(plain),
|
|
92
|
+
'RFC 0150 §C: "Implementations MUST NOT add Unicode normalization outside JCS." JCS does ' +
|
|
93
|
+
'not apply NFC, so offering NFC as the no-JCS fallback makes the two paths produce ' +
|
|
94
|
+
'different bytes for the same input — defeating the portability §D asserts.',
|
|
95
|
+
).toBe(false);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('the recipe carries a version stamp so v1 and v2 digests cannot collide', () => {
|
|
99
|
+
expect(
|
|
100
|
+
plain,
|
|
101
|
+
'RFC 0150 §C: the canonical object carries `recipe: "openwop-semantic-request-v2"`, so a ' +
|
|
102
|
+
'digest computed under the old rules is distinguishable rather than silently comparable.',
|
|
103
|
+
).toContain('openwop-semantic-request-v2');
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('unknown provider options are carried, not dropped', () => {
|
|
107
|
+
expect(
|
|
108
|
+
/providerOptions/.test(plain),
|
|
109
|
+
'RFC 0150 §C: unknown provider options MUST go in a closed, namespaced `providerOptions` ' +
|
|
110
|
+
'object before hashing. "Silently dropping them is nonconformant" — a dropped option that ' +
|
|
111
|
+
'changes output is the collision this section exists to prevent.',
|
|
112
|
+
).toBe(true);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('the Layer-2 cross-reference matches the composition that actually exists', () => {
|
|
116
|
+
// Cross-document staleness introduced by RFC 0150 §B and not caught by it.
|
|
117
|
+
expect(
|
|
118
|
+
/attempt \|\| ':' \|\| providerKey|nodeId ':' attempt/.test(plain),
|
|
119
|
+
'RFC 0150 §B retired the `attempt`-bearing Layer-2 composition as a safety-fix. `replay.md` ' +
|
|
120
|
+
'MUST NOT keep quoting it — a reader following this section would rebuild the exact ' +
|
|
121
|
+
'defect §B removed, and would do so believing they were following the spec.',
|
|
122
|
+
).toBe(false);
|
|
123
|
+
expect(
|
|
124
|
+
plain,
|
|
125
|
+
'replay.md MUST cite the current Layer-2 identity',
|
|
126
|
+
).toContain('logicalInvocationId');
|
|
127
|
+
});
|
|
128
|
+
});
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0150 §C — the golden vectors, and the relationships they exist to pin.
|
|
3
|
+
*
|
|
4
|
+
* §C's acceptance criterion is that TypeScript, Python, and Go compute the same
|
|
5
|
+
* digest. Prose cannot deliver that: three independent readings of "canonicalize
|
|
6
|
+
* via JCS and hash" is precisely how three implementations disagree, and the
|
|
7
|
+
* disagreement is invisible until two hosts replay the same run and get
|
|
8
|
+
* different cache keys.
|
|
9
|
+
*
|
|
10
|
+
* So the vectors are the contract, and this gate holds the TypeScript
|
|
11
|
+
* implementation to them. An SDK in another language reproduces the same file.
|
|
12
|
+
*
|
|
13
|
+
* The vectors are not a flat list of examples. Several are **pairs**, and the
|
|
14
|
+
* relationship between the members is the actual requirement:
|
|
15
|
+
*
|
|
16
|
+
* - tools sorted vs reversed → MUST be equal (order is not semantic)
|
|
17
|
+
* - message order reversed → MUST differ (order IS semantic)
|
|
18
|
+
* - two Unicode forms of "é" → MUST differ, because JCS does not apply NFC
|
|
19
|
+
*
|
|
20
|
+
* That last pair is the one worth keeping. A well-meaning implementer who adds
|
|
21
|
+
* NFC "to be safe" makes those two vectors collide, and every other vector still
|
|
22
|
+
* passes — so a suite that checked only individual digests would go green on an
|
|
23
|
+
* implementation that had silently broken cross-host agreement.
|
|
24
|
+
*
|
|
25
|
+
* Server-free and always-on. The vectors ship with the conformance package.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { describe, it, expect } from 'vitest';
|
|
29
|
+
import { readFileSync } from 'node:fs';
|
|
30
|
+
import { fileURLToPath } from 'node:url';
|
|
31
|
+
import { dirname, resolve } from 'node:path';
|
|
32
|
+
import {
|
|
33
|
+
semanticRequestDigestV2,
|
|
34
|
+
projectSemanticRequestV2,
|
|
35
|
+
canonicalize,
|
|
36
|
+
SEMANTIC_REQUEST_RECIPE_V2,
|
|
37
|
+
} from '../lib/llm-cache-key-recipe.js';
|
|
38
|
+
|
|
39
|
+
interface Vector {
|
|
40
|
+
readonly id: string;
|
|
41
|
+
readonly why: string;
|
|
42
|
+
readonly input: Record<string, unknown>;
|
|
43
|
+
readonly canonical: string;
|
|
44
|
+
readonly digest: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const VECTORS_PATH = resolve(
|
|
48
|
+
dirname(fileURLToPath(import.meta.url)),
|
|
49
|
+
'..',
|
|
50
|
+
'..',
|
|
51
|
+
'vectors',
|
|
52
|
+
'semantic-request-digest-v2.json',
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
const doc = JSON.parse(readFileSync(VECTORS_PATH, 'utf8')) as {
|
|
56
|
+
recipe: string;
|
|
57
|
+
vectors: readonly Vector[];
|
|
58
|
+
};
|
|
59
|
+
const byId = new Map(doc.vectors.map((v) => [v.id, v]));
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Recompute a vector's digest from its INPUT.
|
|
63
|
+
*
|
|
64
|
+
* Deliberately not `v.digest`. Reading the stored value would make the
|
|
65
|
+
* relationship legs below compare two constants out of the same file — they
|
|
66
|
+
* would validate that the vector SET is internally distinct and would pass
|
|
67
|
+
* unchanged against a broken implementation. That is the vacuity this whole
|
|
68
|
+
* program exists to close, and the first draft of this file had it: the NFC
|
|
69
|
+
* sabotage was caught by the per-vector reproduction leg, while the leg written
|
|
70
|
+
* specifically to catch it could not have failed.
|
|
71
|
+
*/
|
|
72
|
+
function digest(id: string): string {
|
|
73
|
+
const v = byId.get(id);
|
|
74
|
+
if (v === undefined) throw new Error(`vector '${id}' is missing from the golden set`);
|
|
75
|
+
return semanticRequestDigestV2(v.input);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The stored expectation, for legs that check the file rather than the code. */
|
|
79
|
+
function storedDigest(id: string): string {
|
|
80
|
+
const v = byId.get(id);
|
|
81
|
+
if (v === undefined) throw new Error(`vector '${id}' is missing from the golden set`);
|
|
82
|
+
return v.digest;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
describe('RFC 0150 §C — semantic request digest golden vectors', () => {
|
|
86
|
+
it('the vector set is present and non-trivial', () => {
|
|
87
|
+
// Guard: an empty or truncated file would make every leg below vacuous, and
|
|
88
|
+
// this gate's whole value is that it fails when an implementation drifts.
|
|
89
|
+
expect(doc.recipe).toBe(SEMANTIC_REQUEST_RECIPE_V2);
|
|
90
|
+
expect(doc.vectors.length, 'the golden set MUST cover the recipe').toBeGreaterThanOrEqual(10);
|
|
91
|
+
expect(new Set(doc.vectors.map((v) => v.id)).size).toBe(doc.vectors.length);
|
|
92
|
+
for (const v of doc.vectors) {
|
|
93
|
+
expect(v.why.length, `vector '${v.id}' MUST say what it pins`).toBeGreaterThan(20);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it.each(doc.vectors.map((v) => [v.id, v] as const))(
|
|
98
|
+
'the implementation reproduces %s',
|
|
99
|
+
(_id, v) => {
|
|
100
|
+
// The preimage is asserted as well as the hash: a mismatch on `canonical`
|
|
101
|
+
// tells an implementer WHICH field they got wrong, where a hash mismatch
|
|
102
|
+
// tells them only that something is.
|
|
103
|
+
expect(canonicalize(projectSemanticRequestV2(v.input)), v.why).toBe(v.canonical);
|
|
104
|
+
expect(semanticRequestDigestV2(v.input), v.why).toBe(v.digest);
|
|
105
|
+
},
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
it('tool order is not semantic — sorted and reversed agree', () => {
|
|
109
|
+
expect(digest('tools-sorted-by-name')).toBe(digest('tools-reversed-same-digest'));
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it('message order IS semantic — reversing changes the digest', () => {
|
|
113
|
+
expect(
|
|
114
|
+
digest('message-order-is-semantic'),
|
|
115
|
+
'messages carry conversational sequence; sorting them would make two different conversations collide',
|
|
116
|
+
).not.toBe(digest('message-order-reversed'));
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('the two Unicode forms do not collide, because JCS does not apply NFC', () => {
|
|
120
|
+
// The pair that catches a well-meaning "add NFC to be safe" change — and it
|
|
121
|
+
// catches it only because `digest()` RECOMPUTES from the input. Comparing
|
|
122
|
+
// stored values here would be two constants from one file.
|
|
123
|
+
expect(storedDigest('non-ascii-not-normalized')).not.toBe(storedDigest('non-ascii-composed'));
|
|
124
|
+
expect(
|
|
125
|
+
digest('non-ascii-not-normalized'),
|
|
126
|
+
'RFC 0150 §C: "Implementations MUST NOT add Unicode normalization outside JCS." An ' +
|
|
127
|
+
'implementation that normalizes makes these two collide and silently loses cross-host ' +
|
|
128
|
+
'byte agreement — the property `replay.md` §D asserts as a normative invariant.',
|
|
129
|
+
).not.toBe(digest('non-ascii-composed'));
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('each field v1 excluded now changes the digest', () => {
|
|
133
|
+
// The defect §C fixes, stated as three inequalities. Under v1 all three of
|
|
134
|
+
// these were equal to `minimal`, so requests producing different completions
|
|
135
|
+
// shared a cache key — a wrong hit, not a miss.
|
|
136
|
+
for (const id of ['stop-changes-digest', 'seed-changes-digest', 'max-output-changes-digest']) {
|
|
137
|
+
expect(digest(id), `${id} MUST NOT equal the minimal request`).not.toBe(digest('minimal'));
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
});
|