@openwop/openwop-conformance 1.135.4 → 1.136.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/fixtures/conformance-agent-memory-cross-tenant.json +41 -8
- package/fixtures/conformance-agent-memory-ttl.json +35 -7
- package/fixtures/conformance-mcp-tool-roundtrip.json +26 -7
- package/fixtures.md +3 -3
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/src/lib/execution-model-version.ts +22 -0
- package/src/scenarios/agent-loop-version5-shape.test.ts +7 -2
- package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +29 -7
- package/src/scenarios/agentMemoryTtlExpiry.test.ts +18 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +4 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +24 -6
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +6 -2
- package/src/scenarios/node-failed-payload-shape.test.ts +72 -0
|
@@ -1,28 +1,61 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "conformance-agent-memory-cross-tenant",
|
|
3
3
|
"name": "Conformance: Agent Memory Cross-Tenant Isolation (CTI-1)",
|
|
4
|
-
"version": "1.
|
|
5
|
-
"description": "Phase 3 / CTI-1.
|
|
4
|
+
"version": "1.1",
|
|
5
|
+
"description": "Phase 3 / CTI-1. TWO-SIDED since 2026-08-17 (S35): the `core.identity` node with `memoryAction: \"cross-tenant-probe\"` MUST (1) write one entry under the run's OWN tenant at `agent.memoryRef` and land its id in `ownerEntryId`, (2) list that owner memoryRef and land the result in `ownerProbe` — a NON-EMPTY array containing the written entry (the positive control: proves the adapter is really being exercised), and (3) issue the deliberately-cross-tenant probe at `config.probeMemoryRef` (a path owned by ANOTHER tenant) and land the result in `crossTenantProbe`, which MUST be exactly `[]` (or `null`) — never another tenant's entries, and never left unset. A host may instead fail the run with a security-related error at step 3. Before S35 the scenario accepted an unset `crossTenantProbe`, so a host that ignored `memoryAction` entirely passed a critical-tier invariant vacuously. See agentMemoryCrossTenantIsolation.test.ts.",
|
|
6
6
|
"nodes": [
|
|
7
7
|
{
|
|
8
8
|
"id": "cross-tenant-probe",
|
|
9
9
|
"typeId": "core.identity",
|
|
10
10
|
"name": "Cross-Tenant Memory Probe",
|
|
11
|
-
"position": {
|
|
11
|
+
"position": {
|
|
12
|
+
"x": 0,
|
|
13
|
+
"y": 0
|
|
14
|
+
},
|
|
12
15
|
"config": {
|
|
13
16
|
"memoryAction": "cross-tenant-probe",
|
|
14
17
|
"probeMemoryRef": "conformance/another-tenant/agent-memory"
|
|
15
18
|
},
|
|
16
|
-
"inputs": {}
|
|
19
|
+
"inputs": {},
|
|
20
|
+
"agent": {
|
|
21
|
+
"agentId": "core.conformance.cti-agent",
|
|
22
|
+
"memoryRef": "conformance/agent-memory-cti"
|
|
23
|
+
}
|
|
17
24
|
}
|
|
18
25
|
],
|
|
19
26
|
"edges": [],
|
|
20
27
|
"triggers": [
|
|
21
|
-
{
|
|
28
|
+
{
|
|
29
|
+
"id": "manual",
|
|
30
|
+
"type": "manual",
|
|
31
|
+
"enabled": true
|
|
32
|
+
}
|
|
22
33
|
],
|
|
23
34
|
"variables": [
|
|
24
|
-
{
|
|
35
|
+
{
|
|
36
|
+
"name": "ownerEntryId",
|
|
37
|
+
"type": "string"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"name": "ownerProbe",
|
|
41
|
+
"type": "array"
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"name": "crossTenantProbe",
|
|
45
|
+
"type": "array"
|
|
46
|
+
}
|
|
25
47
|
],
|
|
26
|
-
"metadata": {
|
|
27
|
-
|
|
48
|
+
"metadata": {
|
|
49
|
+
"tags": [
|
|
50
|
+
"conformance",
|
|
51
|
+
"multi-agent",
|
|
52
|
+
"phase-3",
|
|
53
|
+
"memory",
|
|
54
|
+
"cti-1",
|
|
55
|
+
"security"
|
|
56
|
+
]
|
|
57
|
+
},
|
|
58
|
+
"settings": {
|
|
59
|
+
"timeout": 10000
|
|
60
|
+
}
|
|
28
61
|
}
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "conformance-agent-memory-ttl",
|
|
3
3
|
"name": "Conformance: Agent Memory TTL Expiry",
|
|
4
|
-
"version": "1.
|
|
5
|
-
"description": "Phase 3. Writes two memory entries — one with `expiresAt` in the past, one in the future — then lists
|
|
4
|
+
"version": "1.1",
|
|
5
|
+
"description": "Phase 3. Writes two memory entries under `agent.memoryRef` — one with `expiresAt` in the past, one in the future — lands their ids in `expiredId` / `freshId`, then lists into `memoryList`. TWO-SIDED since 2026-08-17 (S35): the list MUST contain the entry whose id is `freshId` and MUST NOT contain the entry whose id is `expiredId`; before S35 an empty list (a host that filtered everything, or wrote nothing) passed. See agentMemoryTtlExpiry.test.ts.",
|
|
6
6
|
"nodes": [
|
|
7
7
|
{
|
|
8
8
|
"id": "memory-ttl-probe",
|
|
9
9
|
"typeId": "core.identity",
|
|
10
10
|
"name": "Memory TTL Probe",
|
|
11
|
-
"position": {
|
|
11
|
+
"position": {
|
|
12
|
+
"x": 0,
|
|
13
|
+
"y": 0
|
|
14
|
+
},
|
|
12
15
|
"config": {
|
|
13
16
|
"memoryAction": "ttl-probe"
|
|
14
17
|
},
|
|
@@ -21,11 +24,36 @@
|
|
|
21
24
|
],
|
|
22
25
|
"edges": [],
|
|
23
26
|
"triggers": [
|
|
24
|
-
{
|
|
27
|
+
{
|
|
28
|
+
"id": "manual",
|
|
29
|
+
"type": "manual",
|
|
30
|
+
"enabled": true
|
|
31
|
+
}
|
|
25
32
|
],
|
|
26
33
|
"variables": [
|
|
27
|
-
{
|
|
34
|
+
{
|
|
35
|
+
"name": "freshId",
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"name": "expiredId",
|
|
40
|
+
"type": "string"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"name": "memoryList",
|
|
44
|
+
"type": "array"
|
|
45
|
+
}
|
|
28
46
|
],
|
|
29
|
-
"metadata": {
|
|
30
|
-
|
|
47
|
+
"metadata": {
|
|
48
|
+
"tags": [
|
|
49
|
+
"conformance",
|
|
50
|
+
"multi-agent",
|
|
51
|
+
"phase-3",
|
|
52
|
+
"memory",
|
|
53
|
+
"ttl"
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"settings": {
|
|
57
|
+
"timeout": 10000
|
|
58
|
+
}
|
|
31
59
|
}
|
|
@@ -2,13 +2,16 @@
|
|
|
2
2
|
"id": "conformance-mcp-tool-roundtrip",
|
|
3
3
|
"name": "Conformance: MCP Tool Roundtrip",
|
|
4
4
|
"version": "1.0",
|
|
5
|
-
"description": "Track 6 fixture for `mcp-tool-roundtrip.test.ts`. Workflow invokes a single MCP tool (`echo`) on a host-configured MCP server. The conformance suite stands up a synthetic MCP server at startup; operators configure the host to use that server. The fixture's tool call MUST surface in the run's event log with the trust boundary intact per `threat-model-prompt-injection.md` §UNTRUSTED marker.
|
|
5
|
+
"description": "Track 6 fixture for `mcp-tool-roundtrip.test.ts`. Workflow invokes a single MCP tool (`echo`) on a host-configured MCP server. The conformance suite stands up a synthetic MCP server at startup; operators configure the host to use that server. The fixture's tool call MUST surface in the run's event log with the trust boundary intact per `threat-model-prompt-injection.md` §UNTRUSTED marker. Node `core.conformance.mcp-invoke` is a conformance-RESERVED typeId (renamed 2026-08-17 from `core.ai.callPrompt` + a `config.mcp` shape that vendor pack-tier node never read): a host that consumes MCP MUST map it to its own MCP invoke bridge node and honour `config.mcp { tool, arguments }`; a host that does not consume MCP MUST NOT advertise this fixture (the host-mediated leg then records inapplicable).",
|
|
6
6
|
"nodes": [
|
|
7
7
|
{
|
|
8
8
|
"id": "mcp-call",
|
|
9
|
-
"typeId": "core.
|
|
9
|
+
"typeId": "core.conformance.mcp-invoke",
|
|
10
10
|
"name": "Invoke MCP echo tool",
|
|
11
|
-
"position": {
|
|
11
|
+
"position": {
|
|
12
|
+
"x": 0,
|
|
13
|
+
"y": 0
|
|
14
|
+
},
|
|
12
15
|
"config": {
|
|
13
16
|
"mcp": {
|
|
14
17
|
"tool": "echo",
|
|
@@ -22,11 +25,27 @@
|
|
|
22
25
|
],
|
|
23
26
|
"edges": [],
|
|
24
27
|
"triggers": [
|
|
25
|
-
{
|
|
28
|
+
{
|
|
29
|
+
"id": "manual",
|
|
30
|
+
"type": "manual",
|
|
31
|
+
"enabled": true
|
|
32
|
+
}
|
|
26
33
|
],
|
|
27
34
|
"variables": [
|
|
28
|
-
{
|
|
35
|
+
{
|
|
36
|
+
"name": "text",
|
|
37
|
+
"type": "string",
|
|
38
|
+
"defaultValue": "roundtrip-probe"
|
|
39
|
+
}
|
|
29
40
|
],
|
|
30
|
-
"metadata": {
|
|
31
|
-
|
|
41
|
+
"metadata": {
|
|
42
|
+
"tags": [
|
|
43
|
+
"conformance",
|
|
44
|
+
"mcp",
|
|
45
|
+
"roundtrip"
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
"settings": {
|
|
49
|
+
"timeout": 0
|
|
50
|
+
}
|
|
32
51
|
}
|
package/fixtures.md
CHANGED
|
@@ -90,9 +90,9 @@ All fixtures MUST advertise:
|
|
|
90
90
|
| Multi-Agent Handoff (child) | `conformance-multi-agent-handoff-child` | RFC 0037 (`version: 1`) — child for `conformance-multi-agent-handoff`. Declares `childOutcome.defaultValue='handoff-complete'`; the parent's outputMapping harvests it onto `parentResult`, triggering the `output.harvested` transition event. | `completed` | ≤ 5s |
|
|
91
91
|
| Multi-Agent Confidence Escalation | `conformance-multi-agent-confidence-escalation` | RFC 0039 §A (`version: 2`) — exercises the confidence-floor escalation contract. Supervisor's `mockDispatchPlan` carries ONE decision with `confidence: 0.3` (below the 0.5 spec floor). The host MUST emit `core.workflowChain.confidence-escalated` AND suspend with a clarification interrupt BEFORE any dispatch.began fires; conformance asserts zero `core.workflowChain.event` records (no dispatch). Capability-gated on `capabilities.multiAgent.executionModel.version >= 2`. | `waiting-clarification` | ≤ 30s |
|
|
92
92
|
| Agent Memory Round-Trip | `conformance-agent-memory-roundtrip` | Phase 3 — `MemoryAdapter.list/get` write → read | `completed` | ≤ 15s |
|
|
93
|
-
| Agent Memory Cross-Tenant | `conformance-agent-memory-cross-tenant` | Phase 3 / CTI-1 — cross-tenant probe MUST
|
|
93
|
+
| Agent Memory Cross-Tenant | `conformance-agent-memory-cross-tenant` | Phase 3 / CTI-1 — TWO-SIDED (S35, 2026-08-17): owner write+list lands `ownerEntryId` + non-empty `ownerProbe` (positive control); the cross-tenant probe at `config.probeMemoryRef` lands `crossTenantProbe`, which MUST be exactly `[]` / `null` and MUST be set | `completed` | ≤ 10s |
|
|
94
94
|
| Agent Memory Redaction | `conformance-agent-memory-redaction` | Phase 3 / SR-1 — BYOK plaintext surfaces as `[REDACTED:<id>]` on read | `completed` | ≤ 15s |
|
|
95
|
-
| Agent Memory TTL | `conformance-agent-memory-ttl` | Phase 3 — `expiresAt` excludes expired entries from list/get
|
|
95
|
+
| Agent Memory TTL | `conformance-agent-memory-ttl` | Phase 3 — `expiresAt` excludes expired entries from list/get; TWO-SIDED (S35, 2026-08-17): host lands `freshId` / `expiredId`, `memoryList` MUST include the fresh id and exclude the expired id | `completed` | ≤ 10s |
|
|
96
96
|
| Agent Memory Injection Budget | `conformance-agent-memory-injection-budget` | RFC 0113 — token-budgeted `MemoryAdapter.list` (`tokenBudget`/`rank`/`query`): cumulative tokens ≤ budget, over-budget single entry omitted, SR-1 + CTI-1 re-asserted on the budgeted path; `rank:'relevance'` delegates to `memory.search` semantic | `completed` | ≤ 15s |
|
|
97
97
|
| Context-Budget Multi-Turn | `conformance-context-budget-multiturn` | RFC 0111 — multi-iteration `core.orchestrator.supervisor` loop whose per-turn transcript grows; a host bounding it exposes `{ tokenCounter, tokenCount, eventIds, summarizedRanges }` via the `GET /v1/host/sample/agent/transcript-window` seam (`tokenCount ≤ transcriptTokenBudget`, internally consistent, keepLastTurns verbatim); the summarization leg replays and asserts `context.summarized` → `summaryRef` reuse. Reference hosts that do not advertise `contextBudget` soft-skip | `completed` | ≤ 60s |
|
|
98
98
|
| Conversation Lifecycle | `conformance-conversation-lifecycle` | Phase 4 — open → exchange → close event ordering | `completed` | ≤ 20s |
|
|
@@ -102,7 +102,7 @@ All fixtures MUST advertise:
|
|
|
102
102
|
| Orchestrator Dispatch | `conformance-orchestrator-dispatch` | Phase 5 — supervisor → `next-worker` → dispatch round-trip | `completed` | ≤ 60s |
|
|
103
103
|
| Orchestrator Terminate | `conformance-orchestrator-terminate` | Phase 5 / CO-3 — terminate decision is final | `completed` | ≤ 30s |
|
|
104
104
|
| Orchestrator Low-Confidence | `conformance-orchestrator-low-confidence` | Phase 5 / CP-1 — supervisor low-confidence suspend | `waiting-approval` (suspends) | unbounded (suspends) |
|
|
105
|
-
| MCP Tool Roundtrip | `conformance-mcp-tool-roundtrip` | Track 6 — host invokes a tool on the conformance suite's synthetic MCP server; trust-boundary visibility in the event log | `completed` | ≤ 30s |
|
|
105
|
+
| MCP Tool Roundtrip | `conformance-mcp-tool-roundtrip` | Track 6 — host invokes a tool on the conformance suite's synthetic MCP server; trust-boundary visibility in the event log. Node `core.conformance.mcp-invoke` is a **conformance-RESERVED** typeId (renamed 2026-08-17 from `core.ai.callPrompt` with a `config.mcp` shape that vendor pack-tier node never read — it requires `promptId` and ignores `config.mcp`, so the fixture could not execute as declared on the first host that shipped the vendor node): a host that consumes MCP MUST map it to its own MCP invoke bridge node and honour `config.mcp { tool, arguments }`; a host that does not consume MCP MUST NOT advertise this fixture (the host-mediated leg then records inapplicable) — the same treatment as `core.conformance.a2a-invoke` | `completed` | ≤ 30s |
|
|
106
106
|
| A2A Task Roundtrip | `conformance-a2a-task-roundtrip` | Track 6 — host consumes the conformance suite's synthetic A2A peer; covers drift points #3 (`AUTH_REQUIRED`) and #4 (`REJECTED`). Node `core.conformance.a2a-invoke` is a **conformance-RESERVED** typeId (renamed 2026-08-16 from `core.a2a.invoke`, a multi-segment `core.*` id no host shipped and the corpus never defined): a host that consumes A2A peers MUST map it to its A2A bridge node (a host-extension node — `a2a-integration.md` §"State projection"); a host that does not consume A2A MUST NOT advertise this fixture | `failed` or `waiting-input` (per `driftScenario` input) | ≤ 30s |
|
|
107
107
|
| WASM Pack Roundtrip | `conformance-wasm-pack-roundtrip` | RFC 0008 — invokes `vendor.openwop.rust-hello.greet` (loaded WASM pack); exercises required exports + at least one import | `completed` | ≤ 10s |
|
|
108
108
|
| WASM Pack Memory-Cap Breach | `conformance-wasm-pack-memory-cap-breach` | RFC 0008 §K — invokes the deliberately-misbehaving `vendor.openwop.misbehaving.memory-bomb` pack (allocates 1 GiB beyond the host's `memoryPagesMax`). Host MUST emit `cap.breached` with `kind: "wasm-memory"` and drive the run to terminal `failed`. Misbehaving pack lives at `openwop-examples:examples/packs/rust-misbehaving-memory/` (repo-qualified per the 2026-06 monorepo split — the `openwop-examples` sibling repo) and is fixture-only (NOT signed for registry publication). | `failed` (with `cap.breached`) | ≤ 10s |
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of this vendored schemas/ copy. See conformance/README.md \u00a7\"Resolving the contract\". Compare against the stamp in your installed @openwop/openwop-conformance to detect a stale hand-copied contract.",
|
|
3
|
-
"suiteVersion": "1.
|
|
4
|
-
"corpusCommit": "
|
|
3
|
+
"suiteVersion": "1.136.1",
|
|
4
|
+
"corpusCommit": "9db354d7a8215e24252beda93be717a6413a02df"
|
|
5
5
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `capabilities.multiAgent.executionModel.version` ceiling, read from
|
|
3
|
+
* `capabilities.schema.json` so a scenario never restates it. Two scenarios
|
|
4
|
+
* carried a literal `<= 5` past RFC 0090 (version 6, Accepted 2026-06-08) and
|
|
5
|
+
* failed the very host that witnessed version 6 (S31, 2026-08-17).
|
|
6
|
+
*/
|
|
7
|
+
import { readFileSync } from 'node:fs';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
import { SCHEMAS_DIR } from './paths.js';
|
|
10
|
+
|
|
11
|
+
let cached: number | undefined;
|
|
12
|
+
|
|
13
|
+
export function executionModelVersionMax(): number {
|
|
14
|
+
if (cached !== undefined) return cached;
|
|
15
|
+
const schema = JSON.parse(readFileSync(join(SCHEMAS_DIR, 'capabilities.schema.json'), 'utf8')) as {
|
|
16
|
+
properties?: { multiAgent?: { properties?: { executionModel?: { properties?: { version?: { maximum?: number } } } } } };
|
|
17
|
+
};
|
|
18
|
+
const max = schema.properties?.multiAgent?.properties?.executionModel?.properties?.version?.maximum;
|
|
19
|
+
if (typeof max !== 'number') throw new Error('capabilities.schema.json §multiAgent.executionModel.version has no `maximum`');
|
|
20
|
+
cached = max;
|
|
21
|
+
return max;
|
|
22
|
+
}
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import { describe, it, expect } from 'vitest';
|
|
15
15
|
import { softSkip } from '../lib/soft-skip.js';
|
|
16
16
|
import { driver } from '../lib/driver.js';
|
|
17
|
+
import { executionModelVersionMax } from '../lib/execution-model-version.js';
|
|
17
18
|
import { readExecutionModelCap } from '../lib/agentLoop.js';
|
|
18
19
|
|
|
19
20
|
describe('agent-loop-version5-shape: advertisement (RFC 0061 §A)', () => {
|
|
@@ -33,9 +34,13 @@ describe('agent-loop-version5-shape: advertisement (RFC 0061 §A)', () => {
|
|
|
33
34
|
).toBe(true);
|
|
34
35
|
}
|
|
35
36
|
if (typeof em.version === 'number') {
|
|
37
|
+
// S31 (2026-08-17): the ceiling is the schema's `maximum` (6 since RFC 0090,
|
|
38
|
+
// Accepted 2026-06-08 — MyndHyve was the version-6 witness and this leg failed
|
|
39
|
+
// it against a stale literal 5 for ten weeks). Read, never restate.
|
|
40
|
+
const max = executionModelVersionMax();
|
|
36
41
|
expect(
|
|
37
|
-
(em.version as number) >= 1 && (em.version as number) <=
|
|
38
|
-
driver.describe('capabilities.schema.json §multiAgent.executionModel',
|
|
42
|
+
(em.version as number) >= 1 && (em.version as number) <= max,
|
|
43
|
+
driver.describe('capabilities.schema.json §multiAgent.executionModel', `version MUST be within the 1–${max} ladder (schema maximum)`),
|
|
39
44
|
).toBe(true);
|
|
40
45
|
}
|
|
41
46
|
});
|
|
@@ -40,15 +40,37 @@ describe.skipIf(SKIP)('agentMemoryCrossTenantIsolation: CTI-1 invariant', () =>
|
|
|
40
40
|
if (terminal.status === 'completed') {
|
|
41
41
|
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
42
42
|
const body = snap.json as { variables?: Record<string, unknown> };
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
//
|
|
43
|
+
const vars = body.variables ?? {};
|
|
44
|
+
|
|
45
|
+
// S35 (2026-08-17): POSITIVE CONTROL first. Before this, the only assertion
|
|
46
|
+
// was "crossTenantProbe is empty or falsy" — a host whose core.identity node
|
|
47
|
+
// ignored `config.memoryAction` entirely left the variable undefined and
|
|
48
|
+
// PASSED a critical-tier invariant vacuously (measured by openwop-app H49).
|
|
49
|
+
// The owner-side write+list proves the MemoryAdapter is really exercised.
|
|
50
|
+
const ownerProbe = vars.ownerProbe;
|
|
51
|
+
expect(
|
|
52
|
+
Array.isArray(ownerProbe) && ownerProbe.length > 0,
|
|
53
|
+
driver.describe('agent-memory.md §CTI-1 / fixtures.md conformance-agent-memory-cross-tenant', 'ownerProbe MUST be a NON-EMPTY array — the run\'s own tenant can read the entry it just wrote (positive control; an unset or empty ownerProbe means the probe never ran)'),
|
|
54
|
+
).toBe(true);
|
|
55
|
+
const ownerEntryId = vars.ownerEntryId;
|
|
56
|
+
if (typeof ownerEntryId === 'string' && ownerEntryId.length > 0) {
|
|
57
|
+
expect(
|
|
58
|
+
(ownerProbe as Array<{ id?: unknown }>).some((e) => e && typeof e === 'object' && e.id === ownerEntryId),
|
|
59
|
+
driver.describe('agent-memory.md §CTI-1', `ownerProbe MUST contain the entry the host reports writing (ownerEntryId=${ownerEntryId})`),
|
|
60
|
+
).toBe(true);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Cross-tenant list MUST return exactly [] (or null); never another
|
|
64
|
+
// tenant's entries, and never left unset.
|
|
65
|
+
const crossTenantResult = vars.crossTenantProbe;
|
|
66
|
+
expect(
|
|
67
|
+
crossTenantResult !== undefined,
|
|
68
|
+
driver.describe('agent-memory.md §CTI-1', 'crossTenantProbe MUST be set — an unset variable means the cross-tenant probe was never issued, which proves nothing'),
|
|
69
|
+
).toBe(true);
|
|
48
70
|
if (Array.isArray(crossTenantResult)) {
|
|
49
|
-
expect(crossTenantResult.length).toBe(0);
|
|
71
|
+
expect(crossTenantResult.length, driver.describe('agent-memory.md §CTI-1', 'a cross-tenant memoryRef probe MUST return [] — no entries of another tenant')).toBe(0);
|
|
50
72
|
} else {
|
|
51
|
-
expect(crossTenantResult).
|
|
73
|
+
expect(crossTenantResult, driver.describe('agent-memory.md §CTI-1', 'a non-array cross-tenant probe result MUST be null')).toBeNull();
|
|
52
74
|
}
|
|
53
75
|
}
|
|
54
76
|
});
|
|
@@ -44,5 +44,23 @@ describe.skipIf(SKIP)('agentMemoryTtlExpiry: expired entries are excluded from l
|
|
|
44
44
|
expect(new Date(e.expiresAt).getTime()).toBeGreaterThan(now);
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
|
+
// S35 (2026-08-17): TWO-SIDED. `[]` used to pass (a host that filtered
|
|
48
|
+
// everything, or wrote nothing and returned []). The fixture now lands the
|
|
49
|
+
// ids of both entries; the fresh one MUST be present, the expired one MUST
|
|
50
|
+
// be absent.
|
|
51
|
+
const freshId = body.variables?.freshId;
|
|
52
|
+
const expiredId = body.variables?.expiredId;
|
|
53
|
+
expect(
|
|
54
|
+
typeof freshId === 'string' && freshId.length > 0 && typeof expiredId === 'string' && expiredId.length > 0,
|
|
55
|
+
driver.describe('fixtures.md conformance-agent-memory-ttl', 'the host MUST land freshId and expiredId (the ids of the two entries it wrote) so the list can be checked on both sides'),
|
|
56
|
+
).toBe(true);
|
|
57
|
+
expect(
|
|
58
|
+
listResult!.some((e) => e.id === freshId),
|
|
59
|
+
driver.describe('agent-memory.md §TTL', `list() MUST include the fresh entry (freshId=${String(freshId)})`),
|
|
60
|
+
).toBe(true);
|
|
61
|
+
expect(
|
|
62
|
+
listResult!.some((e) => e.id === expiredId),
|
|
63
|
+
driver.describe('agent-memory.md §TTL', `list() MUST NOT include the expired entry (expiredId=${String(expiredId)})`),
|
|
64
|
+
).toBe(false);
|
|
47
65
|
});
|
|
48
66
|
});
|
|
@@ -75,6 +75,10 @@ import { pollUntilTerminal } from '../lib/polling.js';
|
|
|
75
75
|
*/
|
|
76
76
|
export const REQUIRES_HOST_CALLBACK = "the host issues MCP JSON-RPC calls to the suite's fake server";
|
|
77
77
|
|
|
78
|
+
// The fixture's node is the conformance-RESERVED `core.conformance.mcp-invoke`
|
|
79
|
+
// (S33, 2026-08-17 — was `core.ai.callPrompt` + `config.mcp`, a vendor pack-tier
|
|
80
|
+
// node that never read that config): a host that consumes MCP maps it to its own
|
|
81
|
+
// bridge; one that does not MUST NOT advertise the fixture.
|
|
78
82
|
const ROUNDTRIP_FIXTURE = 'conformance-mcp-tool-roundtrip';
|
|
79
83
|
|
|
80
84
|
/**
|
|
@@ -188,13 +188,31 @@ describe.skipIf(BEHAVIORAL_SKIP)('multi-agent-confidence-escalation: behavioral
|
|
|
188
188
|
'payload.escalationKind ∈ {clarify, escalate}',
|
|
189
189
|
).toBe(true);
|
|
190
190
|
|
|
191
|
-
// Causation
|
|
192
|
-
//
|
|
193
|
-
|
|
191
|
+
// Causation (S32, 2026-08-17): this leg used to REQUIRE `causationId` → a
|
|
192
|
+
// `runOrchestrator.decided`, which contradicts RFC 0011 §F CP-1 as asserted by
|
|
193
|
+
// orchestratorConservativePath.test.ts — a low-confidence decision is HELD, so
|
|
194
|
+
// no `runOrchestrator.decided` exists on the log before ratification; the
|
|
195
|
+
// escalation event carries the decision verbatim in `originalDecision` for
|
|
196
|
+
// exactly that reason (multi-agent-execution.md §"Confidence escalation").
|
|
197
|
+
// Rule now: `causationId` MAY be absent; when present it MUST resolve to an
|
|
198
|
+
// event already on this run's log (a host that does emit a pre-ratification
|
|
199
|
+
// decided event may point at it; one that honours CP-1 points at nothing or at
|
|
200
|
+
// the preceding node event).
|
|
201
|
+
if (typeof ev.causationId === 'string' && ev.causationId.length > 0) {
|
|
202
|
+
const cause = events.find((e) => e.eventId === ev.causationId);
|
|
203
|
+
expect(
|
|
204
|
+
cause !== undefined,
|
|
205
|
+
driver.describe(
|
|
206
|
+
'spec/v1/multi-agent-execution.md §"Confidence escalation"',
|
|
207
|
+
`confidence-escalated causationId (${ev.causationId}) MUST resolve to an event on this run's log when present`,
|
|
208
|
+
),
|
|
209
|
+
).toBe(true);
|
|
210
|
+
}
|
|
211
|
+
const original = (ev.payload as { originalDecision?: unknown } | undefined)?.originalDecision;
|
|
194
212
|
expect(
|
|
195
|
-
|
|
196
|
-
'confidence-escalated
|
|
197
|
-
).toBe(
|
|
213
|
+
original !== null && typeof original === 'object',
|
|
214
|
+
driver.describe('run-event-payloads.schema.json §confidence-escalated', 'payload.originalDecision carries the escalated OrchestratorDecision verbatim — the decision is on the log HERE, not on a prior decided event'),
|
|
215
|
+
).toBe(true);
|
|
198
216
|
|
|
199
217
|
// Load-bearing: NO dispatch event fired. RFC 0039 gates BEFORE the loop.
|
|
200
218
|
const chainEvents = events.filter((e) => e.type === 'core.workflowChain.event');
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
import { describe, it, expect } from 'vitest';
|
|
34
34
|
import { softSkip } from '../lib/soft-skip.js';
|
|
35
35
|
import { driver } from '../lib/driver.js';
|
|
36
|
+
import { executionModelVersionMax } from '../lib/execution-model-version.js';
|
|
36
37
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
37
38
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
38
39
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
@@ -81,11 +82,14 @@ describe.skipIf(HTTP_SKIP)('multi-agent-handoff-state-machine: advertisement sha
|
|
|
81
82
|
),
|
|
82
83
|
).toBe('number');
|
|
83
84
|
const v = executionModel.version as number;
|
|
85
|
+
// S31 (2026-08-17): ceiling read from capabilities.schema.json (`maximum`, 6 since
|
|
86
|
+
// RFC 0090) instead of a literal that went stale the day version 6 was Accepted.
|
|
87
|
+
const max = executionModelVersionMax();
|
|
84
88
|
expect(
|
|
85
|
-
Number.isInteger(v) && v >= 1 && v <=
|
|
89
|
+
Number.isInteger(v) && v >= 1 && v <= max,
|
|
86
90
|
driver.describe(
|
|
87
91
|
'RFCS/0037-multi-agent-execution-model.md §C',
|
|
88
|
-
|
|
92
|
+
`version MUST be an integer in [1, ${max}] (1 = Phase 1 only; later phases lift the ceiling additively — 5 = RFC 0061 stateful agent-loop lifecycle, 6 = RFC 0090 verifier; the ceiling is \`capabilities.schema.json\` §multiAgent.executionModel.version maximum)`,
|
|
89
93
|
),
|
|
90
94
|
).toBe(true);
|
|
91
95
|
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `node.failed` payload shape — `run-event-payloads.schema.json` §nodeFailed:
|
|
3
|
+
* `{ nodeId, error: { code, message, details?, retryable? }, attempts? }`. The
|
|
4
|
+
* error is an OBJECT (`_errorObject`), not a bare string, and it is REQUIRED.
|
|
5
|
+
*
|
|
6
|
+
* Why this exists (S34, 2026-08-17): nothing on the wire checked it. The second
|
|
7
|
+
* sibling host measured that it — and the reference hosts — emitted
|
|
8
|
+
* `node.failed { error: "<string>", code }` / `{ data: { code } }` for years
|
|
9
|
+
* while the schema said otherwise, and only noticed because a replay scenario
|
|
10
|
+
* happens to read `payload.error.code`. A schema nobody drives is a wish.
|
|
11
|
+
*
|
|
12
|
+
* Fixture-gated on `conformance-failure` (the run that fails at a node); a host
|
|
13
|
+
* that does not advertise the fixture records inapplicable. Every `node.failed`
|
|
14
|
+
* on that run's log MUST validate against §nodeFailed.
|
|
15
|
+
*
|
|
16
|
+
* @see spec/v1/rest-endpoints.md · schemas/run-event-payloads.schema.json
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { describe, it, expect } from 'vitest';
|
|
20
|
+
import { readFileSync } from 'node:fs';
|
|
21
|
+
import { join } from 'node:path';
|
|
22
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
23
|
+
import addFormats from 'ajv-formats';
|
|
24
|
+
import { driver } from '../lib/driver.js';
|
|
25
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
26
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
27
|
+
import { SCHEMAS_DIR } from '../lib/paths.js';
|
|
28
|
+
|
|
29
|
+
const WORKFLOW_ID = 'conformance-failure';
|
|
30
|
+
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
31
|
+
|
|
32
|
+
function nodeFailedValidator() {
|
|
33
|
+
const payloads = JSON.parse(readFileSync(join(SCHEMAS_DIR, 'run-event-payloads.schema.json'), 'utf8')) as {
|
|
34
|
+
$id?: string;
|
|
35
|
+
$defs: Record<string, unknown>;
|
|
36
|
+
};
|
|
37
|
+
const ajv = new Ajv2020({ strict: false, allErrors: true });
|
|
38
|
+
addFormats(ajv);
|
|
39
|
+
const schema = { $schema: 'https://json-schema.org/draft/2020-12/schema', $id: 'urn:openwop:node-failed-leg', $defs: payloads.$defs, $ref: '#/$defs/nodeFailed' };
|
|
40
|
+
return ajv.compile(schema);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
describe.skipIf(SKIP_NO_FIXTURE)('run-event-payloads.schema.json §nodeFailed — node.failed carries a structured error object', () => {
|
|
44
|
+
it('every node.failed on the conformance-failure run validates against §nodeFailed (error: { code, message } is an object, not a string)', async () => {
|
|
45
|
+
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
46
|
+
expect(create.status, driver.describe('rest-endpoints.md', 'POST /v1/runs MUST return 201 for the failing fixture')).toBe(201);
|
|
47
|
+
const runId = (create.json as { runId: string }).runId;
|
|
48
|
+
const terminal = await pollUntilTerminal(runId);
|
|
49
|
+
expect(terminal.status, driver.describe('fixtures.md conformance-failure §Terminal status', 'fixture MUST reach terminal `failed`')).toBe('failed');
|
|
50
|
+
|
|
51
|
+
const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
52
|
+
expect(res.status).toBe(200);
|
|
53
|
+
const events = ((res.json as { events?: Array<{ type: string; payload?: unknown }> } | undefined)?.events ?? []);
|
|
54
|
+
const failed = events.filter((e) => e.type === 'node.failed');
|
|
55
|
+
expect(
|
|
56
|
+
failed.length,
|
|
57
|
+
driver.describe('rest-endpoints.md §Run events', 'a run that fails at a node MUST carry at least one node.failed event on its log'),
|
|
58
|
+
).toBeGreaterThan(0);
|
|
59
|
+
|
|
60
|
+
const validate = nodeFailedValidator();
|
|
61
|
+
for (const ev of failed) {
|
|
62
|
+
const ok = validate(ev.payload);
|
|
63
|
+
expect(
|
|
64
|
+
ok,
|
|
65
|
+
driver.describe(
|
|
66
|
+
'run-event-payloads.schema.json §nodeFailed',
|
|
67
|
+
`node.failed payload MUST validate — error is a REQUIRED object { code, message } (_errorObject), not a bare string or a data.code: ${JSON.stringify(validate.errors)} — payload: ${JSON.stringify(ev.payload).slice(0, 300)}`,
|
|
68
|
+
),
|
|
69
|
+
).toBe(true);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
});
|