@openwop/openwop-conformance 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +241 -0
- package/api/asyncapi.yaml +481 -0
- package/api/openapi.yaml +830 -0
- package/api/redocly.yaml +8 -0
- package/coverage.md +80 -0
- package/dist/cli.js +161 -0
- package/fixtures/conformance-a2a-task-roundtrip.json +27 -0
- package/fixtures/conformance-agent-identity.json +27 -0
- package/fixtures/conformance-agent-low-confidence.json +29 -0
- package/fixtures/conformance-agent-memory-cross-tenant.json +28 -0
- package/fixtures/conformance-agent-memory-redaction.json +32 -0
- package/fixtures/conformance-agent-memory-roundtrip.json +32 -0
- package/fixtures/conformance-agent-memory-ttl.json +31 -0
- package/fixtures/conformance-agent-pack-export.json +26 -0
- package/fixtures/conformance-agent-pack-install.json +26 -0
- package/fixtures/conformance-agent-pack-provenance.json +31 -0
- package/fixtures/conformance-agent-reasoning.json +29 -0
- package/fixtures/conformance-approval.json +27 -0
- package/fixtures/conformance-cancellable.json +33 -0
- package/fixtures/conformance-cap-breach.json +27 -0
- package/fixtures/conformance-capability-missing.json +23 -0
- package/fixtures/conformance-channel-ttl.json +60 -0
- package/fixtures/conformance-clarification.json +30 -0
- package/fixtures/conformance-conversation-capability-negotiation.json +23 -0
- package/fixtures/conformance-conversation-lifecycle.json +32 -0
- package/fixtures/conformance-conversation-replay.json +33 -0
- package/fixtures/conformance-conversation-vs-clarification.json +26 -0
- package/fixtures/conformance-delay.json +33 -0
- package/fixtures/conformance-dispatch-loop.json +38 -0
- package/fixtures/conformance-failure.json +23 -0
- package/fixtures/conformance-idempotent.json +30 -0
- package/fixtures/conformance-identity.json +32 -0
- package/fixtures/conformance-interrupt-auth-required.json +28 -0
- package/fixtures/conformance-interrupt-external-event.json +33 -0
- package/fixtures/conformance-interrupt-parent-child-cancel-child.json +27 -0
- package/fixtures/conformance-interrupt-parent-child-cancel.json +26 -0
- package/fixtures/conformance-interrupt-quorum.json +30 -0
- package/fixtures/conformance-mcp-tool-roundtrip.json +32 -0
- package/fixtures/conformance-message-reducer.json +31 -0
- package/fixtures/conformance-multi-node.json +21 -0
- package/fixtures/conformance-noop.json +23 -0
- package/fixtures/conformance-orchestrator-dispatch.json +47 -0
- package/fixtures/conformance-orchestrator-low-confidence.json +41 -0
- package/fixtures/conformance-orchestrator-terminate.json +44 -0
- package/fixtures/conformance-stream-text.json +26 -0
- package/fixtures/conformance-subworkflow-child.json +21 -0
- package/fixtures/conformance-subworkflow-parent.json +49 -0
- package/fixtures/conformance-version-fold.json +23 -0
- package/fixtures/conformance-wasm-pack-roundtrip.json +25 -0
- package/fixtures/pack-manifests/pack-private-example.json +26 -0
- package/fixtures.md +404 -0
- package/package.json +48 -0
- package/schemas/README.md +75 -0
- package/schemas/agent-manifest.schema.json +107 -0
- package/schemas/agent-ref.schema.json +53 -0
- package/schemas/capabilities.schema.json +287 -0
- package/schemas/channel-written-payload.schema.json +55 -0
- package/schemas/conversation-event.schema.json +120 -0
- package/schemas/conversation-turn.schema.json +72 -0
- package/schemas/debug-bundle.schema.json +196 -0
- package/schemas/dispatch-config.schema.json +46 -0
- package/schemas/error-envelope.schema.json +25 -0
- package/schemas/memory-entry.schema.json +36 -0
- package/schemas/memory-list-options.schema.json +21 -0
- package/schemas/node-pack-manifest.schema.json +235 -0
- package/schemas/orchestrator-decision.schema.json +60 -0
- package/schemas/run-event-payloads.schema.json +663 -0
- package/schemas/run-event.schema.json +116 -0
- package/schemas/run-options.schema.json +81 -0
- package/schemas/run-orchestrator-decided-event.schema.json +20 -0
- package/schemas/run-snapshot.schema.json +121 -0
- package/schemas/suspend-request.schema.json +182 -0
- package/schemas/workflow-definition.schema.json +430 -0
- package/src/cli.ts +187 -0
- package/src/lib/a2a-fake-peer.ts +233 -0
- package/src/lib/canaries.ts +186 -0
- package/src/lib/driver.ts +96 -0
- package/src/lib/env.ts +49 -0
- package/src/lib/fixtures.ts +93 -0
- package/src/lib/mcp-fake-server.ts +185 -0
- package/src/lib/multi-agent-capabilities.ts +155 -0
- package/src/lib/multiProcess.ts +141 -0
- package/src/lib/otel-collector.ts +312 -0
- package/src/lib/paths.ts +198 -0
- package/src/lib/polling.ts +81 -0
- package/src/lib/profiles.ts +258 -0
- package/src/lib/sse.ts +172 -0
- package/src/scenarios/a2a-task-roundtrip.test.ts +149 -0
- package/src/scenarios/agentConfidenceEscalation.test.ts +61 -0
- package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +54 -0
- package/src/scenarios/agentMemoryRedactionContract.test.ts +46 -0
- package/src/scenarios/agentMemoryRoundTrip.test.ts +52 -0
- package/src/scenarios/agentMemoryTtlExpiry.test.ts +47 -0
- package/src/scenarios/agentMessageReducer.test.ts +57 -0
- package/src/scenarios/agentMetadata.test.ts +56 -0
- package/src/scenarios/agentPackExport.test.ts +45 -0
- package/src/scenarios/agentPackInstall.test.ts +50 -0
- package/src/scenarios/agentPackProvenance.test.ts +53 -0
- package/src/scenarios/agentReasoningEvents.test.ts +72 -0
- package/src/scenarios/append-ordering.test.ts +91 -0
- package/src/scenarios/approval-payload.test.ts +120 -0
- package/src/scenarios/audit-log-integrity.test.ts +106 -0
- package/src/scenarios/auth.test.ts +55 -0
- package/src/scenarios/byok-roundtrip.test.ts +166 -0
- package/src/scenarios/cancellation.test.ts +68 -0
- package/src/scenarios/cap-breach.test.ts +149 -0
- package/src/scenarios/channel-ttl.test.ts +70 -0
- package/src/scenarios/configurable-schema.test.ts +76 -0
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +39 -0
- package/src/scenarios/conversationLifecycle.test.ts +64 -0
- package/src/scenarios/conversationReplayDeterminism.test.ts +52 -0
- package/src/scenarios/conversationVsLegacySuspend.test.ts +46 -0
- package/src/scenarios/cost-attribution.test.ts +207 -0
- package/src/scenarios/debugBundle.test.ts +222 -0
- package/src/scenarios/discovery.test.ts +147 -0
- package/src/scenarios/dispatchLoop.test.ts +52 -0
- package/src/scenarios/errors.test.ts +144 -0
- package/src/scenarios/eventOrdering.test.ts +144 -0
- package/src/scenarios/failure-path.test.ts +46 -0
- package/src/scenarios/fixtures-gating.test.ts +137 -0
- package/src/scenarios/fixtures-valid.test.ts +140 -0
- package/src/scenarios/highConcurrency.test.ts +263 -0
- package/src/scenarios/idempotency.test.ts +83 -0
- package/src/scenarios/idempotencyRetry.test.ts +130 -0
- package/src/scenarios/identity-passthrough.test.ts +54 -0
- package/src/scenarios/interrupt-approval.test.ts +97 -0
- package/src/scenarios/interrupt-auth-required-resume.test.ts +88 -0
- package/src/scenarios/interrupt-clarification.test.ts +45 -0
- package/src/scenarios/interrupt-external-event-correlation.test.ts +113 -0
- package/src/scenarios/interrupt-parent-child-cascade.test.ts +102 -0
- package/src/scenarios/interrupt-quorum-resolution.test.ts +97 -0
- package/src/scenarios/interruptRace.test.ts +176 -0
- package/src/scenarios/maliciousManifest.test.ts +154 -0
- package/src/scenarios/mcp-discoverability.test.ts +129 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +149 -0
- package/src/scenarios/multi-node-ordering.test.ts +60 -0
- package/src/scenarios/multi-region-idempotency.test.ts +52 -0
- package/src/scenarios/orchestratorConservativePath.test.ts +63 -0
- package/src/scenarios/orchestratorDispatch.test.ts +66 -0
- package/src/scenarios/orchestratorTermination.test.ts +54 -0
- package/src/scenarios/otel-emission.test.ts +113 -0
- package/src/scenarios/otel-trace-propagation.test.ts +90 -0
- package/src/scenarios/pack-registry-publish.test.ts +93 -0
- package/src/scenarios/pack-registry.test.ts +328 -0
- package/src/scenarios/pause-resume.test.ts +109 -0
- package/src/scenarios/policies.test.ts +162 -0
- package/src/scenarios/profileDerivation.test.ts +335 -0
- package/src/scenarios/providerPolicyEnforcement.test.ts +132 -0
- package/src/scenarios/rate-limit-envelope.test.ts +97 -0
- package/src/scenarios/redaction.test.ts +254 -0
- package/src/scenarios/redactionAdversarial.test.ts +162 -0
- package/src/scenarios/replay-fork-arbitrary.test.ts +347 -0
- package/src/scenarios/replay-fork.test.ts +216 -0
- package/src/scenarios/replayDeterminism.test.ts +171 -0
- package/src/scenarios/route-coverage.test.ts +129 -0
- package/src/scenarios/runs-lifecycle.test.ts +65 -0
- package/src/scenarios/runtime-capabilities.test.ts +118 -0
- package/src/scenarios/spec-corpus-validity.test.ts +1257 -0
- package/src/scenarios/staleClaim.test.ts +223 -0
- package/src/scenarios/stream-modes-buffer.test.ts +148 -0
- package/src/scenarios/stream-modes-mixed.test.ts +149 -0
- package/src/scenarios/stream-modes.test.ts +139 -0
- package/src/scenarios/streamReconnect.test.ts +162 -0
- package/src/scenarios/subworkflow.test.ts +126 -0
- package/src/scenarios/version-negotiation.test.ts +157 -0
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +47 -0
- package/src/scenarios/wasm-pack-invoke-completed.test.ts +69 -0
- package/src/scenarios/wasm-pack-invoke-suspended.test.ts +74 -0
- package/src/scenarios/wasm-pack-load.test.ts +75 -0
- package/src/scenarios/wasm-pack-memory-cap.test.ts +43 -0
- package/src/scenarios/wasm-pack-replay-determinism.test.ts +61 -0
- package/src/scenarios/webhook-sig-algorithm.test.ts +61 -0
- package/src/setup.ts +173 -0
- package/vitest.config.ts +17 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 3 — SR-1 secret-redaction invariant.
|
|
3
|
+
*
|
|
4
|
+
* Verifies the SR-1 normative invariant: when a memory write contains
|
|
5
|
+
* a value the run's BYOK vault resolved during the run, the persisted
|
|
6
|
+
* entry MUST carry `[REDACTED:<secretId>]` in place of the plaintext.
|
|
7
|
+
* Read-back from MemoryAdapter MUST surface the redacted form.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when host doesn't advertise long-term memory.
|
|
10
|
+
* Fixture-gated: requires `conformance-agent-memory-redaction`
|
|
11
|
+
* (resolves a BYOK secret, writes a memory entry containing the
|
|
12
|
+
* plaintext, reads it back).
|
|
13
|
+
*
|
|
14
|
+
* @see docs/MULTI-AGENT-INTEGRATION-GAPS.md §`Phase 3`
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, it, expect } from 'vitest';
|
|
18
|
+
import { driver } from '../lib/driver.js';
|
|
19
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
20
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
21
|
+
import { hasLongTermMemory } from '../lib/multi-agent-capabilities.js';
|
|
22
|
+
|
|
23
|
+
const FIXTURE = 'conformance-agent-memory-redaction';
|
|
24
|
+
const SKIP = !hasLongTermMemory() || !isFixtureAdvertised(FIXTURE);
|
|
25
|
+
|
|
26
|
+
describe.skipIf(SKIP)('agentMemoryRedactionContract: SR-1 invariant', () => {
|
|
27
|
+
it('BYOK plaintext is redacted to [REDACTED:<secretId>] at the read surface', async () => {
|
|
28
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
29
|
+
expect(create.status).toBe(201);
|
|
30
|
+
const runId = (create.json as { runId: string }).runId;
|
|
31
|
+
|
|
32
|
+
const terminal = await pollUntilTerminal(runId);
|
|
33
|
+
expect(terminal.status).toBe('completed');
|
|
34
|
+
|
|
35
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
36
|
+
const body = snap.json as { variables?: Record<string, unknown> };
|
|
37
|
+
|
|
38
|
+
// Fixture convention: writes a memory entry containing a registered
|
|
39
|
+
// BYOK plaintext, then reads it back into `memoryReadback.content`.
|
|
40
|
+
// The read-side value MUST contain the redaction marker, not the
|
|
41
|
+
// plaintext.
|
|
42
|
+
const readback = body.variables?.memoryReadback as { content?: string } | undefined;
|
|
43
|
+
expect(readback).toBeDefined();
|
|
44
|
+
expect(readback!.content).toMatch(/\[REDACTED:[^\]]+\]/);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 3 — MemoryAdapter list/get round-trip.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that a host advertising `capabilities.agents.memoryBackends:
|
|
5
|
+
* ['long-term']` resolves `AgentRef.memoryRef` to MemoryEntry results
|
|
6
|
+
* via its MemoryAdapter, and that the entries conform to
|
|
7
|
+
* `schemas/memory-entry.schema.json`.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when host doesn't advertise long-term memory.
|
|
10
|
+
* Fixture-gated: requires `conformance-agent-memory-roundtrip`.
|
|
11
|
+
*
|
|
12
|
+
* @see schemas/memory-entry.schema.json
|
|
13
|
+
* @see schemas/memory-list-options.schema.json
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it, expect } from 'vitest';
|
|
17
|
+
import { driver } from '../lib/driver.js';
|
|
18
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
19
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
20
|
+
import { hasLongTermMemory } from '../lib/multi-agent-capabilities.js';
|
|
21
|
+
|
|
22
|
+
const FIXTURE = 'conformance-agent-memory-roundtrip';
|
|
23
|
+
const SKIP = !hasLongTermMemory() || !isFixtureAdvertised(FIXTURE);
|
|
24
|
+
|
|
25
|
+
describe.skipIf(SKIP)('agentMemoryRoundTrip: write → read via MemoryAdapter', () => {
|
|
26
|
+
it('memory entries written during a run are readable via the resolved memoryRef', async () => {
|
|
27
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
28
|
+
expect(create.status).toBe(201);
|
|
29
|
+
const runId = (create.json as { runId: string }).runId;
|
|
30
|
+
|
|
31
|
+
const terminal = await pollUntilTerminal(runId);
|
|
32
|
+
expect(terminal.status).toBe('completed');
|
|
33
|
+
|
|
34
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
35
|
+
const body = snap.json as {
|
|
36
|
+
agent?: { memoryRef?: string };
|
|
37
|
+
variables?: Record<string, unknown>;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
// Fixture convention: writes an entry then reads it back into a
|
|
41
|
+
// variable named `memoryReadback`. The variable's value MUST be a
|
|
42
|
+
// MemoryEntry-shaped object per schemas/memory-entry.schema.json.
|
|
43
|
+
const readback = body.variables?.memoryReadback as
|
|
44
|
+
| { id?: string; content?: string; tags?: string[]; createdAt?: string }
|
|
45
|
+
| undefined;
|
|
46
|
+
expect(readback).toBeDefined();
|
|
47
|
+
expect(typeof readback!.id).toBe('string');
|
|
48
|
+
expect(typeof readback!.content).toBe('string');
|
|
49
|
+
expect(Array.isArray(readback!.tags)).toBe(true);
|
|
50
|
+
expect(typeof readback!.createdAt).toBe('string');
|
|
51
|
+
});
|
|
52
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 3 — TTL expiry semantics for MemoryEntry.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that memory entries carrying `expiresAt` in the past are
|
|
5
|
+
* NOT surfaced by `MemoryAdapter.list()` / `get()`. The fixture writes
|
|
6
|
+
* an entry with `expiresAt` set in the past and another set in the
|
|
7
|
+
* future; the read-back surface only includes the future one.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when host doesn't advertise long-term memory.
|
|
10
|
+
* Fixture-gated: requires `conformance-agent-memory-ttl`.
|
|
11
|
+
*
|
|
12
|
+
* @see schemas/memory-entry.schema.json §`expiresAt`
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { describe, it, expect } from 'vitest';
|
|
16
|
+
import { driver } from '../lib/driver.js';
|
|
17
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
18
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
19
|
+
import { hasLongTermMemory } from '../lib/multi-agent-capabilities.js';
|
|
20
|
+
|
|
21
|
+
const FIXTURE = 'conformance-agent-memory-ttl';
|
|
22
|
+
const SKIP = !hasLongTermMemory() || !isFixtureAdvertised(FIXTURE);
|
|
23
|
+
|
|
24
|
+
describe.skipIf(SKIP)('agentMemoryTtlExpiry: expired entries are excluded from list/get', () => {
|
|
25
|
+
it('list() excludes entries whose expiresAt is in the past', async () => {
|
|
26
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
27
|
+
expect(create.status).toBe(201);
|
|
28
|
+
const runId = (create.json as { runId: string }).runId;
|
|
29
|
+
|
|
30
|
+
const terminal = await pollUntilTerminal(runId);
|
|
31
|
+
expect(terminal.status).toBe('completed');
|
|
32
|
+
|
|
33
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
34
|
+
const body = snap.json as { variables?: Record<string, unknown> };
|
|
35
|
+
|
|
36
|
+
// Fixture writes one expired + one fresh entry, then reads via list().
|
|
37
|
+
// Result MUST include only the fresh entry.
|
|
38
|
+
const listResult = body.variables?.memoryList as Array<{ id?: string; expiresAt?: string }> | undefined;
|
|
39
|
+
expect(Array.isArray(listResult)).toBe(true);
|
|
40
|
+
const now = Date.now();
|
|
41
|
+
for (const e of listResult!) {
|
|
42
|
+
if (e.expiresAt) {
|
|
43
|
+
expect(new Date(e.expiresAt).getTime()).toBeGreaterThan(now);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 1 — `message` reducer idempotency invariant.
|
|
3
|
+
*
|
|
4
|
+
* Verifies the canonical `message` reducer's contract from
|
|
5
|
+
* `spec/v1/channels-and-reducers.md` §`message`:
|
|
6
|
+
* - Append-only — new messages land at the end of the list.
|
|
7
|
+
* - Idempotent on `messageId` — a duplicate emission folds to a
|
|
8
|
+
* single entry.
|
|
9
|
+
* - Replay-deterministic — the same event sequence produces the
|
|
10
|
+
* same final channel value.
|
|
11
|
+
*
|
|
12
|
+
* Capability-gated: skips when host doesn't advertise
|
|
13
|
+
* `capabilities.agents.supported: true`. Fixture-gated: requires
|
|
14
|
+
* `conformance-message-reducer` (intentionally emits duplicate-id
|
|
15
|
+
* messages to exercise idempotency).
|
|
16
|
+
*
|
|
17
|
+
* @see spec/v1/channels-and-reducers.md §`message`
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, it, expect } from 'vitest';
|
|
21
|
+
import { driver } from '../lib/driver.js';
|
|
22
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
23
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
24
|
+
import { isAgentSupported } from '../lib/multi-agent-capabilities.js';
|
|
25
|
+
|
|
26
|
+
const FIXTURE = 'conformance-message-reducer';
|
|
27
|
+
const SKIP = !isAgentSupported() || !isFixtureAdvertised(FIXTURE);
|
|
28
|
+
|
|
29
|
+
describe.skipIf(SKIP)('agentMessageReducer: message reducer idempotency + append-only invariant', () => {
|
|
30
|
+
it('duplicate messageId emissions fold to a single channel entry', async () => {
|
|
31
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
32
|
+
expect(create.status).toBe(201);
|
|
33
|
+
const runId = (create.json as { runId: string }).runId;
|
|
34
|
+
|
|
35
|
+
const terminal = await pollUntilTerminal(runId);
|
|
36
|
+
expect(terminal.status).toBe('completed');
|
|
37
|
+
|
|
38
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
39
|
+
const body = snap.json as { channels?: Record<string, unknown> };
|
|
40
|
+
expect(body.channels).toBeDefined();
|
|
41
|
+
|
|
42
|
+
// Find any channel using the `message` reducer (host names vary;
|
|
43
|
+
// fixture convention is `conversation` or `messages`).
|
|
44
|
+
const channelEntries = Object.entries(body.channels ?? {});
|
|
45
|
+
const messageChannel = channelEntries.find(([, val]) => Array.isArray(val));
|
|
46
|
+
expect(messageChannel, 'fixture MUST expose at least one message-reducer channel').toBeDefined();
|
|
47
|
+
|
|
48
|
+
const messages = messageChannel![1] as Array<{ messageId?: string }>;
|
|
49
|
+
const messageIds = messages
|
|
50
|
+
.map((m) => m.messageId)
|
|
51
|
+
.filter((id): id is string => typeof id === 'string');
|
|
52
|
+
|
|
53
|
+
// Each messageId appears at most once (idempotency).
|
|
54
|
+
const uniq = new Set(messageIds);
|
|
55
|
+
expect(uniq.size).toBe(messageIds.length);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 1 — agent identity (RunSnapshot.agent / runOrchestrator).
|
|
3
|
+
*
|
|
4
|
+
* Verifies:
|
|
5
|
+
* 1. Hosts advertising `capabilities.agents.supported: true` populate
|
|
6
|
+
* `RunSnapshot.agent` and/or `RunSnapshot.runOrchestrator` with
|
|
7
|
+
* AgentRef-shaped values when a run carries agent provenance.
|
|
8
|
+
* 2. The AgentRef shape conforms to `schemas/agent-ref.schema.json`
|
|
9
|
+
* (required `agentId`; optional `name`, `modelClass`, `memoryRef`,
|
|
10
|
+
* `version`, `sourceManifestId`).
|
|
11
|
+
*
|
|
12
|
+
* Capability-gated: skips when host doesn't advertise
|
|
13
|
+
* `capabilities.agents.supported: true`. Fixture-gated: requires
|
|
14
|
+
* `conformance-agent-identity` advertised in `capabilities.fixtures`.
|
|
15
|
+
*
|
|
16
|
+
* @see spec/v1/capabilities.md §`agents`
|
|
17
|
+
* @see schemas/agent-ref.schema.json
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, it, expect } from 'vitest';
|
|
21
|
+
import { driver } from '../lib/driver.js';
|
|
22
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
23
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
24
|
+
import { isAgentSupported } from '../lib/multi-agent-capabilities.js';
|
|
25
|
+
|
|
26
|
+
const FIXTURE = 'conformance-agent-identity';
|
|
27
|
+
const SKIP = !isAgentSupported() || !isFixtureAdvertised(FIXTURE);
|
|
28
|
+
|
|
29
|
+
describe.skipIf(SKIP)('agentMetadata: RunSnapshot.agent + runOrchestrator wire-shape', () => {
|
|
30
|
+
it('host populates AgentRef-shaped agent identity on RunSnapshot', async () => {
|
|
31
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
32
|
+
expect(create.status).toBe(201);
|
|
33
|
+
const runId = (create.json as { runId: string }).runId;
|
|
34
|
+
|
|
35
|
+
const terminal = await pollUntilTerminal(runId);
|
|
36
|
+
expect(['completed', 'failed']).toContain(terminal.status);
|
|
37
|
+
|
|
38
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
39
|
+
expect(snap.status).toBe(200);
|
|
40
|
+
const body = snap.json as { agent?: { agentId?: string }; runOrchestrator?: { agentId?: string } };
|
|
41
|
+
|
|
42
|
+
const hasAgent = body.agent && typeof body.agent.agentId === 'string';
|
|
43
|
+
const hasOrch = body.runOrchestrator && typeof body.runOrchestrator.agentId === 'string';
|
|
44
|
+
expect(
|
|
45
|
+
hasAgent || hasOrch,
|
|
46
|
+
'fixture-runs MUST populate at least one of RunSnapshot.agent / runOrchestrator',
|
|
47
|
+
).toBe(true);
|
|
48
|
+
|
|
49
|
+
if (hasAgent) {
|
|
50
|
+
expect(body.agent!.agentId!.length).toBeGreaterThanOrEqual(3);
|
|
51
|
+
}
|
|
52
|
+
if (hasOrch) {
|
|
53
|
+
expect(body.runOrchestrator!.agentId!.length).toBeGreaterThanOrEqual(3);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 2 — agent-pack export round-trips workspace agents → AgentManifest.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that a host's workspace-scoped agent registry can project
|
|
5
|
+
* agents into the canonical AgentManifest shape for export/distribution.
|
|
6
|
+
* Round-trip: install pack → workspace gets agents → export workspace
|
|
7
|
+
* yields a manifest set that re-installs cleanly.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when host doesn't advertise
|
|
10
|
+
* `capabilities.agents.supported: true`. Fixture-gated: requires
|
|
11
|
+
* `conformance-agent-pack-export`.
|
|
12
|
+
*
|
|
13
|
+
* @see schemas/agent-manifest.schema.json
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it, expect } from 'vitest';
|
|
17
|
+
import { driver } from '../lib/driver.js';
|
|
18
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
19
|
+
import { isAgentSupported } from '../lib/multi-agent-capabilities.js';
|
|
20
|
+
|
|
21
|
+
const FIXTURE = 'conformance-agent-pack-export';
|
|
22
|
+
const SKIP = !isAgentSupported() || !isFixtureAdvertised(FIXTURE);
|
|
23
|
+
|
|
24
|
+
describe.skipIf(SKIP)('agentPackExport: workspace agents project to AgentManifest', () => {
|
|
25
|
+
it('exported manifests contain required AgentManifest fields', async () => {
|
|
26
|
+
const res = await driver.get('/v1/packs/export');
|
|
27
|
+
if (res.status === 404 || res.status === 501) {
|
|
28
|
+
// Host doesn't expose pack-export over REST; treated as skip.
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
expect(res.status).toBe(200);
|
|
32
|
+
|
|
33
|
+
const body = res.json as {
|
|
34
|
+
manifests?: Array<{ agentId?: string; modelClass?: string; sourceManifestId?: string }>;
|
|
35
|
+
};
|
|
36
|
+
const manifests = body.manifests ?? [];
|
|
37
|
+
expect(manifests.length).toBeGreaterThan(0);
|
|
38
|
+
|
|
39
|
+
for (const m of manifests) {
|
|
40
|
+
expect(typeof m.agentId).toBe('string');
|
|
41
|
+
// Exported manifests SHOULD carry sourceManifestId provenance when
|
|
42
|
+
// they originated from a prior install (covered by agentPackProvenance).
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 2 — agent-pack install registers AgentManifest entries.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that a pack containing an `agents[]` array surfaces those
|
|
5
|
+
* agent manifests via the host's pack registry. The wire-shape contract
|
|
6
|
+
* for AgentManifest entries is `schemas/agent-manifest.schema.json`.
|
|
7
|
+
*
|
|
8
|
+
* Capability-gated: skips when host doesn't advertise either
|
|
9
|
+
* `capabilities.agents.supported: true` OR the pack-registry surface
|
|
10
|
+
* (registry-operations.md). Fixture-gated: requires
|
|
11
|
+
* `conformance-agent-pack-install` advertised.
|
|
12
|
+
*
|
|
13
|
+
* @see schemas/agent-manifest.schema.json
|
|
14
|
+
* @see spec/v1/node-packs.md §`Manifest format`
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, it, expect } from 'vitest';
|
|
18
|
+
import { driver } from '../lib/driver.js';
|
|
19
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
20
|
+
import { isAgentSupported } from '../lib/multi-agent-capabilities.js';
|
|
21
|
+
|
|
22
|
+
const FIXTURE = 'conformance-agent-pack-install';
|
|
23
|
+
const SKIP = !isAgentSupported() || !isFixtureAdvertised(FIXTURE);
|
|
24
|
+
|
|
25
|
+
describe.skipIf(SKIP)('agentPackInstall: pack agents[] entries surface as AgentManifest', () => {
|
|
26
|
+
it('host exposes installed agent manifests with required AgentManifest fields', async () => {
|
|
27
|
+
// Host-specific pack-listing endpoint. The conformance suite probes
|
|
28
|
+
// common paths; hosts that don't expose pack listings via REST mark
|
|
29
|
+
// this scenario as skip via their own capability advertisement.
|
|
30
|
+
const res = await driver.get('/v1/packs');
|
|
31
|
+
if (res.status === 404 || res.status === 501) {
|
|
32
|
+
// Host doesn't expose pack registry over REST; scenario assertion is
|
|
33
|
+
// skipped (capability surface is host-internal).
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
expect(res.status).toBe(200);
|
|
37
|
+
|
|
38
|
+
const body = res.json as {
|
|
39
|
+
packs?: Array<{ agents?: Array<{ agentId?: string; modelClass?: string }> }>;
|
|
40
|
+
};
|
|
41
|
+
const packs = body.packs ?? [];
|
|
42
|
+
const allAgents = packs.flatMap((p) => p.agents ?? []);
|
|
43
|
+
|
|
44
|
+
expect(allAgents.length).toBeGreaterThan(0);
|
|
45
|
+
for (const a of allAgents) {
|
|
46
|
+
expect(typeof a.agentId).toBe('string');
|
|
47
|
+
expect(a.agentId!.length).toBeGreaterThanOrEqual(3);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 2 — sourceManifestId provenance round-trip.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that when a workspace agent originates from a pack install,
|
|
5
|
+
* the agent's runtime AgentRef AND the exported AgentManifest both
|
|
6
|
+
* carry the original `sourceManifestId` so audit consumers can trace
|
|
7
|
+
* agents back to their distribution source.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when host doesn't advertise
|
|
10
|
+
* `capabilities.agents.supported: true`. Fixture-gated: requires
|
|
11
|
+
* `conformance-agent-pack-provenance`.
|
|
12
|
+
*
|
|
13
|
+
* @see schemas/agent-ref.schema.json §`sourceManifestId`
|
|
14
|
+
* @see schemas/agent-manifest.schema.json §`sourceManifestId`
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, it, expect } from 'vitest';
|
|
18
|
+
import { driver } from '../lib/driver.js';
|
|
19
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
20
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
21
|
+
import { isAgentSupported } from '../lib/multi-agent-capabilities.js';
|
|
22
|
+
|
|
23
|
+
const FIXTURE = 'conformance-agent-pack-provenance';
|
|
24
|
+
const SKIP = !isAgentSupported() || !isFixtureAdvertised(FIXTURE);
|
|
25
|
+
|
|
26
|
+
describe.skipIf(SKIP)('agentPackProvenance: sourceManifestId survives install + run', () => {
|
|
27
|
+
it('run-level AgentRef carries sourceManifestId pointing back to the install source', async () => {
|
|
28
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
29
|
+
expect(create.status).toBe(201);
|
|
30
|
+
const runId = (create.json as { runId: string }).runId;
|
|
31
|
+
|
|
32
|
+
await pollUntilTerminal(runId);
|
|
33
|
+
|
|
34
|
+
const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
35
|
+
const body = snap.json as {
|
|
36
|
+
agent?: { agentId?: string; sourceManifestId?: string };
|
|
37
|
+
runOrchestrator?: { agentId?: string; sourceManifestId?: string };
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const refs = [body.agent, body.runOrchestrator].filter(
|
|
41
|
+
(r): r is { agentId?: string; sourceManifestId?: string } => Boolean(r),
|
|
42
|
+
);
|
|
43
|
+
expect(refs.length).toBeGreaterThan(0);
|
|
44
|
+
|
|
45
|
+
// At least one of the run's AgentRefs MUST carry sourceManifestId
|
|
46
|
+
// when the fixture's agent originated from a pack install.
|
|
47
|
+
const withProvenance = refs.filter((r) => typeof r.sourceManifestId === 'string');
|
|
48
|
+
expect(
|
|
49
|
+
withProvenance.length,
|
|
50
|
+
'pack-installed agents MUST surface sourceManifestId on their runtime AgentRef projection',
|
|
51
|
+
).toBeGreaterThan(0);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-Agent Shift Phase 1 — agent.* reasoning event family.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that hosts emit the canonical `agent.*` event types per
|
|
5
|
+
* `run-event.schema.json` + per-event payload contract in
|
|
6
|
+
* `run-event-payloads.schema.json` §`agentReasoned` / `agentToolCalled` /
|
|
7
|
+
* `agentToolReturned` / `agentHandoff` / `agentDecided`.
|
|
8
|
+
*
|
|
9
|
+
* Capability-gated: skips when host doesn't advertise
|
|
10
|
+
* `capabilities.agents.supported: true` OR when reasoning verbosity
|
|
11
|
+
* is `'off'` (no `agent.reasoned` events expected).
|
|
12
|
+
*
|
|
13
|
+
* @see schemas/run-event-payloads.schema.json
|
|
14
|
+
* @see spec/v1/capabilities.md §`agents.reasoning`
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, it, expect } from 'vitest';
|
|
18
|
+
import { driver } from '../lib/driver.js';
|
|
19
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
20
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
21
|
+
import {
|
|
22
|
+
isAgentSupported,
|
|
23
|
+
getReasoningVerbosity,
|
|
24
|
+
} from '../lib/multi-agent-capabilities.js';
|
|
25
|
+
|
|
26
|
+
const FIXTURE = 'conformance-agent-reasoning';
|
|
27
|
+
const SKIP =
|
|
28
|
+
!isAgentSupported() ||
|
|
29
|
+
getReasoningVerbosity() === 'off' ||
|
|
30
|
+
!isFixtureAdvertised(FIXTURE);
|
|
31
|
+
|
|
32
|
+
const REASONING_EVENT_TYPES = new Set([
|
|
33
|
+
'agent.reasoned',
|
|
34
|
+
'agent.toolCalled',
|
|
35
|
+
'agent.toolReturned',
|
|
36
|
+
'agent.handoff',
|
|
37
|
+
'agent.decided',
|
|
38
|
+
]);
|
|
39
|
+
|
|
40
|
+
describe.skipIf(SKIP)('agentReasoningEvents: agent.* event family emission', () => {
|
|
41
|
+
it('host emits at least one canonical agent.* event during a reasoning-fixture run', async () => {
|
|
42
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
43
|
+
expect(create.status).toBe(201);
|
|
44
|
+
const runId = (create.json as { runId: string }).runId;
|
|
45
|
+
|
|
46
|
+
await pollUntilTerminal(runId);
|
|
47
|
+
|
|
48
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
49
|
+
expect(events.status).toBe(200);
|
|
50
|
+
const list = (events.json as { events?: Array<{ type: string; payload?: Record<string, unknown> }> })
|
|
51
|
+
.events ?? [];
|
|
52
|
+
|
|
53
|
+
const agentEvents = list.filter((e) => REASONING_EVENT_TYPES.has(e.type));
|
|
54
|
+
expect(agentEvents.length).toBeGreaterThan(0);
|
|
55
|
+
|
|
56
|
+
// Every agent.* event payload MUST carry `agentId` (per RFC 0002 §C).
|
|
57
|
+
for (const ev of agentEvents) {
|
|
58
|
+
expect(typeof ev.payload?.agentId).toBe('string');
|
|
59
|
+
expect((ev.payload!.agentId as string).length).toBeGreaterThanOrEqual(3);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// agent.toolCalled / agent.toolReturned MUST share a `callId` correlation.
|
|
63
|
+
const calls = agentEvents.filter((e) => e.type === 'agent.toolCalled');
|
|
64
|
+
const returns = agentEvents.filter((e) => e.type === 'agent.toolReturned');
|
|
65
|
+
for (const ret of returns) {
|
|
66
|
+
const callId = ret.payload?.callId as string | undefined;
|
|
67
|
+
if (callId === undefined) continue;
|
|
68
|
+
const matched = calls.find((c) => c.payload?.callId === callId);
|
|
69
|
+
expect(matched, `agent.toolReturned.callId=${callId} MUST pair with a prior agent.toolCalled`).toBeDefined();
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
});
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Track 13: append-reducer ordering (channels-and-reducers.md v1.1).
|
|
3
|
+
*
|
|
4
|
+
* Verifies the intra-engine total-order rule: for the `append` reducer
|
|
5
|
+
* (and its bounded variants `votes`/`feedback`), the folded array MUST
|
|
6
|
+
* reflect the per-run `sequence` order of the backing `channel.written`
|
|
7
|
+
* events. Replays MUST NOT reorder.
|
|
8
|
+
*
|
|
9
|
+
* Capability gating: skips unless a host advertises a fixture that
|
|
10
|
+
* writes to an append-reducer channel. `conformance-multi-node` is the
|
|
11
|
+
* primary candidate; hosts MAY seed a dedicated
|
|
12
|
+
* `conformance-append-ordering` fixture.
|
|
13
|
+
*
|
|
14
|
+
* @see spec/v1/channels-and-reducers.md §"Append ordering"
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, it, expect } from 'vitest';
|
|
18
|
+
import { driver } from '../lib/driver.js';
|
|
19
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
20
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
21
|
+
|
|
22
|
+
const CANDIDATES = ['conformance-append-ordering', 'conformance-multi-node'] as const;
|
|
23
|
+
const FIXTURE = CANDIDATES.find((id) => isFixtureAdvertised(id)) ?? null;
|
|
24
|
+
const SKIP = !FIXTURE;
|
|
25
|
+
|
|
26
|
+
interface ChannelWrittenPayload {
|
|
27
|
+
channel?: string;
|
|
28
|
+
value?: unknown;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
interface RunEvent {
|
|
32
|
+
type: string;
|
|
33
|
+
sequence: number;
|
|
34
|
+
payload?: ChannelWrittenPayload;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
describe.skipIf(SKIP)('append-ordering: folded channel reflects event sequence', () => {
|
|
38
|
+
it('append-reducer channels project entries in event-sequence order', async () => {
|
|
39
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE! });
|
|
40
|
+
expect(create.status).toBe(201);
|
|
41
|
+
const runId = (create.json as { runId: string }).runId;
|
|
42
|
+
|
|
43
|
+
await pollUntilTerminal(runId, { timeoutMs: 30_000 });
|
|
44
|
+
|
|
45
|
+
const events = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
46
|
+
const list = (events.json as { events?: RunEvent[] }).events ?? [];
|
|
47
|
+
|
|
48
|
+
// Group channel.written events by channel.
|
|
49
|
+
const byChannel = new Map<string, RunEvent[]>();
|
|
50
|
+
for (const e of list) {
|
|
51
|
+
if (e.type !== 'channel.written') continue;
|
|
52
|
+
const ch = e.payload?.channel;
|
|
53
|
+
if (typeof ch !== 'string') continue;
|
|
54
|
+
const arr = byChannel.get(ch) ?? [];
|
|
55
|
+
arr.push(e);
|
|
56
|
+
byChannel.set(ch, arr);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (byChannel.size === 0) {
|
|
60
|
+
// eslint-disable-next-line no-console
|
|
61
|
+
console.warn(
|
|
62
|
+
'[append-ordering] fixture emitted no channel.written events; skipping ordering assertions',
|
|
63
|
+
);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// For each channel, sequence MUST be strictly increasing within the run.
|
|
68
|
+
for (const [channel, writes] of byChannel) {
|
|
69
|
+
for (let i = 1; i < writes.length; i++) {
|
|
70
|
+
expect(writes[i].sequence, driver.describe(
|
|
71
|
+
'channels-and-reducers.md §"Append ordering"',
|
|
72
|
+
`channel '${channel}': channel.written events MUST appear in sequence order in the event log`,
|
|
73
|
+
)).toBeGreaterThan(writes[i - 1].sequence);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Cross-check against the projected channel state on the run snapshot
|
|
78
|
+
// (when surfaced) — projected array length MUST equal the number of writes.
|
|
79
|
+
const snapshot = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
80
|
+
const channels = (snapshot.json as { channels?: Record<string, unknown> }).channels ?? {};
|
|
81
|
+
for (const [channel, writes] of byChannel) {
|
|
82
|
+
const projected = (channels as Record<string, unknown>)[channel];
|
|
83
|
+
if (Array.isArray(projected)) {
|
|
84
|
+
expect(projected.length, driver.describe(
|
|
85
|
+
'channels-and-reducers.md §"Append ordering"',
|
|
86
|
+
`channel '${channel}': projected array length MUST equal #channel.written events`,
|
|
87
|
+
)).toBe(writes.length);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
});
|