@openwop/openwop-conformance 1.6.1 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +2 -2
- package/api/asyncapi.yaml +127 -0
- package/api/openapi.yaml +518 -1
- package/coverage.md +44 -2
- package/fixtures/conformance-run-duration-breach.json +33 -0
- package/fixtures/oauth-providers/synthetic.json +38 -0
- package/fixtures.md +29 -0
- package/package.json +1 -1
- package/schemas/README.md +22 -0
- package/schemas/agent-deployment-transition.schema.json +49 -0
- package/schemas/agent-deployment.schema.json +54 -0
- package/schemas/agent-eval-suite.schema.json +140 -0
- package/schemas/agent-inventory-response.schema.json +115 -0
- package/schemas/agent-manifest.schema.json +5 -0
- package/schemas/agent-org-chart.schema.json +82 -0
- package/schemas/agent-ref.schema.json +12 -2
- package/schemas/agent-roster-entry.schema.json +81 -0
- package/schemas/agent-roster-response.schema.json +21 -0
- package/schemas/ai-envelope.schema.json +28 -0
- package/schemas/artifact-type-pack-manifest.schema.json +160 -0
- package/schemas/budget-policy.schema.json +18 -0
- package/schemas/capabilities.schema.json +448 -4
- package/schemas/chat-card-pack-manifest.schema.json +158 -0
- package/schemas/credential-provenance.schema.json +18 -0
- package/schemas/envelopes/media.audio.schema.json +38 -0
- package/schemas/envelopes/media.file.schema.json +37 -0
- package/schemas/envelopes/media.image.schema.json +33 -0
- package/schemas/eval-summary.schema.json +92 -0
- package/schemas/heartbeat-evaluated.schema.json +14 -0
- package/schemas/heartbeat-state-changed.schema.json +14 -0
- package/schemas/node-pack-manifest.schema.json +33 -1
- package/schemas/org-chart-responsibility-view.schema.json +26 -0
- package/schemas/run-event-payloads.schema.json +380 -6
- package/schemas/run-event.schema.json +23 -0
- package/schemas/tool-descriptor.schema.json +63 -0
- package/schemas/trigger-subscription.schema.json +26 -0
- package/schemas/workflow-definition.schema.json +5 -0
- package/schemas/workspace-file-create.schema.json +20 -0
- package/schemas/workspace-file.schema.json +39 -0
- package/src/lib/agentLoop.ts +44 -0
- package/src/lib/agentRoster.ts +76 -0
- package/src/lib/agentRuntime.ts +45 -0
- package/src/lib/artifactTypes.ts +96 -0
- package/src/lib/cardPacks.ts +52 -0
- package/src/lib/discovery-capabilities.ts +50 -0
- package/src/lib/distillation.ts +38 -0
- package/src/lib/feedback.ts +3 -3
- package/src/lib/heartbeat.ts +31 -0
- package/src/lib/liveRuntime.ts +59 -0
- package/src/lib/memoryAttribution.ts +48 -0
- package/src/lib/profiles.ts +157 -0
- package/src/lib/runtimeRequires.ts +38 -0
- package/src/lib/safeFetch.ts +87 -0
- package/src/lib/subRunAttestation.ts +35 -0
- package/src/lib/toolHooks.ts +33 -0
- package/src/scenarios/agent-deployment-shape.test.ts +139 -0
- package/src/scenarios/agent-eval-suite-shape.test.ts +167 -0
- package/src/scenarios/agent-live-allowlist-enforced.test.ts +53 -0
- package/src/scenarios/agent-live-invocation-bracket.test.ts +98 -0
- package/src/scenarios/agent-live-runtime-shape.test.ts +98 -0
- package/src/scenarios/agent-live-structured-output.test.ts +58 -0
- package/src/scenarios/agent-loop-iteration-monotonic.test.ts +33 -0
- package/src/scenarios/agent-loop-stateful-resume.test.ts +28 -0
- package/src/scenarios/agent-loop-version5-shape.test.ts +41 -0
- package/src/scenarios/agent-loop-workspace-snapshot.test.ts +33 -0
- package/src/scenarios/agent-manifest-runtime.test.ts +85 -0
- package/src/scenarios/agent-org-chart-shape.test.ts +127 -0
- package/src/scenarios/agent-platform-profile.test.ts +158 -0
- package/src/scenarios/agent-roster-attribution.test.ts +179 -0
- package/src/scenarios/agent-roster-shape.test.ts +146 -0
- package/src/scenarios/ai-envelope-shape.test.ts +14 -18
- package/src/scenarios/aiEnvelope.capBreached.test.ts +2 -1
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +2 -1
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +2 -1
- package/src/scenarios/approval-gate-flow.test.ts +4 -6
- package/src/scenarios/artifact-schema-compile-bounded.test.ts +126 -0
- package/src/scenarios/artifact-type-pack-install.test.ts +78 -0
- package/src/scenarios/artifact-type-pack-manifest-validation.test.ts +140 -0
- package/src/scenarios/artifact-type-store-without-render.test.ts +54 -0
- package/src/scenarios/audit-log-integrity.test.ts +3 -2
- package/src/scenarios/auth-api-key-rotation.test.ts +2 -1
- package/src/scenarios/auth-mtls.test.ts +2 -1
- package/src/scenarios/auth-oauth2-client-credentials.test.ts +2 -1
- package/src/scenarios/auth-oidc-user-bearer.test.ts +2 -1
- package/src/scenarios/auth-saml-profile.test.ts +2 -1
- package/src/scenarios/auth-scim-profile.test.ts +2 -1
- package/src/scenarios/authorization-fail-closed.test.ts +2 -1
- package/src/scenarios/authorization-roles-shape.test.ts +2 -1
- package/src/scenarios/budget-policy-shape.test.ts +136 -0
- package/src/scenarios/byok-auth-modes.test.ts +141 -0
- package/src/scenarios/chat-card-pack-execution.test.ts +56 -0
- package/src/scenarios/chat-card-pack-manifest-validation.test.ts +128 -0
- package/src/scenarios/commitment-fired.test.ts +83 -0
- package/src/scenarios/credential-payload-redaction.test.ts +2 -1
- package/src/scenarios/credentials-capability-shape.test.ts +2 -1
- package/src/scenarios/cross-engine-append-ordering.test.ts +2 -1
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +3 -2
- package/src/scenarios/cross-host-causation-shape.test.ts +3 -2
- package/src/scenarios/deadletter-capability-shape.test.ts +2 -1
- package/src/scenarios/deadletter-retry-exhaustion.test.ts +2 -1
- package/src/scenarios/distillation-index-roundtrip.test.ts +35 -0
- package/src/scenarios/distillation-secret-carryforward.test.ts +35 -0
- package/src/scenarios/distillation-shape.test.ts +41 -0
- package/src/scenarios/distillation-stable-archive.test.ts +37 -0
- package/src/scenarios/distillation-token-budget.test.ts +45 -0
- package/src/scenarios/egress-provenance-shape.test.ts +137 -0
- package/src/scenarios/envelope-completion-distinguishes-truncation.test.ts +4 -3
- package/src/scenarios/envelope-reasoning-secret-redaction.test.ts +5 -4
- package/src/scenarios/envelope-reasoning-shape.test.ts +3 -2
- package/src/scenarios/envelope-refusal-shape.test.ts +3 -2
- package/src/scenarios/envelope-rendering-hint.test.ts +95 -0
- package/src/scenarios/envelope-retry-attempted.test.ts +2 -1
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +3 -2
- package/src/scenarios/exec-not-protocol-tier.test.ts +137 -0
- package/src/scenarios/experimental-tier-shape.test.ts +5 -4
- package/src/scenarios/fs-path-traversal.test.ts +2 -1
- package/src/scenarios/heartbeat-capability-shape.test.ts +35 -0
- package/src/scenarios/heartbeat-fires-once-per-tick.test.ts +28 -0
- package/src/scenarios/heartbeat-idempotent-no-spam.test.ts +43 -0
- package/src/scenarios/heartbeat-runtime-bound.test.ts +30 -0
- package/src/scenarios/http-client-ssrf.test.ts +10 -13
- package/src/scenarios/mcp-toolcall-redaction.test.ts +3 -2
- package/src/scenarios/media-url-inline-cap.test.ts +167 -0
- package/src/scenarios/memory-attribution-emits-on-write.test.ts +54 -0
- package/src/scenarios/memory-attribution-no-content.test.ts +45 -0
- package/src/scenarios/memory-attribution-replay-stable.test.ts +60 -0
- package/src/scenarios/memory-attribution-shape.test.ts +28 -0
- package/src/scenarios/memory-attribution-tenant-scoped.test.ts +44 -0
- package/src/scenarios/memory-capability-model-shape.test.ts +186 -0
- package/src/scenarios/memory-compaction-event-emitted.test.ts +2 -1
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +2 -1
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +2 -1
- package/src/scenarios/memory-consolidation-idempotent.test.ts +77 -0
- package/src/scenarios/memory-consolidation-shape.test.ts +90 -0
- package/src/scenarios/model-capability-substituted.test.ts +2 -1
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +5 -4
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +6 -5
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +4 -3
- package/src/scenarios/multi-region-idempotency.test.ts +10 -10
- package/src/scenarios/oauth-authorization-code-roundtrip.test.ts +145 -0
- package/src/scenarios/oauth-capability-shape.test.ts +2 -1
- package/src/scenarios/oauth-connector-redaction.test.ts +2 -1
- package/src/scenarios/pause-resume.test.ts +3 -3
- package/src/scenarios/production-backpressure.test.ts +2 -2
- package/src/scenarios/production-retention-expiry.test.ts +2 -2
- package/src/scenarios/prompt-all-four-kinds-events.test.ts +2 -1
- package/src/scenarios/prompt-composed-secret-redaction.test.ts +2 -1
- package/src/scenarios/prompt-composed-trust-marker.test.ts +2 -1
- package/src/scenarios/prompt-end-to-end-events.test.ts +2 -1
- package/src/scenarios/prompt-list-and-fetch.test.ts +2 -1
- package/src/scenarios/prompt-mutable-lifecycle.test.ts +2 -1
- package/src/scenarios/prompt-mutation-workspace-membership-enforced.test.ts +2 -1
- package/src/scenarios/prompt-pack-install.test.ts +2 -1
- package/src/scenarios/prompt-read-workspace-membership-enforced.test.ts +2 -1
- package/src/scenarios/prompt-render-deterministic.test.ts +2 -1
- package/src/scenarios/prompt-resolution-chain-agent-intrinsic.test.ts +2 -1
- package/src/scenarios/prompt-resolution-chain-fallback-cascade.test.ts +2 -1
- package/src/scenarios/prompt-resolution-chain-node-wins.test.ts +2 -1
- package/src/scenarios/prompt-template-shape.test.ts +2 -1
- package/src/scenarios/provider-usage.test.ts +2 -1
- package/src/scenarios/replay-divergence-at-refusal.test.ts +4 -3
- package/src/scenarios/replay-fork-arbitrary.test.ts +3 -1
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +2 -1
- package/src/scenarios/replayDeterminism.test.ts +3 -1
- package/src/scenarios/run-execution-bounds-shape.test.ts +133 -0
- package/src/scenarios/runtime-requires-install-gate.test.ts +92 -0
- package/src/scenarios/runtime-requires-shape.test.ts +134 -0
- package/src/scenarios/safefetch-behavior.test.ts +99 -0
- package/src/scenarios/safefetch-live-audit.test.ts +175 -0
- package/src/scenarios/sandbox-memory-cap.test.ts +2 -1
- package/src/scenarios/sandbox-mvp-behavior.test.ts +2 -1
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +2 -1
- package/src/scenarios/sandbox-timeout-cap.test.ts +2 -1
- package/src/scenarios/scheduling-capability-shape.test.ts +2 -1
- package/src/scenarios/scheduling-cron-fires-once.test.ts +2 -1
- package/src/scenarios/secret-leakage-otel-attribute.test.ts +7 -6
- package/src/scenarios/spec-corpus-validity.test.ts +20 -4
- package/src/scenarios/subrun-approval-fail-closed.test.ts +33 -0
- package/src/scenarios/subrun-approval-gate.test.ts +35 -0
- package/src/scenarios/subrun-attestation-shape.test.ts +30 -0
- package/src/scenarios/subrun-checksum-stable.test.ts +43 -0
- package/src/scenarios/tool-descriptor-shape.test.ts +133 -0
- package/src/scenarios/tool-hooks-authorization-fail-closed.test.ts +39 -0
- package/src/scenarios/tool-hooks-content-free.test.ts +40 -0
- package/src/scenarios/tool-hooks-rate-limit.test.ts +32 -0
- package/src/scenarios/tool-hooks-secret-redaction.test.ts +34 -0
- package/src/scenarios/tool-hooks-shape.test.ts +34 -0
- package/src/scenarios/trigger-bridge-shape.test.ts +135 -0
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +3 -10
- package/src/scenarios/wasm-pack-invoke-completed.test.ts +2 -2
- package/src/scenarios/wasm-pack-invoke-suspended.test.ts +2 -2
- package/src/scenarios/wasm-pack-load.test.ts +2 -2
- package/src/scenarios/wasm-pack-memory-cap.test.ts +3 -6
- package/src/scenarios/wasm-pack-replay-determinism.test.ts +2 -2
- package/src/scenarios/workflow-primary-output-annotation.test.ts +142 -0
- package/src/scenarios/workspace-behavior.test.ts +134 -0
- package/src/scenarios/workspace-capability-shape.test.ts +73 -0
- package/src/scenarios/workspace-cross-tenant-isolation.test.ts +84 -0
- package/src/scenarios/x-openwop-form-pack-manifest.test.ts +155 -0
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
|
|
38
38
|
import { describe, it, expect } from 'vitest';
|
|
39
39
|
import { driver } from '../lib/driver.js';
|
|
40
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
40
41
|
|
|
41
42
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
42
43
|
|
|
@@ -66,7 +67,7 @@ async function isSandboxAdvertised(): Promise<boolean> {
|
|
|
66
67
|
try {
|
|
67
68
|
const res = await driver.get('/.well-known/openwop');
|
|
68
69
|
if (res.status !== 200) return false;
|
|
69
|
-
return (res.json as DiscoveryDoc)
|
|
70
|
+
return capabilityFamily((res.json as DiscoveryDoc), 'sandbox')?.supported === true;
|
|
70
71
|
} catch {
|
|
71
72
|
return false;
|
|
72
73
|
}
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
|
|
27
27
|
import { describe, it, expect } from 'vitest';
|
|
28
28
|
import { driver } from '../lib/driver.js';
|
|
29
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
29
30
|
|
|
30
31
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
31
32
|
|
|
@@ -45,7 +46,7 @@ async function readSandboxCaps(): Promise<SandboxCaps | null> {
|
|
|
45
46
|
try {
|
|
46
47
|
const res = await driver.get('/.well-known/openwop');
|
|
47
48
|
if (res.status !== 200) return null;
|
|
48
|
-
return (res.json as DiscoveryDoc)
|
|
49
|
+
return capabilityFamily((res.json as DiscoveryDoc), 'sandbox') ?? null;
|
|
49
50
|
} catch {
|
|
50
51
|
return null;
|
|
51
52
|
}
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
import { describe, it, expect } from 'vitest';
|
|
17
17
|
import { driver } from '../lib/driver.js';
|
|
18
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
18
19
|
|
|
19
20
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
20
21
|
|
|
@@ -26,7 +27,7 @@ async function readSandbox(): Promise<{ supported: boolean; wallClockLimitMs?: n
|
|
|
26
27
|
try {
|
|
27
28
|
const r = await driver.get('/.well-known/openwop');
|
|
28
29
|
if (r.status !== 200) return null;
|
|
29
|
-
const sb = (r.json as D)
|
|
30
|
+
const sb = capabilityFamily((r.json as D), 'sandbox');
|
|
30
31
|
if (!sb || sb.supported !== true) return null;
|
|
31
32
|
return {
|
|
32
33
|
supported: true,
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import { describe, it, expect } from 'vitest';
|
|
22
22
|
import { driver } from '../lib/driver.js';
|
|
23
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
23
24
|
|
|
24
25
|
interface DiscoveryScheduling {
|
|
25
26
|
supported?: boolean;
|
|
@@ -39,7 +40,7 @@ const ISO_DURATION = /^P(?:\d+Y)?(?:\d+M)?(?:\d+W)?(?:\d+D)?(?:T(?:\d+H)?(?:\d+M
|
|
|
39
40
|
async function readScheduling(): Promise<DiscoveryScheduling | null> {
|
|
40
41
|
const res = await driver.get('/.well-known/openwop');
|
|
41
42
|
const body = res.json as DiscoveryDoc | undefined;
|
|
42
|
-
return body
|
|
43
|
+
return capabilityFamily(body, 'scheduling') ?? null;
|
|
43
44
|
}
|
|
44
45
|
|
|
45
46
|
describe('scheduling-capability-shape: advertisement shape (RFC 0052 §A)', () => {
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
|
|
27
27
|
import { describe, it, expect } from 'vitest';
|
|
28
28
|
import { driver } from '../lib/driver.js';
|
|
29
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
29
30
|
|
|
30
31
|
interface DiscoveryDoc {
|
|
31
32
|
capabilities?: { scheduling?: { supported?: boolean; cron?: boolean } };
|
|
@@ -33,7 +34,7 @@ interface DiscoveryDoc {
|
|
|
33
34
|
|
|
34
35
|
async function readScheduling(): Promise<{ supported?: boolean; cron?: boolean } | null> {
|
|
35
36
|
const res = await driver.get('/.well-known/openwop');
|
|
36
|
-
return (res.json as DiscoveryDoc | undefined)
|
|
37
|
+
return capabilityFamily((res.json as DiscoveryDoc | undefined), 'scheduling') ?? null;
|
|
37
38
|
}
|
|
38
39
|
|
|
39
40
|
describe('scheduling-cron-fires-once: once-per-tick + missed-tick (RFC 0052 §B)', () => {
|
|
@@ -55,6 +55,7 @@ import { describe, it, expect } from 'vitest';
|
|
|
55
55
|
import { driver } from '../lib/driver.js';
|
|
56
56
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
57
57
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
58
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
58
59
|
|
|
59
60
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
60
61
|
const BYOK_WORKFLOW_ID = 'openwop-smoke-byok-roundtrip';
|
|
@@ -99,8 +100,8 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
99
100
|
return;
|
|
100
101
|
}
|
|
101
102
|
const d = await readDiscovery();
|
|
102
|
-
const secretsOk = d
|
|
103
|
-
const seamOk = d
|
|
103
|
+
const secretsOk = capabilityFamily<{ supported?: unknown }>(d, 'secrets')?.supported === true;
|
|
104
|
+
const seamOk = capabilityFamily<{ testSeams?: Record<string, unknown> }>(d, 'observability')?.testSeams?.otelScrape === true;
|
|
104
105
|
if (!secretsOk || !seamOk) {
|
|
105
106
|
ctx.skip();
|
|
106
107
|
return;
|
|
@@ -168,8 +169,8 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
168
169
|
return;
|
|
169
170
|
}
|
|
170
171
|
const d = await readDiscovery();
|
|
171
|
-
const secretsOk = d
|
|
172
|
-
const seamOk = d
|
|
172
|
+
const secretsOk = capabilityFamily<{ supported?: unknown }>(d, 'secrets')?.supported === true;
|
|
173
|
+
const seamOk = capabilityFamily<{ testSeams?: Record<string, unknown> }>(d, 'observability')?.testSeams?.debugBundleExport === true;
|
|
173
174
|
if (!secretsOk || !seamOk) {
|
|
174
175
|
ctx.skip();
|
|
175
176
|
return;
|
|
@@ -209,11 +210,11 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
|
|
|
209
210
|
() => {
|
|
210
211
|
it('when secrets.supported is true, observability.testSeams advertisements MUST be boolean if present', async (ctx) => {
|
|
211
212
|
const d = await readDiscovery();
|
|
212
|
-
if (d
|
|
213
|
+
if (capabilityFamily<{ supported?: unknown }>(d, 'secrets')?.supported !== true) {
|
|
213
214
|
ctx.skip();
|
|
214
215
|
return;
|
|
215
216
|
}
|
|
216
|
-
const seams = d
|
|
217
|
+
const seams = capabilityFamily<{ testSeams?: Record<string, unknown> }>(d, 'observability')?.testSeams;
|
|
217
218
|
if (seams === undefined) {
|
|
218
219
|
ctx.skip(); // host honest about not exposing the seams — Drift #17 path
|
|
219
220
|
return;
|
|
@@ -384,16 +384,32 @@ function extractReadmeDocumentIndex(readme: string): string {
|
|
|
384
384
|
return readme.slice(start, end);
|
|
385
385
|
}
|
|
386
386
|
|
|
387
|
-
function listMarkdownFilesRecursive(dir: string): string[] {
|
|
387
|
+
function listMarkdownFilesRecursive(dir: string, repoRoot: string = dir): string[] {
|
|
388
388
|
const ignoredDirs = new Set(['.git', 'node_modules', 'dist']);
|
|
389
|
+
// Repo-relative directory paths to prune. These are subtrees whose
|
|
390
|
+
// content shouldn't be link-checked because either (a) they're
|
|
391
|
+
// generated build output (`site/out`) or (b) they're a vendored
|
|
392
|
+
// mirror of a canonical source whose READMEs use links relative to
|
|
393
|
+
// the canonical path, not the vendored path:
|
|
394
|
+
//
|
|
395
|
+
// - `apps/workflow-engine/packs/` mirrors repo-root `packs/`, synced
|
|
396
|
+
// via `apps/workflow-engine/scripts/sync-packs.sh` so the Cloud
|
|
397
|
+
// Run image's `apps/workflow-engine/` build context can ship them.
|
|
398
|
+
// Pack READMEs use `../../RFCS/...` / `../../spec/v1/...` links
|
|
399
|
+
// that resolve from the canonical location (which this walker
|
|
400
|
+
// DOES check) but break from the deeper vendored path. The
|
|
401
|
+
// canonical copies are authoritative; the vendored copies are
|
|
402
|
+
// byte-for-byte identical via cp -R.
|
|
403
|
+
const prunedRepoRelative = new Set(['site/out', 'apps/workflow-engine/packs']);
|
|
389
404
|
const files: string[] = [];
|
|
390
405
|
|
|
391
406
|
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
392
407
|
if (entry.isDirectory()) {
|
|
393
408
|
if (ignoredDirs.has(entry.name)) continue;
|
|
394
409
|
const child = join(dir, entry.name);
|
|
395
|
-
|
|
396
|
-
|
|
410
|
+
const repoRelChild = relative(repoRoot, child);
|
|
411
|
+
if (prunedRepoRelative.has(repoRelChild)) continue;
|
|
412
|
+
files.push(...listMarkdownFilesRecursive(child, repoRoot));
|
|
397
413
|
continue;
|
|
398
414
|
}
|
|
399
415
|
if (entry.isFile() && entry.name.endsWith('.md')) {
|
|
@@ -1019,7 +1035,7 @@ describe('spec-corpus: AsyncAPI 3.1 spec is structurally valid', () => {
|
|
|
1019
1035
|
// `run.annotated` (RFC 0056) is a live SSE notification carrying an
|
|
1020
1036
|
// Annotation — NOT a RunEventDoc and deliberately NOT in the RunEventType
|
|
1021
1037
|
// enum (annotations are a side-resource, excluded from fork/replay).
|
|
1022
|
-
const syntheticMessageNames = new Set(['state.snapshot', 'ai.message.chunk', 'any', 'run.annotated']);
|
|
1038
|
+
const syntheticMessageNames = new Set(['state.snapshot', 'ai.message.chunk', 'any', 'run.annotated', 'heartbeat.evaluated', 'heartbeat.stateChanged']);
|
|
1023
1039
|
|
|
1024
1040
|
expect(messageNames.length, 'AsyncAPI MUST declare named SSE messages').toBeGreaterThan(0);
|
|
1025
1041
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* subrun-approval-fail-closed — RFC 0063 §C. A parent that terminates or whose
|
|
3
|
+
* approval interrupt expires WITHOUT an `accept`/`edit-accept` MUST NOT merge the
|
|
4
|
+
* child outputs. Absence of an approval is denial — backs the proposed
|
|
5
|
+
* protocol-tier SECURITY invariant `subrun-merge-approval-fail-closed` (lands
|
|
6
|
+
* with this test promoted to load-bearing at reference-host implementation).
|
|
7
|
+
*
|
|
8
|
+
* Gated on `capabilities.agents.subRunAttestation` + the host sub-run attestation
|
|
9
|
+
* seam; soft-skips when either is absent.
|
|
10
|
+
*
|
|
11
|
+
* @see RFCS/0063-subrun-output-attestation-and-merge-gating.md §C
|
|
12
|
+
* @see SECURITY/invariants.yaml — subrun-merge-approval-fail-closed (lands at impl)
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { describe, it, expect } from 'vitest';
|
|
16
|
+
import { driver } from '../lib/driver.js';
|
|
17
|
+
import { readSubRunAttestationCap, invokeSubRunAttest } from '../lib/subRunAttestation.js';
|
|
18
|
+
|
|
19
|
+
describe('subrun-approval-fail-closed (RFC 0063 §C)', () => {
|
|
20
|
+
it('no accept/edit-accept (terminated or expired) MUST NOT merge', async () => {
|
|
21
|
+
if ((await readSubRunAttestationCap()) !== true) return;
|
|
22
|
+
// approvalAction omitted models a run that terminated without a response.
|
|
23
|
+
const res = await invokeSubRunAttest({
|
|
24
|
+
childOutputs: { artifact: 'unverified' },
|
|
25
|
+
outputAttestation: { requireApproval: true },
|
|
26
|
+
});
|
|
27
|
+
if (res === null) return; // seam absent — soft-skip
|
|
28
|
+
expect(
|
|
29
|
+
res.merged,
|
|
30
|
+
driver.describe('RFC 0063 §C', 'an unresolved approval MUST fail closed — outputs MUST NOT be merged'),
|
|
31
|
+
).toBe(false);
|
|
32
|
+
});
|
|
33
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* subrun-approval-gate — RFC 0063 §C. When `requireApproval: true`, the host
|
|
3
|
+
* suspends before merge; `accept` merges the child outputs, `reject` does not.
|
|
4
|
+
*
|
|
5
|
+
* Gated on `capabilities.agents.subRunAttestation` + the host sub-run attestation
|
|
6
|
+
* seam; soft-skips when either is absent.
|
|
7
|
+
*
|
|
8
|
+
* @see RFCS/0063-subrun-output-attestation-and-merge-gating.md §C
|
|
9
|
+
* @see spec/v1/interrupt.md — `approval` kind + resume actions (RFC 0051, reused)
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { describe, it, expect } from 'vitest';
|
|
13
|
+
import { driver } from '../lib/driver.js';
|
|
14
|
+
import { readSubRunAttestationCap, invokeSubRunAttest } from '../lib/subRunAttestation.js';
|
|
15
|
+
|
|
16
|
+
describe('subrun-approval-gate (RFC 0063 §C)', () => {
|
|
17
|
+
it('accept merges the child outputs; reject does not', async () => {
|
|
18
|
+
if ((await readSubRunAttestationCap()) !== true) return;
|
|
19
|
+
const base = { childOutputs: { artifact: 'x' }, outputAttestation: { requireApproval: true } };
|
|
20
|
+
|
|
21
|
+
const accepted = await invokeSubRunAttest({ ...base, approvalAction: 'accept' });
|
|
22
|
+
if (accepted === null) return; // seam absent — soft-skip
|
|
23
|
+
expect(
|
|
24
|
+
accepted.merged,
|
|
25
|
+
driver.describe('RFC 0063 §C', 'an `accept` approval MUST merge the child outputs'),
|
|
26
|
+
).toBe(true);
|
|
27
|
+
|
|
28
|
+
const rejected = await invokeSubRunAttest({ ...base, approvalAction: 'reject' });
|
|
29
|
+
if (rejected === null) return;
|
|
30
|
+
expect(
|
|
31
|
+
rejected.merged,
|
|
32
|
+
driver.describe('RFC 0063 §C', 'a `reject` approval MUST NOT merge the child outputs'),
|
|
33
|
+
).toBe(false);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* subrun-attestation-shape — RFC 0063 §A. The `capabilities.agents.subRunAttestation`
|
|
3
|
+
* advertisement flag is either absent or a boolean.
|
|
4
|
+
*
|
|
5
|
+
* Status: ACTIVE (advertisement-shape; always runs). Behavioral coverage lives
|
|
6
|
+
* in the sibling subrun-*.test.ts scenarios, gated on the flag + the host
|
|
7
|
+
* sub-run attestation seam.
|
|
8
|
+
*
|
|
9
|
+
* @see RFCS/0063-subrun-output-attestation-and-merge-gating.md §A
|
|
10
|
+
* @see spec/v1/node-packs.md §"`outputAttestation` — verify-before-merge"
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest';
|
|
14
|
+
import { driver } from '../lib/driver.js';
|
|
15
|
+
import { readSubRunAttestationCap } from '../lib/subRunAttestation.js';
|
|
16
|
+
|
|
17
|
+
describe('subrun-attestation-shape: advertisement (RFC 0063 §A)', () => {
|
|
18
|
+
it('capabilities.agents.subRunAttestation is absent or a boolean', async () => {
|
|
19
|
+
const cap = await readSubRunAttestationCap();
|
|
20
|
+
// null = unadvertised (no agents block OR flag omitted) — valid.
|
|
21
|
+
if (cap === null) return;
|
|
22
|
+
expect(
|
|
23
|
+
typeof cap,
|
|
24
|
+
driver.describe(
|
|
25
|
+
'capabilities.schema.json §agents.subRunAttestation',
|
|
26
|
+
'agents.subRunAttestation MUST be a boolean when present',
|
|
27
|
+
),
|
|
28
|
+
).toBe('boolean');
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* subrun-checksum-stable — RFC 0063 §B. A child's output checksum is byte-stable
|
|
3
|
+
* for identical outputs and host-independent (the RFC 8785 JCS + SHA-256 recipe
|
|
4
|
+
* pinned in replay.md), and is surfaced as the `attestation` object on the
|
|
5
|
+
* existing `core.workflowChain.event { phase: 'output.harvested' }`.
|
|
6
|
+
*
|
|
7
|
+
* Gated on `capabilities.agents.subRunAttestation` + the host sub-run attestation
|
|
8
|
+
* seam; soft-skips when either is absent.
|
|
9
|
+
*
|
|
10
|
+
* @see RFCS/0063-subrun-output-attestation-and-merge-gating.md §B
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest';
|
|
14
|
+
import { driver } from '../lib/driver.js';
|
|
15
|
+
import { readSubRunAttestationCap, invokeSubRunAttest } from '../lib/subRunAttestation.js';
|
|
16
|
+
|
|
17
|
+
describe('subrun-checksum-stable (RFC 0063 §B)', () => {
|
|
18
|
+
it('identical child outputs produce an identical sha256 attestation checksum', async () => {
|
|
19
|
+
if ((await readSubRunAttestationCap()) !== true) return;
|
|
20
|
+
const childOutputs = { report: 'done', score: 0.9, tags: ['a', 'b'] };
|
|
21
|
+
const a = await invokeSubRunAttest({ childOutputs, outputAttestation: { checksum: true } });
|
|
22
|
+
if (a === null) return; // seam absent — soft-skip
|
|
23
|
+
// Key-reordered but value-identical: JCS canonicalization MUST yield the same hash.
|
|
24
|
+
const b = await invokeSubRunAttest({
|
|
25
|
+
childOutputs: { tags: ['a', 'b'], score: 0.9, report: 'done' },
|
|
26
|
+
outputAttestation: { checksum: true },
|
|
27
|
+
});
|
|
28
|
+
if (b === null) return;
|
|
29
|
+
const att = a.attestation ?? {};
|
|
30
|
+
expect(
|
|
31
|
+
typeof att.checksum === 'string' && (att.checksum as string).length > 0,
|
|
32
|
+
driver.describe('RFC 0063 §B', 'output.harvested MUST carry a non-empty attestation.checksum when checksum:true'),
|
|
33
|
+
).toBe(true);
|
|
34
|
+
expect(
|
|
35
|
+
att.algorithm,
|
|
36
|
+
driver.describe('RFC 0063 §B', 'attestation.algorithm MUST be "sha256" (the v1 recipe)'),
|
|
37
|
+
).toBe('sha256');
|
|
38
|
+
expect(
|
|
39
|
+
(b.attestation ?? {}).checksum,
|
|
40
|
+
driver.describe('RFC 0063 §B', 'JCS canonicalization MUST make the checksum invariant to key order — same content, same hash'),
|
|
41
|
+
).toBe(att.checksum);
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Portable tool catalog — descriptor + capability + session-event shapes (RFC 0078).
|
|
3
|
+
*
|
|
4
|
+
* Always-on, server-free schema-shape probe. Verifies that:
|
|
5
|
+
* - `tool-descriptor.schema.json` compiles and round-trips a conforming
|
|
6
|
+
* `ToolDescriptor`, and rejects a descriptor missing the REQUIRED
|
|
7
|
+
* `safetyTier`.
|
|
8
|
+
* - the §C-1 / §F-4 cross-field MUST is enforced: a `safetyTier: "exec"`
|
|
9
|
+
* descriptor MUST carry `source: "host-extension"` (RFC 0069 — exec is never
|
|
10
|
+
* protocol-tier); an `exec` + `node-pack` descriptor is rejected, an `exec`
|
|
11
|
+
* + `host-extension` descriptor is accepted.
|
|
12
|
+
* - `capabilities.toolCatalog` is declared with its `supported` / `sources` /
|
|
13
|
+
* `sessionLifecycle` sub-flags.
|
|
14
|
+
* - the `tool.session.opened` / `tool.session.closed` payload $defs validate
|
|
15
|
+
* conforming content-free records and reject malformed ones (a `closed`
|
|
16
|
+
* missing `outcome`; an out-of-enum `outcome`), and both event names appear
|
|
17
|
+
* in the RunEventType enum.
|
|
18
|
+
*
|
|
19
|
+
* Behavioral assertions (a live `GET /v1/tools` returning authorization-scoped
|
|
20
|
+
* descriptors, the `404` non-disclosure, the `tool.session.*` bracket ordering)
|
|
21
|
+
* are gated on `capabilities.toolCatalog.supported` and land in
|
|
22
|
+
* `tool-catalog-projection.test.ts` + `tool-session-lifecycle.test.ts` (deferred
|
|
23
|
+
* per RFC 0078 §Conformance — reference host deferred). This scenario asserts the
|
|
24
|
+
* wire contract, not host behavior.
|
|
25
|
+
*
|
|
26
|
+
* Spec references:
|
|
27
|
+
* - https://github.com/openwop/openwop/blob/main/spec/v1/tool-catalog.md
|
|
28
|
+
* - https://github.com/openwop/openwop/blob/main/RFCS/0078-portable-tool-catalog-and-tool-session-contract.md
|
|
29
|
+
* - https://github.com/openwop/openwop/blob/main/RFCS/0069-exec-class-tool-host-extension-safety-contract.md (exec ⇒ host-extension)
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { describe, it, expect } from 'vitest';
|
|
33
|
+
import { readFileSync } from 'node:fs';
|
|
34
|
+
import { join } from 'node:path';
|
|
35
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
36
|
+
import addFormats from 'ajv-formats';
|
|
37
|
+
import { SCHEMAS_DIR } from '../lib/paths.js';
|
|
38
|
+
|
|
39
|
+
const why = (specRef: string, requirement: string): string => `${specRef} — ${requirement}`;
|
|
40
|
+
|
|
41
|
+
function loadSchema(name: string): Record<string, unknown> {
|
|
42
|
+
return JSON.parse(readFileSync(join(SCHEMAS_DIR, name), 'utf8')) as Record<string, unknown>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
describe('tool-descriptor-shape: ToolDescriptor (RFC 0078 §C, server-free)', () => {
|
|
46
|
+
const ajv = addFormats(new Ajv2020({ strict: false }));
|
|
47
|
+
const validate = ajv.compile(loadSchema('tool-descriptor.schema.json'));
|
|
48
|
+
|
|
49
|
+
it('a conforming descriptor validates', () => {
|
|
50
|
+
expect(
|
|
51
|
+
validate({
|
|
52
|
+
toolId: 'mcp:fs.read', source: 'mcp', title: 'Read file',
|
|
53
|
+
inputSchema: { type: 'object' }, auth: { scopes: ['tools:fs:read'] },
|
|
54
|
+
egress: 'none', approval: 'never', replayPolicy: 'idempotent',
|
|
55
|
+
safetyTier: 'read', costHint: 'low', latencyHint: 'low',
|
|
56
|
+
}),
|
|
57
|
+
why('tool-catalog.md §C', 'a conforming ToolDescriptor MUST validate'),
|
|
58
|
+
).toBe(true);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('a descriptor missing the REQUIRED safetyTier is rejected', () => {
|
|
62
|
+
expect(
|
|
63
|
+
validate({ toolId: 'x', source: 'mcp' }),
|
|
64
|
+
why('tool-catalog.md §C', 'safetyTier is REQUIRED'),
|
|
65
|
+
).toBe(false);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it('enforces exec ⇒ host-extension (RFC 0069; §C-1/§F-4)', () => {
|
|
69
|
+
expect(
|
|
70
|
+
validate({ toolId: 'x-host-acme-shell', source: 'host-extension', safetyTier: 'exec', approval: 'always', egress: 'host-owned' }),
|
|
71
|
+
why('tool-catalog.md §C-1', 'an exec tool sourced from host-extension MUST validate'),
|
|
72
|
+
).toBe(true);
|
|
73
|
+
expect(
|
|
74
|
+
validate({ toolId: 'openwop:run-shell', source: 'node-pack', safetyTier: 'exec' }),
|
|
75
|
+
why('tool-catalog.md §C-1 / RFC 0069', 'an exec tool MUST NOT be protocol-tier (node-pack)'),
|
|
76
|
+
).toBe(false);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('rejects an unknown property (additionalProperties:false)', () => {
|
|
80
|
+
expect(
|
|
81
|
+
validate({ toolId: 'x', source: 'mcp', safetyTier: 'read', danger: true }),
|
|
82
|
+
why('tool-catalog.md §C', 'ToolDescriptor MUST be additionalProperties:false'),
|
|
83
|
+
).toBe(false);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe('tool-descriptor-shape: capability advertisement (RFC 0078 §A, server-free)', () => {
|
|
88
|
+
it('capabilities.toolCatalog is declared with its sub-flags', () => {
|
|
89
|
+
const caps = loadSchema('capabilities.schema.json');
|
|
90
|
+
const toolCatalog = (caps.properties as Record<string, { properties?: Record<string, unknown> }>).toolCatalog;
|
|
91
|
+
expect(
|
|
92
|
+
toolCatalog,
|
|
93
|
+
why('capabilities.md §toolCatalog', 'capabilities.toolCatalog MUST be declared'),
|
|
94
|
+
).toBeDefined();
|
|
95
|
+
for (const flag of ['supported', 'sources', 'sessionLifecycle']) {
|
|
96
|
+
expect(
|
|
97
|
+
toolCatalog?.properties?.[flag],
|
|
98
|
+
why('tool-catalog.md §A', `capabilities.toolCatalog.${flag} MUST be declared`),
|
|
99
|
+
).toBeDefined();
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
describe('tool-descriptor-shape: session lifecycle events (RFC 0078 §D, server-free)', () => {
|
|
105
|
+
const payloads = loadSchema('run-event-payloads.schema.json');
|
|
106
|
+
const ajv = addFormats(new Ajv2020({ strict: false }));
|
|
107
|
+
const compile = (defName: string) => ajv.compile({
|
|
108
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
109
|
+
$defs: (payloads as { $defs: Record<string, unknown> }).$defs,
|
|
110
|
+
$ref: `#/$defs/${defName}`,
|
|
111
|
+
} as Record<string, unknown>);
|
|
112
|
+
|
|
113
|
+
it('tool.session.opened validates a content-free record', () => {
|
|
114
|
+
const v = compile('toolSessionOpened');
|
|
115
|
+
expect(v({ sessionId: 's1', toolId: 'mcp:fs.read' }), why('tool-catalog.md §D', 'opened MUST validate')).toBe(true);
|
|
116
|
+
expect(v({ toolId: 'mcp:fs.read' }), why('tool-catalog.md §D', 'opened requires sessionId')).toBe(false);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('tool.session.closed validates + enforces the closed outcome enum', () => {
|
|
120
|
+
const v = compile('toolSessionClosed');
|
|
121
|
+
expect(v({ sessionId: 's1', toolId: 'mcp:fs.read', outcome: 'completed' }), why('tool-catalog.md §D', 'closed MUST validate')).toBe(true);
|
|
122
|
+
expect(v({ sessionId: 's1', toolId: 'mcp:fs.read' }), why('tool-catalog.md §D', 'closed requires outcome')).toBe(false);
|
|
123
|
+
expect(v({ sessionId: 's1', toolId: 'mcp:fs.read', outcome: 'exploded' }), why('tool-catalog.md §D', 'outcome is a closed enum')).toBe(false);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('both session event names appear in the RunEventType enum', () => {
|
|
127
|
+
const runEvent = loadSchema('run-event.schema.json');
|
|
128
|
+
const enumVals = ((runEvent.$defs as Record<string, { enum?: string[] }>).RunEventType?.enum) ?? [];
|
|
129
|
+
for (const name of ['tool.session.opened', 'tool.session.closed']) {
|
|
130
|
+
expect(enumVals.includes(name), why('run-event.schema.json', `${name} MUST be in the RunEventType enum`)).toBe(true);
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
});
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-hooks-authorization-fail-closed — RFC 0064 §C. A principal lacking a
|
|
3
|
+
* tool's required scope (or whose authorization cannot be evaluated) gets
|
|
4
|
+
* `agent.toolReturned { status: 'forbidden' }` and the tool is never invoked —
|
|
5
|
+
* the per-tool application of RFC 0049's `authorization-fail-closed` invariant.
|
|
6
|
+
*
|
|
7
|
+
* Gated on `capabilities.toolHooks.perToolAuthorization` + the host tool-hooks
|
|
8
|
+
* seam; soft-skips when either is absent.
|
|
9
|
+
*
|
|
10
|
+
* @see RFCS/0064-tool-invocation-hooks-and-authorization.md §C
|
|
11
|
+
* @see SECURITY/invariants.yaml — authorization-fail-closed (RFC 0049, reused)
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, it, expect } from 'vitest';
|
|
15
|
+
import { driver } from '../lib/driver.js';
|
|
16
|
+
import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
|
|
17
|
+
|
|
18
|
+
describe('tool-hooks-authorization-fail-closed (RFC 0064 §C)', () => {
|
|
19
|
+
it('a principal lacking a tool scope is denied and the tool is not invoked', async () => {
|
|
20
|
+
const cap = await readToolHooksCap();
|
|
21
|
+
if (cap?.perToolAuthorization !== true) return;
|
|
22
|
+
// A principal with no scopes against a tool requiring one MUST be denied.
|
|
23
|
+
const res = await invokeToolHook({
|
|
24
|
+
principal: 'conformance-unprivileged',
|
|
25
|
+
toolName: 'db.delete',
|
|
26
|
+
requiredScopes: ['db:write'],
|
|
27
|
+
args: {},
|
|
28
|
+
});
|
|
29
|
+
if (res === null) return; // seam absent — soft-skip
|
|
30
|
+
expect(
|
|
31
|
+
(res.toolReturned ?? {}).status,
|
|
32
|
+
driver.describe('RFC 0064 §C', 'a missing/unevaluable tool scope MUST fail closed → status:"forbidden"'),
|
|
33
|
+
).toBe('forbidden');
|
|
34
|
+
expect(
|
|
35
|
+
(res.toolReturned ?? {}).durationMs,
|
|
36
|
+
driver.describe('RFC 0064 §C', 'a forbidden call never starts, so durationMs MUST be absent'),
|
|
37
|
+
).toBeUndefined();
|
|
38
|
+
});
|
|
39
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-hooks-content-free — RFC 0064 §B. When `prePostEvents`, a tool call's
|
|
3
|
+
* `agent.toolCalled` carries `argsHash` (the content-free, SIEM-safe
|
|
4
|
+
* alternative to raw `inputs`) + `agent.toolReturned` carries `status` +
|
|
5
|
+
* `durationMs`.
|
|
6
|
+
*
|
|
7
|
+
* Gated on `capabilities.toolHooks.prePostEvents` + the host tool-hooks seam;
|
|
8
|
+
* soft-skips when either is absent.
|
|
9
|
+
*
|
|
10
|
+
* @see RFCS/0064-tool-invocation-hooks-and-authorization.md §B
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest';
|
|
14
|
+
import { driver } from '../lib/driver.js';
|
|
15
|
+
import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
|
|
16
|
+
|
|
17
|
+
describe('tool-hooks-content-free (RFC 0064 §B)', () => {
|
|
18
|
+
it('toolCalled carries argsHash; toolReturned carries status + durationMs', async () => {
|
|
19
|
+
const cap = await readToolHooksCap();
|
|
20
|
+
if (cap?.prePostEvents !== true) return;
|
|
21
|
+
const res = await invokeToolHook({ principal: 'core.system', toolName: 'web.search', args: { q: 'openwop' } });
|
|
22
|
+
if (res === null) return; // seam absent — soft-skip
|
|
23
|
+
const called = res.toolCalled ?? {};
|
|
24
|
+
const returned = res.toolReturned ?? {};
|
|
25
|
+
expect(
|
|
26
|
+
typeof called.argsHash === 'string' && (called.argsHash as string).length > 0,
|
|
27
|
+
driver.describe('RFC 0064 §B', 'agent.toolCalled MUST carry a non-empty argsHash when prePostEvents'),
|
|
28
|
+
).toBe(true);
|
|
29
|
+
expect(
|
|
30
|
+
['ok', 'error', 'forbidden', 'rate_limited'].includes(returned.status as string),
|
|
31
|
+
driver.describe('RFC 0064 §B', 'agent.toolReturned MUST carry a tool-hooks status'),
|
|
32
|
+
).toBe(true);
|
|
33
|
+
if (returned.status === 'ok') {
|
|
34
|
+
expect(
|
|
35
|
+
typeof returned.durationMs === 'number' && (returned.durationMs as number) >= 0,
|
|
36
|
+
driver.describe('RFC 0064 §B', 'a completed tool call MUST record a non-negative durationMs'),
|
|
37
|
+
).toBe(true);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-hooks-rate-limit — RFC 0064 §D. Exhausting a `(principal, tool)` token
|
|
3
|
+
* bucket → `agent.toolReturned { status: 'rate_limited' }` and the tool is not
|
|
4
|
+
* invoked, surfacing the existing `rate_limited` (429) error.
|
|
5
|
+
*
|
|
6
|
+
* Gated on `capabilities.toolHooks.perToolRateLimit` + the host tool-hooks
|
|
7
|
+
* seam; soft-skips when either is absent.
|
|
8
|
+
*
|
|
9
|
+
* @see RFCS/0064-tool-invocation-hooks-and-authorization.md §D
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { describe, it, expect } from 'vitest';
|
|
13
|
+
import { driver } from '../lib/driver.js';
|
|
14
|
+
import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
|
|
15
|
+
|
|
16
|
+
describe('tool-hooks-rate-limit (RFC 0064 §D)', () => {
|
|
17
|
+
it('an exhausted (principal, tool) bucket yields status:"rate_limited"', async () => {
|
|
18
|
+
const cap = await readToolHooksCap();
|
|
19
|
+
if (cap?.perToolRateLimit !== true) return;
|
|
20
|
+
const res = await invokeToolHook({
|
|
21
|
+
principal: 'core.system',
|
|
22
|
+
toolName: 'web.search',
|
|
23
|
+
args: { q: 'x' },
|
|
24
|
+
simulateRateLimitExhausted: true,
|
|
25
|
+
});
|
|
26
|
+
if (res === null) return; // seam absent — soft-skip
|
|
27
|
+
expect(
|
|
28
|
+
(res.toolReturned ?? {}).status,
|
|
29
|
+
driver.describe('RFC 0064 §D', 'an exhausted token bucket MUST yield status:"rate_limited" without invoking the tool'),
|
|
30
|
+
).toBe('rate_limited');
|
|
31
|
+
});
|
|
32
|
+
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-hooks-secret-redaction — RFC 0064 §B/§E. A tool arg containing a
|
|
3
|
+
* resolved secret is redacted (SR-1) before hashing; the raw value never
|
|
4
|
+
* appears in `argsHash` or anywhere in the emitted `agent.toolCalled` /
|
|
5
|
+
* `agent.toolReturned` pair.
|
|
6
|
+
*
|
|
7
|
+
* Gated on `capabilities.toolHooks.prePostEvents` + the host tool-hooks seam;
|
|
8
|
+
* soft-skips when either is absent.
|
|
9
|
+
*
|
|
10
|
+
* @see RFCS/0064-tool-invocation-hooks-and-authorization.md §B (argsHash SR-1 redaction), §E (credentials)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest';
|
|
14
|
+
import { driver } from '../lib/driver.js';
|
|
15
|
+
import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
|
|
16
|
+
|
|
17
|
+
const CANARY = 'sk-canary-rfc0064-do-not-leak-xyz789';
|
|
18
|
+
|
|
19
|
+
describe('tool-hooks-secret-redaction (RFC 0064 §E)', () => {
|
|
20
|
+
it('a secret-shaped tool arg never appears in the emitted events', async () => {
|
|
21
|
+
const cap = await readToolHooksCap();
|
|
22
|
+
if (cap?.prePostEvents !== true) return;
|
|
23
|
+
const res = await invokeToolHook({
|
|
24
|
+
principal: 'core.system',
|
|
25
|
+
toolName: 'web.search',
|
|
26
|
+
args: { apiKey: CANARY, q: 'openwop' },
|
|
27
|
+
});
|
|
28
|
+
if (res === null) return; // seam absent — soft-skip
|
|
29
|
+
expect(
|
|
30
|
+
JSON.stringify(res).includes(CANARY),
|
|
31
|
+
driver.describe('RFC 0064 §B', 'a resolved secret MUST be redacted before hashing; the raw value MUST NOT appear in argsHash or any emitted field (SR-1)'),
|
|
32
|
+
).toBe(false);
|
|
33
|
+
});
|
|
34
|
+
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tool-hooks-shape — RFC 0064 §A. The `capabilities.toolHooks` advertisement
|
|
3
|
+
* block is either absent or a well-formed object.
|
|
4
|
+
*
|
|
5
|
+
* Status: ACTIVE (advertisement-shape; always runs). Behavioral coverage lives
|
|
6
|
+
* in the sibling tool-hooks-*.test.ts scenarios, gated on the sub-flags + the
|
|
7
|
+
* host tool-hooks seam.
|
|
8
|
+
*
|
|
9
|
+
* @see RFCS/0064-tool-invocation-hooks-and-authorization.md §A
|
|
10
|
+
* @see spec/v1/host-capabilities.md §host.toolHooks
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest';
|
|
14
|
+
import { driver } from '../lib/driver.js';
|
|
15
|
+
import { readToolHooksCap } from '../lib/toolHooks.js';
|
|
16
|
+
|
|
17
|
+
describe('tool-hooks-shape: advertisement (RFC 0064 §A)', () => {
|
|
18
|
+
it('capabilities.toolHooks is absent or a well-formed object', async () => {
|
|
19
|
+
const cap = await readToolHooksCap();
|
|
20
|
+
if (cap === null) return; // not advertised — valid
|
|
21
|
+
expect(
|
|
22
|
+
typeof cap.supported,
|
|
23
|
+
driver.describe('capabilities.schema.json §toolHooks', 'toolHooks.supported MUST be a boolean when the block is present'),
|
|
24
|
+
).toBe('boolean');
|
|
25
|
+
for (const k of ['prePostEvents', 'perToolAuthorization', 'perToolRateLimit'] as const) {
|
|
26
|
+
if (cap[k] !== undefined) {
|
|
27
|
+
expect(
|
|
28
|
+
typeof cap[k],
|
|
29
|
+
driver.describe('capabilities.schema.json §toolHooks', `toolHooks.${k} MUST be a boolean when present`),
|
|
30
|
+
).toBe('boolean');
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
});
|