@responsibleai/agent-hooks 0.1.0-beta.1 → 0.1.0-beta.3

package/dist/ctk/runner.js

@@ -1,21 +1,28 @@
 "use strict";
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
+// Copyright (c) Microsoft. All rights reserved.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadVectors = loadVectors;
+exports.redactPaths = redactPaths;
+exports.builderFromValue = builderFromValue;
+exports.provePaths = provePaths;
+exports.assessHarness = assessHarness;
+exports.runVector = runVector;
+exports.runVectors = runVectors;
 /**
  * CTK runner: load vectors, drive a harness, assert `expect`.
  *
  * The assertion engine, capability skip check, and scripted
  * interceptor/resolver evaluation live in the Rust core (native.ctk*).
- * This module keeps only vector globbing, the recording wrapper, and
- * the orchestration loop that calls the native `Harness`.
+ * This module keeps only vector globbing, the recording wrapper, the
+ * host declaration seam (§7.7.9: the CTK registry, the construction
+ * path proof and the `load` outcome) and the orchestration loop that
+ * calls the native `Harness`.
  */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadVectors = loadVectors;
-exports.runVector = runVector;
-exports.runVectors = runVectors;
 const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const index_1 = require("../index");
+const declaration_1 = require("../declaration");
 const native_1 = require("../native");
 function loadVectors(dir) {
     const vectors = (0, node_fs_1.readdirSync)(dir)
@@ -83,6 +90,83 @@ class ScriptedResolver {
         return r;
     }
 }
+/** §9 redaction seam, CTK convention: each listed path is replaced
+ * with "[redacted]" via the §5.2/§4.3 transform machinery; a path that
+ * does not resolve at the escalating point is left untouched. */
+function redactPaths(ctx, paths) {
+    let out = JSON.stringify(ctx);
+    for (const path of paths) {
+        try {
+            out = native_1.native.applyTransformCtx(out, path, '"[redacted]"');
+        }
+        catch {
+            /* unresolvable at this point: skip */
+        }
+    }
+    return JSON.parse(out);
+}
+/** The scripted interceptors and resolver a vector carries, shared by
+ * the field-based and the declaration path. */
+class Scripts {
+    scripts;
+    approval;
+    redact;
+    /** Script 0 records: `expect.interceptions` describes each emission
+     * as the first interceptor saw it. One recorder per vector run, so
+     * the declaration path and the field path record alike. */
+    recorder;
+    constructor(v) {
+        // Multi-interceptor vectors (§7.1 fold-through) use
+        // interceptor_scripts; single-interceptor vectors use
+        // interceptor_script. An empty interceptor_scripts registers zero
+        // interceptors (§7 fail-closed vector).
+        this.scripts = v.interceptor_scripts ?? [v.interceptor_script];
+        this.approval = v.approval_script ?? [];
+        this.redact = v.redact_for_approval ?? [];
+        this.recorder = this.scripts.length > 0 ? new RecordingInterceptor(this.scripts[0]) : null;
+    }
+    /** Scripted interceptor `i`; only index 0 records. */
+    interceptor(i) {
+        if (i >= this.scripts.length)
+            return undefined;
+        return i === 0 ? this.recorder : new ScriptedInterceptor(this.scripts[i]);
+    }
+    interceptors() {
+        return this.scripts.map((_, i) => this.interceptor(i));
+    }
+    /** NB: [] is truthy in JS, so an empty approval_script registers NO
+     * resolver (matches the Rust/Python runners; exercised by AH-CTK-032). */
+    resolver() {
+        return this.approval.length > 0 ? new ScriptedResolver(this.approval) : null;
+    }
+    recorded() {
+        return this.recorder?.recorded ?? [];
+    }
+    /** The CTK registry for a declaration vector (§7.7.9): kind
+     * `ctk.scripted` (config `{"script": i}`), identity provider
+     * `ctk-fault`, approval resolver `ctk-scripted`, redactor
+     * `ctk-redact`. */
+    registry(surface) {
+        const redact = [...this.redact];
+        return index_1.HostRegistry.forConformance(surface)
+            .kind("ctk.scripted", (config, ctx) => {
+            const script = config?.["script"];
+            if (typeof script !== "number" || !Number.isInteger(script) || script < 0) {
+                throw new Error("config.script must be an unsigned integer index");
+            }
+            const i = this.interceptor(script);
+            if (i === undefined) {
+                throw new Error(`config.script ${script} is out of range for binding ${ctx.id}`);
+            }
+            return i;
+        })
+            .identityProvider("ctk-fault", () => {
+            throw new Error("ctk scripted provider fault");
+        })
+            .approvalResolver("ctk-scripted", new ScriptedResolver(this.approval))
+            .approvalRedactor("ctk-redact", (ctx) => redactPaths(ctx, redact));
+    }
+}
 function runRecordToWire(rr, postures) {
     return JSON.stringify({
         outcome: rr.outcome,
@@ -94,36 +178,412 @@ function runRecordToWire(rr, postures) {
         // Harness *declarations* (§13.1), not observed behavior: the engine
         // selects expect.run_outcome_by_posture entries by them.
         postures,
+        ...(rr.load !== undefined ? { load: rr.load } : {}),
+    });
+}
+// ---- builder from a value (§7.7.7, the code path) ----------------------------
+const POINTS = Object.values(index_1.InterceptionPoint);
+const PROFILES = Object.values(index_1.CompositionProfile);
+function isObject(v) {
+    return v !== null && typeof v === "object" && !Array.isArray(v);
+}
+function pointsOf(v) {
+    if (!Array.isArray(v))
+        return undefined;
+    const out = [];
+    for (const p of v) {
+        if (typeof p !== "string" || !POINTS.includes(p))
+            return undefined;
+        out.push(p);
+    }
+    return out;
+}
+function stringsOf(v) {
+    if (!Array.isArray(v))
+        return undefined;
+    const out = [];
+    for (const s of v) {
+        if (typeof s !== "string")
+            return undefined;
+        out.push(s);
+    }
+    return out;
+}
+function isUint(v) {
+    return typeof v === "number" && Number.isInteger(v) && v >= 0;
+}
+/** Rebuild a document through {@link DeclarationBuilder}, member by
+ * member (§7.7.7, the code path). Members the typed setters cannot
+ * express exactly (an unknown member, a wrong type) go through
+ * {@link DeclarationBuilder.raw}, so the result is validated like the
+ * file it came from. Mirrors the Rust runner's `builder_from_value`. */
+function builderFromValue(doc) {
+    let b = index_1.DeclarationBuilder.empty();
+    if (!isObject(doc))
+        return b;
+    for (const [k, v] of Object.entries(doc)) {
+        if (k === "declaration" && typeof v === "string")
+            b = b.version(v);
+        else if (k === "spec" && typeof v === "string")
+            b = b.spec(v);
+        else if (k === "id" && typeof v === "string")
+            b = b.id(v);
+        else if (k === "host" &&
+            isObject(v) &&
+            Object.keys(v).every((hk) => hk === "name" || hk === "version") &&
+            typeof v["name"] === "string" &&
+            (v["version"] === undefined || typeof v["version"] === "string")) {
+            b = b.host(v["name"], v["version"]);
+        }
+        else if (k === "configuration" && isObject(v)) {
+            b = typedConfiguration(b, v) ?? b.raw(k, v);
+        }
+        else if (k === "surface" && isObject(v)) {
+            b = typedSurface(b, v) ?? b.raw(k, v);
+        }
+        else if (k === "bindings" && Array.isArray(v)) {
+            b = typedBindings(b, v) ?? b.raw(k, v);
+        }
+        else if (k === "extensions" && isObject(v)) {
+            for (const [ek, ev] of Object.entries(v))
+                b = b.extension(ek, ev);
+        }
+        else {
+            b = b.raw(k, v);
+        }
+    }
+    return b;
+}
+function typedConfiguration(b, c) {
+    for (const [k, v] of Object.entries(c)) {
+        if (k === "mode" && typeof v === "string") {
+            if (v !== "enforce" && v !== "evaluate_only")
+                return undefined;
+            b = b.mode(v);
+        }
+        else if (k === "composition" && isObject(v)) {
+            const known = ["profile", "on_approval", "on_disagreement", "on_transform_conflict"];
+            if (!Object.keys(v).every((ck) => known.includes(ck)))
+                return undefined;
+            const cfg = { profile: index_1.CompositionProfile.SequentialFirstDeny };
+            const profile = v["profile"];
+            if (profile !== undefined) {
+                if (typeof profile !== "string" || !PROFILES.includes(profile))
+                    return undefined;
+                cfg.profile = profile;
+            }
+            for (const [kk, vv] of Object.entries(v)) {
+                if (kk === "profile")
+                    continue;
+                if (typeof vv !== "string")
+                    return undefined;
+                if (kk === "on_approval" && (vv === "stop" || vv === "resume"))
+                    cfg.on_approval = vv;
+                else if (kk === "on_disagreement" && (vv === "deny" || vv === "approval")) {
+                    cfg.on_disagreement = vv;
+                }
+                else if (kk === "on_transform_conflict" && (vv === "deny" || vv === "approval")) {
+                    cfg.on_transform_conflict = vv;
+                }
+                else
+                    return undefined;
+            }
+            b = b.composition(cfg);
+        }
+        else if (k === "identity_provider" && (v === null || typeof v === "string")) {
+            b = b.identityProvider(v);
+        }
+        else if (k === "approval" && isObject(v)) {
+            for (const [ak, av] of Object.entries(v)) {
+                if (av !== null && typeof av !== "string")
+                    return undefined;
+                if (ak === "resolver")
+                    b = b.approvalResolver(av);
+                else if (ak === "redactor")
+                    b = b.approvalRedactor(av);
+                else
+                    return undefined;
+            }
+        }
+        else if (k === "posture" && isObject(v)) {
+            if (Object.keys(v).length !== 1)
+                return undefined;
+            const p = v["tool_seam_host_error"];
+            if (p !== "continue" && p !== "terminate")
+                return undefined;
+            b = b.toolSeamHostError(p);
+        }
+        else if (k === "timeouts" && isObject(v)) {
+            for (const [tk, tv] of Object.entries(v)) {
+                if (tv !== null && !isUint(tv))
+                    return undefined;
+                if (tk === "interceptor_ms")
+                    b = b.interceptorTimeoutMs(tv);
+                else if (tk === "approval_resolver_ms")
+                    b = b.approvalResolverTimeoutMs(tv);
+                else
+                    return undefined;
+            }
+        }
+        else if (k === "records" && isObject(v)) {
+            if (Object.keys(v).length !== 1)
+                return undefined;
+            const n = v["max_buffered"];
+            if (n !== null && !isUint(n))
+                return undefined;
+            b = b.maxBufferedRecords(n ?? null);
+        }
+        else {
+            return undefined;
+        }
+    }
+    return b;
+}
+function typedSurface(b, sf) {
+    const buffered = sf["buffered_output"];
+    if (buffered !== undefined && typeof buffered !== "boolean")
+        return undefined;
+    const bound = sf["exposure_bound"];
+    if (bound !== undefined && typeof bound !== "string")
+        return undefined;
+    if (buffered !== undefined)
+        b = b.bufferedOutput(buffered, bound);
+    else if (bound !== undefined)
+        return undefined;
+    for (const [k, v] of Object.entries(sf)) {
+        if (k === "interception_points") {
+            const points = pointsOf(v);
+            if (points === undefined)
+                return undefined;
+            b = b.surfacePoints(points);
+        }
+        else if (k === "capabilities") {
+            const caps = stringsOf(v);
+            if (caps === undefined)
+                return undefined;
+            b = b.surfaceCapabilities(caps);
+        }
+        else if (k === "declaration_versions") {
+            const versions = stringsOf(v);
+            if (versions === undefined)
+                return undefined;
+            b = b.surfaceDeclarationVersions(versions);
+        }
+        else if (k === "profiles") {
+            if (!isObject(v))
+                return undefined;
+            for (const [name, knobs] of Object.entries(v)) {
+                if (!PROFILES.includes(name) || !isObject(knobs))
+                    return undefined;
+                const support = {};
+                for (const [knob, values] of Object.entries(knobs)) {
+                    const list = stringsOf(values);
+                    if (list === undefined)
+                        return undefined;
+                    if (knob === "on_approval") {
+                        if (!list.every((x) => x === "stop" || x === "resume"))
+                            return undefined;
+                        support.on_approval = list;
+                    }
+                    else if (knob === "on_disagreement" || knob === "on_transform_conflict") {
+                        if (!list.every((x) => x === "deny" || x === "approval"))
+                            return undefined;
+                        support[knob] = list;
+                    }
+                    else
+                        return undefined;
+                }
+                b = b.surfaceProfile(name, support);
+            }
+        }
+        else if (k === "buffered_output" || k === "exposure_bound") {
+            /* handled above */
+        }
+        else {
+            return undefined;
+        }
+    }
+    return b;
+}
+function typedBindings(b, items) {
+    const known = ["id", "kind", "config", "at", "timeout_ms"];
+    if (items.length === 0) {
+        // The written-down empty-deny host (§7.7.5): `bind` is never
+        // called, so the member is set explicitly.
+        return b.raw("bindings", []);
+    }
+    for (const item of items) {
+        if (!isObject(item))
+            return undefined;
+        if (!Object.keys(item).every((k) => known.includes(k)))
+            return undefined;
+        const id = item["id"];
+        const kind = item["kind"];
+        if (typeof id !== "string" || typeof kind !== "string")
+            return undefined;
+        // An explicit `null` is a value the spec allows; only an absent
+        // member takes the default, as in the Rust runner.
+        const config = "config" in item ? item["config"] : {};
+        let at;
+        if (item["at"] !== undefined) {
+            at = pointsOf(item["at"]);
+            if (at === undefined)
+                return undefined;
+        }
+        let timeout;
+        const t = item["timeout_ms"];
+        if (t === undefined)
+            timeout = undefined;
+        else if (t === null)
+            timeout = null;
+        else if (isUint(t))
+            timeout = t;
+        else
+            return undefined;
+        b = b.bind(id, kind, config, at, timeout);
+    }
+    return b;
+}
+// ---- construction path proof (§7.7.7) -------------------------------------------
+/** Resolve `doc` through the four construction paths (§7.7.7) and
+ * compare: value, JSON text, a temporary file and the builder. Equal
+ * canonical forms, or equal refusal classes, prove the paths
+ * equivalent. Returns `[equivalent, detail]`. */
+async function provePaths(doc, registry, tag) {
+    const text = JSON.stringify(doc);
+    // A private directory, not a guessable name in the shared tmpdir: a
+    // planted symlink there could redirect the write.
+    let dir;
+    let path = "";
+    const resolveWith = (load) => {
+        try {
+            return { ok: true, resolved: (0, declaration_1.resolveDeclaration)(load(), registry) };
+        }
+        catch (e) {
+            return { ok: false, error: e };
+        }
+    };
+    let fromFile;
+    let fileError;
+    try {
+        dir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "agent-hooks-ctk-"));
+        path = (0, node_path_1.join)(dir, `${tag}.json`);
+        (0, node_fs_1.writeFileSync)(path, text, { encoding: "utf8", flag: "wx" });
+        try {
+            fromFile = await index_1.HostDeclaration.fromPath(path);
+        }
+        catch (e) {
+            fileError = e;
+        }
+    }
+    catch (e) {
+        fileError = index_1.DeclarationError.single("declaration_error:unreadable", "", `cannot write temporary file: ${e?.constructor?.name ?? "Error"}`);
+    }
+    finally {
+        if (dir !== undefined) {
+            try {
+                (0, node_fs_1.rmSync)(dir, { recursive: true, force: true });
+            }
+            catch {
+                /* best effort */
+            }
+        }
+    }
+    const outcomes = [
+        ["value", resolveWith(() => index_1.HostDeclaration.fromValue(doc))],
+        ["json", resolveWith(() => index_1.HostDeclaration.fromJson(text))],
+        [
+            "file",
+            fromFile !== undefined
+                ? resolveWith(() => fromFile)
+                : { ok: false, error: fileError },
+        ],
+        ["builder", resolveWith(() => builderFromValue(doc).build())],
+    ];
+    const keyOf = (r) => {
+        if (r.ok)
+            return `ok:${(0, index_1.canonicalDeclaration)(r.resolved)}`;
+        const e = r.error;
+        return `err:${e instanceof index_1.DeclarationError ? e.code : `other:${String(e)}`}`;
+    };
+    const keys = outcomes.map(([, r]) => keyOf(r));
+    if (keys.every((k) => k === keys[0]))
+        return [true, ""];
+    let detail = "";
+    outcomes.forEach(([name, r]) => {
+        detail += `${name}: ${r.ok ? "accepted" : String(r.error)}; `;
     });
+    const oks = keys.filter((k) => k.startsWith("ok:"));
+    if (oks.length >= 2 && oks[0] !== oks[1]) {
+        let pos = 0;
+        while (pos < oks[0].length && pos < oks[1].length && oks[0][pos] === oks[1][pos])
+            pos++;
+        detail += `first difference at byte ${pos}`;
+    }
+    return [false, detail];
+}
+// ---- one vector ---------------------------------------------------------------------
+function fail(v, failures) {
+    return {
+        id: v.id,
+        title: v.title,
+        part: v.part ?? undefined,
+        status: "fail",
+        detail: "",
+        failures,
+    };
+}
+/** The code surface a harness reports (§13.1), or the one derived from
+ * its capabilities and posture. */
+function codeSurfaceOf(harness) {
+    return (harness.hostSurface?.() ??
+        index_1.HostSurface.fromCapabilities(harness.capabilities, harness.toolSeamHostError ?? "continue"));
+}
+/** Resolve the harness's own declaration against its code surface.
+ * Throws {@link DeclarationError} on refusal. {@link runVectors} calls
+ * this once before the first vector, as §7.7.9 asks; {@link runVector}
+ * calls it when no resolved surface is handed in. */
+function assessHarness(harness) {
+    const own = harness.declaration?.();
+    if (own === undefined) {
+        return {
+            capabilities: [...harness.capabilities],
+            posture: harness.toolSeamHostError ?? "continue",
+        };
+    }
+    const resolved = (0, declaration_1.resolveSurfaceOnly)(index_1.HostDeclaration.fromValue(own), codeSurfaceOf(harness));
+    return {
+        capabilities: [...resolved.surface.capabilities],
+        posture: resolved.configuration.posture.tool_seam_host_error,
+    };
 }
-async function runVector(harness, vector) {
+async function runVector(harness, vector, assessed) {
     const v = vector;
     const vectorJson = JSON.stringify(vector);
-    const capsJson = JSON.stringify([...harness.capabilities].sort());
+    // §7.7.9: a harness with its own declaration is assessed against the
+    // resolved document, so what ran is what a claim cites.
+    const codeSurface = codeSurfaceOf(harness);
+    let caps;
+    let posture;
+    try {
+        ({ capabilities: caps, posture } = assessed ?? assessHarness(harness));
+    }
+    catch (e) {
+        return fail(v, [`harness declaration refused: ${e}`]);
+    }
+    const capsJson = JSON.stringify([...caps].sort());
     const skip = JSON.parse(native_1.native.ctkShouldSkip(vectorJson, capsJson));
     if (skip !== null) {
         return {
             id: v.id,
             title: v.title,
+            part: v.part ?? undefined,
             status: "skip",
             detail: skip,
             failures: [],
         };
     }
-    // Multi-interceptor vectors (§7.1 fold-through) use interceptor_scripts;
-    // single-interceptor vectors use interceptor_script. Only the FIRST
-    // interceptor records: expect.interceptions describes each emission as
-    // the first-registered interceptor saw it. An empty interceptor_scripts
-    // registers zero interceptors (§7 fail-closed vector).
-    const scripts = v.interceptor_scripts ?? [v.interceptor_script];
-    const first = scripts.length > 0 ? new RecordingInterceptor(scripts[0]) : null;
-    const interceptors = first ? [first] : [];
-    for (const s of scripts.slice(1))
-        interceptors.push(new ScriptedInterceptor(s));
-    const approval = v.approval_script;
-    // NB: [] is truthy in JS — an empty approval_script registers NO
-    // resolver (matches the Rust/Python runners; exercised by AH-CTK-032).
-    const resolver = approval && approval.length > 0 ? new ScriptedResolver(approval) : null;
+    const scripts = new Scripts(v);
     const mode = (v.mode ?? "enforce");
     // §13.2: composition vectors carry the profile/knobs they apply to;
     // absent means the pre-P-003 default (`sequential/first_deny, stop`).
@@ -132,32 +592,99 @@ async function runVector(harness, vector) {
     const identityProvider = 'identity_provider' in v
         ? v.identity_provider
         : 'jcs-sha256';
-    const redactForApproval = v.redact_for_approval ?? [];
-    harness.setup(v.scenario, interceptors, resolver, mode, composition, identityProvider, redactForApproval);
     let rr;
-    try {
-        rr = await harness.run();
-    }
-    catch (e) {
-        return {
-            id: v.id,
-            title: v.title,
-            status: "fail",
-            detail: "",
-            failures: [`harness.run threw: ${e}`],
-        };
+    const document = v.host_declaration;
+    if (document !== undefined) {
+        // §7.7.9: a declaration vector builds the emitter through the
+        // loader. The runner proves the construction paths itself, then
+        // hands the document and the CTK registry to the harness.
+        const registry = scripts.registry(codeSurface);
+        const [pathsEquivalent, pathDetail] = await provePaths(document, registry, v.id);
+        if (harness.setupDeclared === undefined) {
+            return fail(v, [
+                `harness ${JSON.stringify(harness.name)} declares host_declaration but does not implement setupDeclared`,
+            ]);
+        }
+        let refused = null;
+        try {
+            harness.setupDeclared(v.scenario, document, registry);
+        }
+        catch (e) {
+            if (!(e instanceof index_1.DeclarationError)) {
+                harness.teardown();
+                return fail(v, [`harness.setupDeclared threw a non-declaration error: ${e}`]);
+            }
+            refused = e;
+        }
+        if (refused !== null) {
+            harness.teardown();
+            const load = {
+                outcome: "refused",
+                class: refused.code,
+                paths_equivalent: pathsEquivalent,
+                detail: pathDetail === "" ? refused.message : `${refused.message}; paths: ${pathDetail}`,
+            };
+            rr = {
+                outcome: "error",
+                final_output: null,
+                tool_invocations: [],
+                error: refused.message,
+                identities: [],
+                records: [],
+                load,
+            };
+        }
+        else {
+            try {
+                rr = await harness.run();
+            }
+            catch (e) {
+                return fail(v, [`harness.run threw: ${e}`]);
+            }
+            finally {
+                harness.teardown();
+            }
+            rr.load = {
+                outcome: "accepted",
+                paths_equivalent: pathsEquivalent,
+                ...(pathDetail !== "" ? { detail: pathDetail } : {}),
+            };
+        }
     }
-    finally {
-        harness.teardown();
+    else {
+        harness.setup(v.scenario, scripts.interceptors(), scripts.resolver(), mode, composition, identityProvider, scripts.redact);
+        try {
+            rr = await harness.run();
+        }
+        catch (e) {
+            return fail(v, [`harness.run threw: ${e}`]);
+        }
+        finally {
+            harness.teardown();
+        }
     }
     // §13.1 posture declaration; absent means the spec default.
-    const postures = { tool_seam_host_error: harness.toolSeamHostError ?? "continue" };
-    return JSON.parse(native_1.native.ctkAssert(vectorJson, JSON.stringify(first?.recorded ?? []), runRecordToWire(rr, postures)));
+    const postures = { tool_seam_host_error: posture };
+    return JSON.parse(native_1.native.ctkAssert(vectorJson, JSON.stringify(scripts.recorded()), runRecordToWire(rr, postures)));
 }
 async function runVectors(harnessFactory, vectors) {
     const out = [];
+    // Resolve the harness declaration once, before the first vector
+    // (§7.7.9); a refusal fails every vector with the findings.
+    let assessed;
+    let refusal;
+    try {
+        assessed = assessHarness(harnessFactory());
+    }
+    catch (e) {
+        refusal = e;
+    }
     for (const v of vectors) {
-        out.push(await runVector(harnessFactory(), v));
+        if (assessed === undefined) {
+            out.push(fail(v, [`harness declaration refused: ${refusal}`]));
+            continue;
+        }
+        out.push(await runVector(harnessFactory(), v, assessed));
     }
     return out;
 }