@skill-harness/core 0.11.0 → 0.13.0

package/dist/qualification-capture.js

@@ -0,0 +1,163 @@
+import { spawn } from "node:child_process";
+import { closeSync, constants, fstatSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs";
+import { join } from "node:path";
+import { StringDecoder } from "node:string_decoder";
+import { redactText } from "./capture.js";
+import { qualificationSha256 } from "./qualification-config.js";
+export function redactQualificationOutput(text, secretValues = []) {
+    let output = text;
+    for (const secret of [...new Set(secretValues)].sort((left, right) => right.length - left.length)) {
+        if (secret.length >= 8)
+            output = output.split(secret).join("[REDACTED credential]");
+    }
+    output = output
+        .replace(/\b([A-Z][A-Z0-9_]*(?:API_KEY|ACCESS_KEY|SECRET_KEY|AUTH_TOKEN|BEARER_TOKEN|SESSION_TOKEN))\s*[:=]\s*[^\s,;"']+/gi, "$1=[REDACTED credential]")
+        .replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED credential]")
+        .replace(/\b(?:sk|sess|key|token)-[A-Za-z0-9._-]{8,}/gi, "[REDACTED credential]")
+        .replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "[REDACTED credential]");
+    return redactText(output);
+}
+export function openQualificationBoundedCapture(path, limit, redact) {
+    const fd = openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
+    const decoder = new StringDecoder("utf8");
+    let pending = "";
+    let acceptedRawBytes = 0;
+    let closed = false;
+    const persist = (text) => {
+        if (!text)
+            return;
+        const sanitized = Buffer.from(redact(text), "utf8");
+        const remaining = Math.max(0, limit - state.captured_bytes);
+        const part = sanitized.subarray(0, remaining);
+        if (part.length > 0) {
+            writeSync(fd, part);
+            state.captured_bytes += part.length;
+            fsyncSync(fd);
+        }
+        if (sanitized.length > remaining)
+            state.truncated = true;
+    };
+    const flushLines = () => {
+        let newline = pending.indexOf("\n");
+        while (newline >= 0) {
+            persist(pending.slice(0, newline + 1));
+            pending = pending.slice(newline + 1);
+            newline = pending.indexOf("\n");
+        }
+    };
+    const state = {
+        path, total_bytes: 0, captured_bytes: 0, truncated: false,
+        write(chunk) {
+            state.total_bytes += chunk.length;
+            const remainingRaw = Math.max(0, limit - acceptedRawBytes);
+            const accepted = chunk.subarray(0, remainingRaw);
+            acceptedRawBytes += accepted.length;
+            if (accepted.length > 0) {
+                pending += decoder.write(accepted);
+                flushLines();
+            }
+            if (chunk.length > remainingRaw)
+                state.truncated = true;
+        },
+        close() {
+            if (closed)
+                return;
+            pending += decoder.end();
+            // If the raw limit split a line, discard that incomplete tail rather than
+            // persisting a credential prefix that cannot be matched as a whole value.
+            if (!state.truncated)
+                persist(pending);
+            pending = "";
+            fsyncSync(fd);
+            closeSync(fd);
+            closed = true;
+        },
+    };
+    return state;
+}
+export function qualificationOutputReceipt(spoolRoot, path, capture) {
+    const bytes = readFileSync(path);
+    return {
+        path: path.slice(spoolRoot.length + 1),
+        total_bytes: capture.total_bytes,
+        captured_bytes: bytes.length,
+        truncated: capture.truncated,
+        sha256: qualificationSha256(bytes),
+    };
+}
+export function verifyQualificationOutputReceipt(spoolRoot, receipt) {
+    const path = join(spoolRoot, receipt.path);
+    let fd;
+    try {
+        fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+        const stat = fstatSync(fd);
+        if (!stat.isFile())
+            throw new Error(`qualification output ${receipt.path} must be a regular non-symlink file`);
+        const bytes = readFileSync(fd);
+        if (bytes.length !== receipt.captured_bytes || qualificationSha256(bytes) !== receipt.sha256) {
+            throw new Error(`qualification output ${receipt.path} digest/size mismatch`);
+        }
+    }
+    finally {
+        if (fd !== undefined)
+            closeSync(fd);
+    }
+}
+export function attestQualificationPiJsonl(text, requested) {
+    const records = [];
+    try {
+        for (const line of text.split("\n")) {
+            if (!line.trim())
+                continue;
+            const value = JSON.parse(line);
+            if (!plainObject(value))
+                throw new Error("event is not an object");
+            records.push(value);
+        }
+    }
+    catch {
+        return { ok: false, actual: null, fallback: false, refused: false, error: "artifact is not valid JSONL" };
+    }
+    const fallback = records.some((record) => record.type === "provider_fallback" || record.type === "model_fallback" || Object.hasOwn(record, "fallback_provider") || Object.hasOwn(record, "fallback_model"));
+    const messages = records
+        .filter((record) => record.type === "message_end" && plainObject(record.message) && record.message.role === "assistant")
+        .map((record) => record.message);
+    if (messages.length === 0)
+        return { ok: false, actual: null, fallback, refused: false, error: "artifact has no authoritative assistant provider/model identity" };
+    const identities = messages.map((message) => ({ provider: stringValue(message.provider), model: stringValue(message.model) }));
+    if (identities.some((identity) => !identity.provider || !identity.model))
+        return { ok: false, actual: null, fallback, refused: false, error: "artifact assistant identity is missing provider or model" };
+    const actual = { provider: identities.at(-1).provider, model: identities.at(-1).model };
+    if (fallback)
+        return { ok: false, actual, fallback: true, refused: false, error: "artifact reports provider/model fallback" };
+    if (identities.some((identity) => identity.provider !== requested.provider))
+        return { ok: false, actual, fallback: false, refused: false, error: `provider substitution: requested ${requested.provider}, observed ${actual.provider}` };
+    if (identities.some((identity) => identity.model !== requested.model))
+        return { ok: false, actual, fallback: false, refused: false, error: `model substitution: requested ${requested.model}, observed ${actual.model}` };
+    const terminal = messages.at(-1);
+    const errorMessage = stringValue(terminal.errorMessage) ?? "";
+    const refused = terminal.stopReason === "error" && /(?:refus|not supported|not available|subscription|usage limit|unauthor)/i.test(errorMessage);
+    if (terminal.stopReason === "error" && !refused)
+        return { ok: false, actual, fallback: false, refused: false, error: errorMessage || "assistant ended with an error" };
+    return { ok: true, actual, fallback: false, refused, error: refused ? errorMessage : null };
+}
+export async function spawnQualificationCapture(command, argv, options) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, argv, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"] });
+        let stdout = Buffer.alloc(0), stderr = Buffer.alloc(0), timedOut = false;
+        const append = (current, chunk) => Buffer.concat([current, chunk]).subarray(0, options.output_limit_bytes);
+        child.stdout.on("data", (chunk) => { stdout = append(stdout, chunk); });
+        child.stderr.on("data", (chunk) => { stderr = append(stderr, chunk); });
+        const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, options.timeout_ms);
+        child.once("error", (error) => { clearTimeout(timer); reject(error); });
+        child.once("close", (code) => { clearTimeout(timer); resolve({ stdout: stdout.toString("utf8"), stderr: stderr.toString("utf8"), code, timed_out: timedOut }); });
+    });
+}
+function plainObject(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+function stringValue(value) { return typeof value === "string" ? value : undefined; }
+//# sourceMappingURL=qualification-capture.js.map

package/dist/qualification-config.d.ts

@@ -0,0 +1,233 @@
+import { QUALIFICATION_JUDGE_PANEL_POLICY } from "./qualification-panels.js";
+export declare const QUALIFICATION_CONFIG_VERSION: "qualification-config-v1";
+export declare const QUALIFICATION_RUNNER_VERSION: "qualification-runner-v1";
+export declare const QUALIFICATION_REQUEST_VERSION: "qualification-invocation-request-v1";
+export declare const QUALIFICATION_BOARD_VERSION: "qualification-board-v1";
+export declare const QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3: "qualification-terminal-receipt-v3";
+/** Historical configurations omit this field and retain the original closed policy. */
+export declare const QUALIFICATION_OAUTH_DIRECTORY_POLICY_V1: "qualification-oauth-directory-policy-v1";
+/** New qualification configurations select this policy explicitly. */
+export declare const QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2: "qualification-oauth-directory-policy-v2";
+export type QualificationOAuthDirectoryPolicy = typeof QUALIFICATION_OAUTH_DIRECTORY_POLICY_V1 | typeof QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2;
+export declare const PRINCIPAL_QUALIFICATION_PRODUCT_PIN: {
+    readonly repository: "https://github.com/mojomanyana/principal-pi-skills";
+    readonly commit: "a6596950d64a3a525f95329d5dbd3e38948be408";
+    readonly tree: "960359d69deb6f216724b86e13eef67e2f6a6aa1";
+    readonly package_sha256: "3677f76fb31dcaf6a28c9e8b9cf9d6358f998b78bd370f0a0f250d05e136fcc8";
+    readonly package_bytes: 110168;
+};
+/** Immutable pi-daddy ledger-v3 producer merge head. */
+export declare const PI_DADDY_QUALIFICATION_PRODUCER_PIN: {
+    readonly repository: "https://github.com/mojomanyana/pi-daddy";
+    readonly commit: "4a9524394ca995fd74ed9bbb836dc4e73cda3b8c";
+    readonly tree: "7c006bff213142634f0f911ba9bd6add363ecaae";
+    readonly version: "0.21.1";
+    readonly ledger_version: 3;
+    readonly ledger_schema_sha256: "64e3d875e74bc32fa43fb96892605548259cd16f6ed6678646d73cc56280c511";
+};
+export declare const QUALIFICATION_ACCOUNTING_POLICY: {
+    readonly wave_a: {
+        readonly subject: 54;
+        readonly judge: 54;
+    };
+    readonly complete_program: {
+        readonly subject: 642;
+        readonly judge: 642;
+    };
+    readonly ceilings: {
+        readonly subject: 700;
+        readonly judge: 700;
+    };
+    readonly initial: {
+        readonly subject: 0;
+        readonly judge: 0;
+    };
+};
+/** Prospective call ceilings for the universal two-plus-conditional-third panel. */
+export declare const QUALIFICATION_PANEL_ACCOUNTING_POLICY: {
+    readonly wave_a: {
+        readonly subject: 54;
+        readonly judge: 162;
+    };
+    readonly complete_program: {
+        readonly subject: 642;
+        readonly judge: 1926;
+    };
+    readonly ceilings: {
+        readonly subject: 700;
+        readonly judge: 2100;
+    };
+    readonly initial: {
+        readonly subject: 0;
+        readonly judge: 0;
+    };
+};
+export type QualificationAccountingPolicy = typeof QUALIFICATION_ACCOUNTING_POLICY | typeof QUALIFICATION_PANEL_ACCOUNTING_POLICY;
+export type QualificationRole = "holdout-author" | "holdout-reviewer" | "subject" | "judge" | "calibration" | "canary";
+export type QualificationArmKind = "subject" | "judge";
+export type QualificationConfigMode = "production" | "test";
+export type QualificationConflictPolicy = "refuse" | "remove-and-record";
+export interface QualificationExecutablePin {
+    path: string;
+    sha256: string;
+}
+export interface QualificationRepositoryPin {
+    repository: string;
+    commit: string;
+    tree: string;
+}
+export interface QualificationProductPin extends QualificationRepositoryPin {
+    checkout_path: string;
+    package_path: string;
+    package_sha256: string;
+    package_bytes: number;
+}
+export interface QualificationEnginePin extends QualificationRepositoryPin {
+    checkout_path: string;
+    package_paths: {
+        core: string;
+        adapters: string;
+        cli: string;
+        meta: string;
+    };
+    package_sha256: {
+        core: string;
+        adapters: string;
+        cli: string;
+        meta: string;
+    };
+}
+export interface QualificationProducerPin extends QualificationRepositoryPin {
+    checkout_path: string;
+    version: string;
+    ledger_version: 3;
+    ledger_schema_sha256: string;
+}
+export interface QualificationResourcePin {
+    kind: "extension" | "skill" | "system-prompt";
+    path: string;
+    sha256: string;
+}
+export interface QualificationArmV1 {
+    id: string;
+    kind: QualificationArmKind;
+    provider: string;
+    model: string;
+    authentication: "chatgpt-oauth" | "test-oauth";
+    executable: QualificationExecutablePin;
+    resources: QualificationResourcePin[];
+    arguments: string[];
+    allowed_environment_names: string[];
+    timeout_ms: number;
+    output_limit_bytes: number;
+    artifact: {
+        type: "pi-jsonl";
+        relative_path_template: "artifacts/{invocation_id}.jsonl";
+    };
+    fallback: false;
+    metered_override: false;
+}
+export interface QualificationBoardV1 {
+    schema_version: typeof QUALIFICATION_BOARD_VERSION;
+    read_only: true;
+    cells: {
+        id: string;
+        scenario_id: string;
+        measurement_identity_sha256: string;
+        scenario_version: string;
+        stimulus_sha256: string;
+        rubric_sha256: string;
+        subject_input_sha256: string;
+        subject_arm: string;
+        judge_arms: [string, string, string];
+        panels: {
+            id: string;
+            repetition: number;
+        }[];
+        critical: boolean;
+        pass_threshold: number;
+    }[];
+}
+export interface QualificationConfigV1 {
+    schema_version: typeof QUALIFICATION_CONFIG_VERSION;
+    /** Omission is the historical v1 policy and is preserved in canonical digests. */
+    oauth_directory_policy?: typeof QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2;
+    /** Omission preserves historical v1/v2 terminal-receipt selection. */
+    terminal_receipt_version?: typeof QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3;
+    /** Omission preserves the historical single-draw judge methodology. */
+    judge_panel?: typeof QUALIFICATION_JUDGE_PANEL_POLICY;
+    /** Externally approved board authority; required with judge_panel. */
+    board?: QualificationBoardV1;
+    mode: QualificationConfigMode;
+    product: QualificationProductPin;
+    engine: QualificationEnginePin;
+    producer: QualificationProducerPin;
+    runner: {
+        version: typeof QUALIFICATION_RUNNER_VERSION;
+        executable: QualificationExecutablePin;
+        conflicting_parent_environment: QualificationConflictPolicy;
+    };
+    accounting: QualificationAccountingPolicy;
+    arms: QualificationArmV1[];
+}
+export interface QualificationInvocationRequestV1 {
+    schema_version: typeof QUALIFICATION_REQUEST_VERSION;
+    measurement_identity_sha256: string;
+    invocation_id: string;
+    continuation_authority_sha256: string;
+    continuation_authority_expires_at: string;
+    scenario: {
+        id: string;
+        version: string;
+        stimulus_sha256: string;
+        rubric_sha256: string;
+        input_path: string;
+        input_sha256: string;
+        working_directory: string;
+    };
+    role: QualificationRole;
+    counts_as_measurement: boolean;
+    arms: {
+        subject: string;
+        judge: string;
+    };
+    selected_arm: string;
+    repetition: number;
+    /** Present only for judge calls under the prospective panel policy. */
+    panel?: {
+        id: string;
+        member_ordinal: 1 | 2 | 3;
+        subject_invocation_id: string;
+        subject_artifact_sha256: string;
+    };
+}
+/**
+ * Names that can alter provider credentials/routing or inject code into the child.
+ * Qualification children start from an allow-list regardless; this predicate also
+ * makes a dirty parent shell observable by NAME without ever retaining its value.
+ */
+export declare function isQualificationConflictingEnvironmentName(name: string): boolean;
+export declare function sanitizeQualificationEnvironment(parent: NodeJS.ProcessEnv, allowedNames: readonly string[], conflictPolicy: QualificationConflictPolicy): {
+    env: NodeJS.ProcessEnv;
+    removed_names: string[];
+};
+export declare function qualificationCanonicalJson(value: unknown): string;
+export declare function qualificationSha256(value: string | Buffer): string;
+export declare function qualificationConfigDigest(config: QualificationConfigV1): string;
+export declare function qualificationOAuthDirectoryPolicy(config: QualificationConfigV1): QualificationOAuthDirectoryPolicy;
+export declare function verifyQualificationPins(config: QualificationConfigV1): void;
+export declare function verifyQualificationResource(pin: QualificationResourcePin): {
+    realpath: string;
+    bytes: number;
+    sha256: string;
+};
+export declare function verifyQualificationExecutable(pin: QualificationExecutablePin): {
+    realpath: string;
+    bytes: number;
+    sha256: string;
+    device: number;
+    inode: number;
+    mtime_ms: number;
+};
+export declare function parseQualificationConfig(value: unknown): QualificationConfigV1;
+export declare function parseQualificationRequest(value: unknown, config: QualificationConfigV1): QualificationInvocationRequestV1;