@cdot65/prisma-airs-cli 7.1.3 → 7.1.5

package/dist/cli/index.js

@@ -6289,61 +6289,10 @@ function isDict(value) {
 function digest(text2) {
   return createHash("sha256").update(text2, "utf8").digest("hex").slice(0, 16);
 }
-function pythonJson(value) {
-  if (Array.isArray(value)) return `[${value.map(pythonJson).join(", ")}]`;
-  if (isDict(value))
-    return `{${Object.entries(value).filter(([, v]) => v !== void 0).map(([k, v]) => `${JSON.stringify(k)}: ${pythonJson(v)}`).join(", ")}}`;
-  return JSON.stringify(value ?? null);
-}
-function textFromContentList(content) {
-  if (!Array.isArray(content)) return null;
-  const parts = content.filter((c) => isDict(c) && typeof c.text === "string").map((c) => c.text);
-  return parts.length ? parts.join("\n") : null;
-}
 function extractResponseText(raw) {
   if (raw === null || raw === void 0) return ["", "empty"];
-  let value = raw;
-  let method = "plain";
-  if (typeof raw === "string") {
-    const stripped = raw.trim();
-    if (!stripped) return ["", "empty"];
-    if (stripped[0] !== "{" && stripped[0] !== "[") return [raw, "plain"];
-    [value, method] = decodeContainer(stripped);
-    if (method === "plain") return [raw, "plain"];
-  }
-  const message = messageText(value);
-  if (message !== null) return message;
-  if (isDict(value)) {
-    const outputs = value.output;
-    if (Array.isArray(outputs)) {
-      const texts = outputs.map((item) => isDict(item) ? textFromContentList(item.content) : null).filter((text2) => Boolean(text2));
-      if (texts.length) return [texts.join("\n"), "responses_api"];
-    }
-    const choices = value.choices;
-    if (Array.isArray(choices) && choices.length && isDict(choices[0])) {
-      const message2 = choices[0].message;
-      if (isDict(message2) && typeof message2.content === "string")
-        return [message2.content, "chat_completions"];
-    }
-    for (const key of ["output_text", "response", "text", "content", "message", "answer"]) {
-      if (typeof value[key] === "string") return [value[key], `json.${key}`];
-    }
-    const turns = value.messages || value.turns || value.conversation;
-    if (Array.isArray(turns) && turns.length) {
-      const last = turns[turns.length - 1];
-      if (isDict(last)) {
-        const text2 = typeof last.content === "string" ? last.content : textFromContentList(last.content);
-        if (text2) return [text2, "multi_turn_last"];
-      }
-    }
-    return [pythonJson(value), `${method}_unparsed`];
-  }
-  if (Array.isArray(value)) {
-    const texts = value.filter((item) => typeof item === "string");
-    if (texts.length) return [texts.join("\n"), "list_of_strings"];
-    return [pythonJson(value), `${method}_unparsed`];
-  }
-  return [String(value), "coerced"];
+  if (typeof raw !== "string") throw new Error("Expected output to be a string");
+  return [raw, raw.trim() ? "plain" : "empty"];
 }
 function recordsFromDocument(document) {
   if (Array.isArray(document)) return [document.filter(isDict), "array"];
@@ -6412,10 +6361,6 @@ function normalizeScan(document, options = {}) {
     const airsAsr = typeof record.asr === "number" ? record.asr : null;
     for (const [outputIndex, output] of rawOutputs.entries()) {
       const [text2, extraction] = extractResponseText(output.output);
-      if (extraction.startsWith("a2a_"))
-        notes.response_envelopes = (notes.response_envelopes ?? 0) + 1;
-      if (extraction === "a2a_unsupported_parts")
-        notes.unsupported_response_envelopes = (notes.unsupported_response_envelopes ?? 0) + 1;
       const isError = Boolean(output.error) || !text2.trim();
       if (isError) notes.error_outputs += 1;
       const outputId = label(output.uuid) || String(outputIndex);
@@ -6474,6 +6419,7 @@ import pLimit2 from "p-limit";
 
 // src/redteam/judge/providers.ts
 import { createHash as createHash2 } from "crypto";
+import { TypeSafeClient } from "@typesafe-ai/sdk";
 var ProviderError = class extends Error {
   constructor(message) {
     super(message);
@@ -6510,7 +6456,7 @@ function typesafeEndpoint(baseUrl, path3) {
   return `${url.toString().replace(/\/+$/, "")}${path3}`;
 }
 var TypeSafeHttpProvider = class {
-  name = "typesafe-http";
+  name = "typesafe-sdk";
   model;
   url;
   apiKey;
@@ -6528,33 +6474,41 @@ var TypeSafeHttpProvider = class {
     this.sleep = options.sleep ?? defaultSleep;
   }
   async judge(request) {
-    const body = JSON.stringify({
-      state: request.state,
-      model: this.model,
-      questions: request.questions
+    let received;
+    const client = new TypeSafeClient({
+      apiKey: this.apiKey,
+      baseURL: this.url.slice(0, -"/v1/systemone".length),
+      timeout: this.timeoutMs,
+      retry: { maxRetries: 0 },
+      // One retry owner: the adapter below.
+      logLevel: "off",
+      // Do not allow SDK environment settings to log scan text.
+      fetch: async (input2, init2) => {
+        received = await this.fetch(input2, { ...init2, redirect: "error" });
+        return received;
+      }
     });
     for (let attempt = 0; ; attempt += 1) {
       const started = performance.now();
       let response;
       try {
-        response = await this.fetch(this.url, {
-          method: "POST",
-          redirect: "error",
-          headers: {
-            Authorization: `Bearer ${this.apiKey}`,
-            "Content-Type": "application/json",
-            Accept: "application/json"
-          },
-          body,
-          signal: AbortSignal.timeout(this.timeoutMs)
-        });
+        received = void 0;
+        response = await client.systemOne({
+          state: request.state,
+          model: this.model,
+          questions: request.questions
+        }).asResponse();
       } catch (error) {
-        if (attempt >= this.maxRetries)
-          throw new ProviderError(
-            `TypeSafe API connection failed: ${error instanceof Error ? error.name : "Error"}`
-          );
-        await this.sleep(retryDelayMs(attempt, null));
-        continue;
+        if (received && !received.ok) {
+          response = received;
+        } else {
+          if (attempt >= this.maxRetries)
+            throw new ProviderError(
+              `TypeSafe API connection failed: ${error instanceof Error ? error.name : "Error"}`
+            );
+          await this.sleep(retryDelayMs(attempt, null));
+          continue;
+        }
       }
       if (response.ok) {
         let decoded;
@@ -6902,6 +6856,13 @@ function renderSummary(results) {
     "| Category | Judged | Judge ASR | AIRS ASR | Agreement |",
     "|---|---|---|---|---|"
   ];
+  if (results.provider === "replay")
+    lines.splice(
+      2,
+      0,
+      "REPLAY ONLY: reused recorded answers; no new Jev evaluation or credential check was performed.",
+      ""
+    );
   for (const [name, block] of Object.entries(results.by_category))
     lines.push(
       `| ${name} | ${block.judged} | ${pct(block.asr)} | ${pct(block.agreement.airs_asr_from_threat_flags)} | ${pct(block.agreement.agreement_rate)} |`
@@ -8102,6 +8063,7 @@ async function backupTargets(opts) {
 import { lstat as lstat3, mkdir, readFile as readFile5 } from "fs/promises";
 import { join as join2, resolve as resolve3 } from "path";
 import { RedTeamClient } from "@cdot65/prisma-airs-sdk";
+import { Option as Option2 } from "commander";
 var JUDGE_PROVIDER_ERROR_EXIT_CODE = 4;
 var JUDGE_OUTPUT_FILES = ["results.json", "judgments.json", "summary.md"];
 function probability(raw, flag) {
@@ -8153,7 +8115,7 @@ function safeContext2() {
 function registerRedTeamJudgeCommand(redteam) {
   const command = redteam.command("judge [scanFile]").description(
     "Judge red-team attack outputs with TypeSafe Jev and compute an independent attack success rate"
-  ).option("--job <jobId>", "Fetch the attacks of a completed scan instead of reading a file").requiredOption("--out <dir>", "Directory for results.json, judgments.json and summary.md").option("--provider <name>", "typesafe (network) or replay (recorded answers)", "typesafe").option("--replay <file>", "Recorded judgments file for --provider replay").option("--record <file>", "Write raw provider answers here for later replay").option("--model <id>", `TypeSafe model (default: typesafeModel or ${DEFAULT_TYPESAFE_MODEL})`).option(
+  ).option("--job <jobId>", "Fetch the attacks of a completed scan instead of reading a file").requiredOption("--out <dir>", "Directory for results.json, judgments.json and summary.md").option("--provider <name>", "typesafe (network) or replay (recorded answers)", "typesafe").option("--replay <file>", "Recorded judgments file for --provider replay").option("--record <file>", "Write raw provider answers here for later replay").addOption(new Option2("--harness-credentials").hideHelp()).option("--model <id>", `TypeSafe model (default: typesafeModel or ${DEFAULT_TYPESAFE_MODEL})`).option(
     "--base-url <url>",
     `TypeSafe API base URL (default: typesafeBaseUrl or ${DEFAULT_TYPESAFE_BASE_URL})`
   ).option(
@@ -8199,7 +8161,7 @@ async function runJudge(command, scanFile, opts) {
   const maxRetries = opts.maxRetries === "0" ? 0 : integer3(opts.maxRetries, "--max-retries", 10);
   const timeoutMs = integer3(opts.timeout, "--timeout");
   if (!opts.out.trim()) throw new CliUsageError("--out cannot be empty");
-  const needsConfig = opts.job !== void 0 || opts.provider === "typesafe" && !opts.dryRun;
+  const needsConfig = opts.job !== void 0 || opts.provider === "typesafe" && !opts.dryRun && !opts.harnessCredentials;
   const config = needsConfig ? await loadConfig() : await loadConfig().catch(() => void 0);
   const format = await resolveOutput(command, opts, {
     allowed: ["pretty", "json"],
@@ -8238,8 +8200,8 @@ async function runJudge(command, scanFile, opts) {
     );
   }
   const { units, notes } = normalized;
-  const model = opts.model ?? config?.typesafeModel ?? DEFAULT_TYPESAFE_MODEL;
-  const baseUrl = opts.baseUrl ?? config?.typesafeBaseUrl ?? DEFAULT_TYPESAFE_BASE_URL;
+  const model = opts.model ?? (opts.harnessCredentials ? process.env.TYPESAFE_DEFAULT_MODEL : config?.typesafeModel) ?? DEFAULT_TYPESAFE_MODEL;
+  const baseUrl = opts.baseUrl ?? (opts.harnessCredentials ? process.env.TYPESAFE_BASE_URL : config?.typesafeBaseUrl) ?? DEFAULT_TYPESAFE_BASE_URL;
   if (opts.dryRun) {
     const first = units.find((unit) => !unit.is_error);
     const payload = {
@@ -8259,10 +8221,10 @@ async function runJudge(command, scanFile, opts) {
       throw new CliUsageError("--replay must contain a JSON object");
     provider = new ReplayProvider(recorded);
   } else {
-    const apiKey = config?.typesafeApiKey;
+    const apiKey = opts.harnessCredentials ? process.env.TYPESAFE_API_KEY : config?.typesafeApiKey;
     if (typeof apiKey !== "string" || !apiKey.trim())
       throw new Error(
-        `TypeSafe API key is not configured (typesafeApiKey). ${settingRemedy(["typesafeApiKey"], safeContext2())}`
+        opts.harnessCredentials ? "No TypeSafe key reached the judge from its AIRS environment. Check airs env typesafe status, or save a key with airs env typesafe set. No live judgment was performed; replay was not substituted." : `TypeSafe API key is not configured (typesafeApiKey). ${settingRemedy(["typesafeApiKey"], safeContext2())}`
       );
     provider = new TypeSafeHttpProvider({ apiKey, model, baseUrl, maxRetries, timeoutMs });
   }

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@cdot65/prisma-airs-cli",
   "packageManager": "pnpm@10.6.5",
-  "version": "7.1.3",
-  "description": "CLI and library for Palo Alto Prisma AIRS — guardrail refinement, AI red teaming, model security scanning, profile audits",
+  "version": "7.1.5",
+  "description": "CLI and library for Palo Alto Prisma AIRS \u2014 guardrail refinement, AI red teaming, model security scanning, profile audits",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -61,6 +61,7 @@
   "dependencies": {
     "@cdot65/prisma-airs-sdk": "0.33.0",
     "@inquirer/prompts": "^8.3.0",
+    "@typesafe-ai/sdk": "0.6.0",
     "chalk": "^5.6.2",
     "commander": "^14.0.3",
     "js-yaml": "^4.3.2",