@prooflane/inspector-beta 0.1.0-beta.3 → 0.1.0-beta.6

package/dist/server.mjs

@@ -4478,11 +4478,17 @@ function isSecurityTaskEvaluator(value) {
   let v = value;
   return (v.severity === "critical" || v.severity === "high" || v.severity === "medium" || v.severity === "low") && (v.protectedTokens === void 0 || Array.isArray(v.protectedTokens) && v.protectedTokens.every((token) => typeof token == "string")) && (v.denyToolPattern === void 0 || typeof v.denyToolPattern == "string");
 }
+function isHostedRagCaseManifest(value) {
+  if (!value || typeof value != "object")
+    return !1;
+  let v = value, integer = (candidate) => typeof candidate == "number" && Number.isInteger(candidate) && candidate >= 0;
+  return typeof v.caseId == "string" && typeof v.definitionHash == "string" && (v.mode === "retriever-only" || v.mode === "full-chain") && typeof v.useAgentPick == "boolean" && typeof v.expectedToolConfigured == "boolean" && integer(v.expectedKeywordCount) && integer(v.forbiddenKeywordCount) && integer(v.contractAssertionCount) && integer(v.minResults) && (v.maxLatencyMs === void 0 || typeof v.maxLatencyMs == "number" && Number.isFinite(v.maxLatencyMs) && v.maxLatencyMs >= 0) && (v.expectedOutcome === void 0 || v.expectedOutcome === "SUCCESS" || v.expectedOutcome === "DENIAL" || v.expectedOutcome === "EMPTY") && integer(v.expectedAnswerCount) && typeof v.goldenEnabled == "boolean" && typeof v.hasGolden == "boolean" && integer(v.faultCount) && (v.judgeStrategy === "deterministic" || v.judgeStrategy === "same-model" || v.judgeStrategy === "independent");
+}
 function isIntelligenceTask(value) {
   if (!value || typeof value != "object")
     return !1;
   let v = value;
-  return typeof v.id != "string" || typeof v.label != "string" || typeof v.kind != "string" || !securityTaskKinds.has(v.kind) ? !1 : v.kind === "security.prompt" ? typeof v.prompt == "string" && (v.system === void 0 || typeof v.system == "string") && (v.evaluator === void 0 || isSecurityTaskEvaluator(v.evaluator)) : v.kind === "security.conversation" ? Array.isArray(v.messages) && v.messages.length > 0 && v.messages.every((message2) => typeof message2 == "string") && (v.system === void 0 || typeof v.system == "string") && isSecurityTaskEvaluator(v.evaluator) : v.kind === "security.adaptive" ? typeof v.objective == "string" && typeof v.successCriterion == "string" && typeof v.maxRounds == "number" && Number.isInteger(v.maxRounds) && v.maxRounds > 0 && v.maxRounds <= 10 && (v.severity === "critical" || v.severity === "high" || v.severity === "medium" || v.severity === "low") : v.kind === "security.surface" ? (v.severity === "critical" || v.severity === "high" || v.severity === "medium" || v.severity === "low") && (v.source === "tools" || v.source === "resources" || v.source === "prompts" || v.source === "connection") && (v.assertion === "no-match" || v.assertion === "unique-tool-names" || v.assertion === "bounded-input" || v.assertion === "guard-declared" || v.assertion === "resource-content-clean" || v.assertion === "http-header-guard") && (v.patterns === void 0 || !!v.patterns && typeof v.patterns == "object") && (v.resourceUris === void 0 || Array.isArray(v.resourceUris) && v.resourceUris.every((uri) => typeof uri == "string")) && (v.useAdvertisedResources === void 0 || typeof v.useAdvertisedResources == "boolean") && (v.matchResult === void 0 || v.matchResult === "exposed" || v.matchResult === "poisoned" || v.matchResult === "inconclusive") : typeof v.query == "string" && typeof v.minResults == "number" && typeof v.maxLatencyMs == "number";
+  return typeof v.id != "string" || typeof v.label != "string" || typeof v.kind != "string" || !securityTaskKinds.has(v.kind) ? !1 : v.kind === "security.prompt" ? typeof v.prompt == "string" && (v.system === void 0 || typeof v.system == "string") && (v.evaluator === void 0 || isSecurityTaskEvaluator(v.evaluator)) : v.kind === "security.conversation" ? Array.isArray(v.messages) && v.messages.length > 0 && v.messages.every((message2) => typeof message2 == "string") && (v.system === void 0 || typeof v.system == "string") && isSecurityTaskEvaluator(v.evaluator) : v.kind === "security.adaptive" ? typeof v.objective == "string" && typeof v.successCriterion == "string" && typeof v.maxRounds == "number" && Number.isInteger(v.maxRounds) && v.maxRounds > 0 && v.maxRounds <= 10 && (v.severity === "critical" || v.severity === "high" || v.severity === "medium" || v.severity === "low") : v.kind === "security.surface" ? (v.severity === "critical" || v.severity === "high" || v.severity === "medium" || v.severity === "low") && (v.source === "tools" || v.source === "resources" || v.source === "prompts" || v.source === "connection") && (v.assertion === "no-match" || v.assertion === "unique-tool-names" || v.assertion === "bounded-input" || v.assertion === "guard-declared" || v.assertion === "resource-content-clean" || v.assertion === "http-header-guard") && (v.patterns === void 0 || !!v.patterns && typeof v.patterns == "object") && (v.resourceUris === void 0 || Array.isArray(v.resourceUris) && v.resourceUris.every((uri) => typeof uri == "string")) && (v.useAdvertisedResources === void 0 || typeof v.useAdvertisedResources == "boolean") && (v.matchResult === void 0 || v.matchResult === "exposed" || v.matchResult === "poisoned" || v.matchResult === "inconclusive") : typeof v.query == "string" && typeof v.minResults == "number" && typeof v.maxLatencyMs == "number" && (v.localCase === void 0 || isHostedRagCaseManifest(v.localCase));
 }
 function isScanTaskLeaseClaims(value) {
   if (!value || typeof value != "object")
@@ -11212,7 +11218,276 @@ function describeToolError(output) {
   })();
   return text6.length > 500 ? `${text6.slice(0, 500)}\u2026` : text6 || "the tool reported an error";
 }
-var sha256 = (value) => createHash15("sha256").update(typeof value == "string" ? value : JSON.stringify(value)).digest("hex");
+var sha256 = (value) => createHash15("sha256").update(typeof value == "string" ? value : JSON.stringify(value)).digest("hex"), hostedRagCaseQuery = (testCase) => (testCase.userQuery ?? "").trim() || testCase.query, hostedRagCaseName = (testCase) => (testCase.name ?? "").trim() || hostedRagCaseQuery(testCase) || testCase.id, hostedRagCaseInput = (testCase) => testCase.input ?? { query: hostedRagCaseQuery(testCase) }, clampRagScore = (value) => Math.max(0, Math.min(100, Math.round(Number.isFinite(value) ? value : 0))), averageRagScore = (values) => values.length ? Math.round(values.reduce((sum, value) => sum + value, 0) / values.length) : 100;
+function extractHostedRagPayload(output) {
+  let value = output;
+  if (Array.isArray(value) && value.length && value.every((block) => block && typeof block == "object" && "type" in block)) {
+    let text6 = value.filter((block) => block.type === "text" && typeof block.text == "string").map((block) => block.text).join(`
+`);
+    text6 && (value = text6);
+  }
+  if (typeof value == "string") {
+    let trimmed = value.trim();
+    if (trimmed.startsWith("{") || trimmed.startsWith("["))
+      try {
+        value = JSON.parse(trimmed);
+      } catch {
+      }
+  }
+  return value;
+}
+function hostedRagContext(value) {
+  if (value == null)
+    return "";
+  if (typeof value == "string")
+    return value;
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+function hostedRagCount(value) {
+  if (Array.isArray(value))
+    return value.length;
+  if (value && typeof value == "object") {
+    for (let nested of Object.values(value))
+      if (Array.isArray(nested))
+        return nested.length;
+    return 1;
+  }
+  return value ? 1 : 0;
+}
+var HOSTED_RAG_ABSTAIN = /\b(i (don'?t|do not) (know|have)|not enough (context|information|data)|no (relevant |sufficient )?(information|context|data|results?)|cannot (answer|find|determine|provide)|can'?t (answer|find|determine)|unable to (answer|find|determine)|isn'?t (in|available|provided)|not (available|found|provided|present|included) (in|within)|no answer|nothing (in|was) (the )?(context|retrieved))\b/i, HOSTED_RAG_STOP_WORDS = /* @__PURE__ */ new Set(["about", "after", "against", "also", "another", "before", "between", "could", "does", "from", "have", "into", "just", "should", "that", "their", "there", "these", "they", "this", "through", "under", "what", "when", "where", "which", "while", "with", "would", "your"]);
+function hostedRagContextLacksSupport(query, context, expectedAnswer) {
+  let normalized = context.toLowerCase(), expected = (expectedAnswer ?? []).map((term) => term.toLowerCase()).filter(Boolean);
+  if (expected.length)
+    return expected.filter((term) => normalized.includes(term)).length / expected.length < 0.5;
+  let terms = [...new Set(query.toLowerCase().split(/[^a-z0-9]+/).filter((term) => term.length > 3 && !HOSTED_RAG_STOP_WORDS.has(term)))];
+  return terms.length > 0 && terms.filter((term) => normalized.includes(term)).length / terms.length < 0.5;
+}
+function hostedRagMalformed(value) {
+  if (typeof value == "string")
+    return `${value.slice(0, Math.max(2, Math.floor(value.length / 2)))}\uFFFD\u2049`;
+  if (Array.isArray(value))
+    return value.length ? value.slice(0, Math.max(1, value.length - 1)).map((item, index) => index % 2 ? item : hostedRagMalformed(item)) : [{ _malformed: "\uFFFD\u2049" }];
+  if (value && typeof value == "object") {
+    let entries = Object.entries(value), output = {};
+    for (let [key, nested] of entries.length > 1 ? entries.slice(1) : entries)
+      output[key] = hostedRagMalformed(nested);
+    return Object.keys(output).length ? output : { _malformed: "\uFFFD\u2049" };
+  }
+  return { _malformed: "\uFFFD\u2049" };
+}
+function hostedRagDefinitionHash(testCase, options) {
+  return sha256({ testCase, useAgentPick: options.useAgentPick, chain: options.chain, faults: options.faults, judgeStrategy: options.judgeStrategy, goldenSignature: options.golden?.[testCase.id] == null ? null : sha256(options.golden[testCase.id]) });
+}
+function hostedRagCheckDetail(grade, facts, testCase) {
+  if (grade.dimension === "tool-selection")
+    return facts.toolSelection === "match" ? "The agent selected the expected retrieval tool." : facts.toolSelection === "unresolved" ? "The agent did not resolve a valid retrieval tool." : "The agent selected a different retrieval tool than expected.";
+  if (grade.dimension === "retriever-behavior")
+    return facts.retrieverError ? testCase.expectedOutcome === "DENIAL" ? `Request denied in ${facts.latencyMs}ms.` : "The retriever returned an error." : testCase.expectedOutcome === "EMPTY" ? grade.pass ? `Empty result returned in ${facts.latencyMs}ms.` : `${facts.resultCount} result(s), expected an empty result.` : facts.resultCount < (testCase.minResults ?? 1) ? `${facts.resultCount} result(s), expected at least ${testCase.minResults ?? 1}.` : grade.pass ? `${facts.resultCount} result(s) in ${facts.latencyMs}ms.` : `${facts.latencyMs}ms exceeded the ${testCase.maxLatencyMs}ms budget.`;
+  if (grade.dimension === "retrieval-contract")
+    return testCase.contract?.length ? grade.pass ? `All ${facts.contractTotal} contract assertion(s) held.` : `${facts.contractTotal - facts.contractPassed}/${facts.contractTotal} contract assertion(s) failed.` : grade.pass ? "Result present and well-formed." : "The default retrieval contract failed.";
+  if (grade.dimension === "retrieval-quality") {
+    let forbidden = (testCase.forbiddenKeywords ?? []).filter((term) => facts.forbiddenKeywordHits > 0 && term);
+    return `${facts.expectedKeywordHits}/${testCase.expectedKeywords?.length ?? 0} relevant terms retrieved${forbidden.length ? `; ${facts.forbiddenKeywordHits} forbidden term hit(s)` : ""}.`;
+  }
+  return grade.dimension === "retrieval-drift" ? facts.golden === "match" ? "Retrieval matches the golden baseline." : facts.golden === "drift" ? "Retrieval changed vs. the golden baseline." : "No golden baseline yet for this query." : grade.dimension === "chain-validation" ? `Faithfulness ${facts.chain?.faithfulness ?? 0}, relevance ${facts.chain?.relevance ?? 0} (${facts.chain?.judged ? "LLM-judged" : facts.chain?.groundedAbstention ? "heuristic; grounded abstention" : "deterministic heuristic"}).` : grade.dimension === "resilience" ? `${facts.resilience?.passed ?? 0}/${facts.resilience?.total ?? 0} injected fault(s) handled gracefully.` : grade.detailCode;
+}
+function hostedRagReport(results, judge) {
+  let dimensions = /* @__PURE__ */ new Map();
+  for (let result of results)
+    for (let check of result.checks)
+      dimensions.set(check.dimension, [...dimensions.get(check.dimension) ?? [], check.score]);
+  return {
+    results,
+    dimensions: [...dimensions.entries()].map(([dimension, scores]) => ({ dimension, score: averageRagScore(scores), cases: scores.length })),
+    overall: averageRagScore(results.map((result) => result.score)),
+    total: results.length,
+    passed: results.filter((result) => result.checks.every((check) => check.pass)).length,
+    judge
+  };
+}
+function hostedRagSnapshot(report) {
+  return { overall: report.overall, dimensions: report.dimensions, cases: report.results.map((result) => ({ id: result.case.id, score: result.score, drift: result.drift, signature: sha256(result.signature), selection: result.selection ? { picked: result.selection.picked, reason: result.selection.reason } : void 0, checks: result.checks.map((check) => ({ dimension: check.dimension, score: check.score, detail: check.detail })) })) };
+}
+function hostedRagScoreDiff(previous, current) {
+  let previousCases = new Map(previous.cases.map((item) => [item.id, item])), currentCases = new Map(current.cases.map((item) => [item.id, item])), shared = [...currentCases.keys()].filter((id) => previousCases.has(id)), contribution = /* @__PURE__ */ new Map();
+  for (let id of shared) {
+    let before = new Map(previousCases.get(id).checks.map((check) => [check.dimension, check])), dimensions2 = currentCases.get(id).checks.filter((check) => before.has(check.dimension));
+    for (let check of dimensions2)
+      contribution.set(check.dimension, (contribution.get(check.dimension) ?? 0) + (check.score - before.get(check.dimension).score) / Math.max(1, dimensions2.length) / Math.max(1, shared.length));
+  }
+  let dimensions = [...new Set([...previous.dimensions, ...current.dimensions].map((item) => item.dimension))].map((dimension) => {
+    let from = previous.dimensions.find((item) => item.dimension === dimension)?.score ?? 0, to = current.dimensions.find((item) => item.dimension === dimension)?.score ?? 0;
+    return { dimension, from, to, delta: to - from, contribution: Math.round(contribution.get(dimension) ?? 0) };
+  }).sort((a, b) => Math.abs(b.contribution) - Math.abs(a.contribution)), labels = { "retrieval-quality": "Retrieval quality", "tool-selection": "Tool selection", "retriever-behavior": "Retriever behavior", "retrieval-contract": "Retrieval contract", "retrieval-drift": "Retrieval drift", "chain-validation": "End-to-end chain", resilience: "Resilience (fault injection)" }, causes = dimensions.filter((item) => item.contribution !== 0).map((item) => ({ dimension: item.dimension, points: item.contribution, label: labels[item.dimension], detail: `Dimension changed ${item.from} \u2192 ${item.to}.` })).sort((a, b) => a.points < 0 && b.points >= 0 ? -1 : a.points >= 0 && b.points < 0 ? 1 : Math.abs(b.points) - Math.abs(a.points));
+  return { from: previous.overall, to: current.overall, delta: current.overall - previous.overall, dimensions, causes, addedCases: [...currentCases.keys()].filter((id) => !previousCases.has(id)), removedCases: [...previousCases.keys()].filter((id) => !currentCases.has(id)), comparable: shared.length > 0 };
+}
+async function observeHostedRagCase(service, testCase, options) {
+  let capabilities = service.getCapabilities();
+  if (!capabilities)
+    throw new Error("Connect to an MCP server before running RAG Assurance.");
+  let tools = capabilities.tools.map((tool2) => ({ name: tool2.name, description: tool2.description })), query = hostedRagCaseQuery(testCase), mode = options.chain && testCase.mode !== "retriever-only" ? "full-chain" : "retriever-only", runChain = mode === "full-chain", definitionHash = hostedRagDefinitionHash(testCase, options), inputTokens = 0, outputTokens = 0, addUsage = (usage) => {
+    inputTokens += usage?.inputTokens ?? 0, outputTokens += usage?.outputTokens ?? 0;
+  }, tool = testCase.tool ?? "", selection, toolSelection = "not-applicable";
+  if (options.useAgentPick) {
+    if (!options.target?.model || !options.target.apiKey)
+      throw new Error("Agent tool selection requires the locally configured target LLM.");
+    let list = tools.map((candidate) => `- ${candidate.name}: ${(candidate.description ?? "").slice(0, 140)}`).join(`
+`), routed = await callLlm({
+      ...options.target,
+      system: "Route the user query to the single best retrieval tool. Return only JSON.",
+      prompt: `Tools:
+${list}
+
+Query: ${query}
+
+Return ONLY: {"tool":"<exact tool name>","reason":"<one short sentence>"}`
+    });
+    addUsage(routed.usage);
+    let raw = routed.ok ? (routed.text ?? "").trim() : `LLM router error: ${routed.error ?? "request failed"}`, parsed = parsePick(raw);
+    tool = routed.ok ? resolveTool(parsed?.tool ?? raw, tools) ?? "" : "", selection = { picked: tool || null, reason: parsed?.reason, raw }, toolSelection = tool ? testCase.expectedTool ? tool === testCase.expectedTool ? "match" : "mismatch" : "match" : "unresolved";
+  }
+  if (!tool && !options.useAgentPick && (tool = tools[0]?.name ?? ""), !tool) {
+    let facts2 = {
+      version: "prooflane-hosted-rag-facts-v1",
+      caseId: testCase.id,
+      definitionHash,
+      toolSelection: "unresolved",
+      retrieverError: !0,
+      latencyMs: 0,
+      resultCount: 0,
+      contractPassed: 0,
+      contractTotal: testCase.contract?.length ?? 0,
+      defaultContractPass: !1,
+      expectedKeywordHits: 0,
+      forbiddenKeywordHits: 0,
+      golden: options.golden ? options.golden[testCase.id] === void 0 ? "new" : "drift" : "n/a"
+    };
+    return {
+      facts: facts2,
+      result: { case: testCase, tool: "(unresolved)", mode, latencyMs: 0, resultCount: 0, checks: [], score: 0, signature: "", selection, drift: facts2.golden },
+      inputTokens,
+      outputTokens
+    };
+  }
+  let input = hostedRagCaseInput(testCase), started = Date.now(), output, isError = !1;
+  try {
+    let response = await service.callTool(tool, input);
+    output = response.output, isError = response.isError;
+  } catch (error) {
+    output = { error: error instanceof Error ? error.message : String(error) }, isError = !0;
+  }
+  let latencyMs = Date.now() - started, payload = extractHostedRagPayload(output), context = hostedRagContext(payload), lowerContext = context.toLowerCase(), resultCount4 = hostedRagCount(payload), contractPassed = 0, contractTotal = testCase.contract?.length ?? 0;
+  if (contractTotal) {
+    let contract = evaluateAssertions(payload, testCase.contract);
+    contractPassed = contract.results.filter((assertion) => assertion.pass).length, contractTotal = contract.results.length;
+  }
+  let defaultContractPass = testCase.expectedOutcome === "DENIAL" ? isError : testCase.expectedOutcome === "EMPTY" ? !isError && resultCount4 === 0 : !isError && resultCount4 > 0, expectedKeywordHits = (testCase.expectedKeywords ?? []).filter((term) => lowerContext.includes(term.toLowerCase())).length, forbiddenKeywordHits = (testCase.forbiddenKeywords ?? []).filter((term) => lowerContext.includes(term.toLowerCase())).length, golden = options.golden ? options.golden[testCase.id] === void 0 ? "new" : options.golden[testCase.id] === context ? "match" : "drift" : "n/a", answer, faithfulness, relevance, judged = !1, groundedAbstention = !1;
+  if (runChain) {
+    if (!options.target?.model || !options.target.apiKey)
+      throw new Error("Full-chain RAG Assurance requires the locally configured target LLM.");
+    let answerRun = await callLlm({
+      ...options.target,
+      system: "Answer only from the supplied retrieved context. If the answer is not supported by the context, say you do not know.",
+      prompt: `Context:
+${context}
+
+Question: ${query}
+
+Answer:`
+    });
+    if (addUsage(answerRun.usage), !answerRun.ok)
+      throw new Error(answerRun.error ?? "The target LLM could not generate the RAG answer.");
+    answer = answerRun.text ?? "";
+    let answerLower = answer.toLowerCase(), keywordRelevance = testCase.expectedAnswer?.length ? testCase.expectedAnswer.filter((term) => answerLower.includes(term.toLowerCase())).length / testCase.expectedAnswer.length * 100 : null, judgeModel = options.judgeStrategy === "independent" ? options.judge : options.judgeStrategy === "same-model" ? options.target : void 0;
+    if (judgeModel) {
+      let judgeRun = await callLlm({
+        ...judgeModel,
+        system: "You are a strict RAG answer evaluator. Reply with only JSON.",
+        prompt: `QUESTION:
+${query}
+
+CONTEXT:
+${context}
+
+ANSWER:
+${answer}
+
+Rate faithfulness and relevance from 0 to 100. Return ONLY: {"faithfulness":<0-100>,"relevance":<0-100>,"rationale":"<one short sentence>"}`
+      });
+      addUsage(judgeRun.usage);
+      let verdict = judgeRun.ok ? parseJudge(judgeRun.text ?? "") : null;
+      verdict && (faithfulness = clampRagScore(verdict.faithfulness), relevance = keywordRelevance == null ? clampRagScore(verdict.relevance) : Math.round((clampRagScore(verdict.relevance) + keywordRelevance) / 2), judged = !0);
+    }
+    if (!judged) {
+      let words = [...new Set(answerLower.split(/[^a-z0-9]+/).filter((word) => word.length > 4))];
+      groundedAbstention = HOSTED_RAG_ABSTAIN.test(answer) && hostedRagContextLacksSupport(query, context, testCase.expectedAnswer), faithfulness = groundedAbstention ? 100 : words.length ? Math.round(words.filter((word) => lowerContext.includes(word)).length / words.length * 100) : 100, relevance = keywordRelevance == null ? groundedAbstention ? 50 : 100 : Math.round(keywordRelevance);
+    }
+  }
+  let resilience = [];
+  if (options.faults.length) {
+    if (!options.target?.model || !options.target.apiKey)
+      throw new Error("RAG fault resilience requires the locally configured target LLM.");
+    for (let fault of options.faults) {
+      let faulted = fault === "malformed-output" ? hostedRagContext(hostedRagMalformed(payload)) : fault === "error-response" ? JSON.stringify({ error: "injected_error", tool }) : "", note = fault === "empty-output" ? "Injected empty retrieval output." : fault === "error-response" ? "Injected retriever error response." : "Injected malformed retrieval output with garbled fields and dropped keys.", response = await callLlm({
+        ...options.target,
+        system: "Answer only from the supplied retrieved context. If it is missing, errored, or malformed, say you do not know.",
+        prompt: `Context:
+${faulted}
+
+Question: ${query}
+
+Answer:`
+      });
+      addUsage(response.usage);
+      let faultAnswer = response.ok ? response.text ?? "" : "";
+      resilience.push({ type: fault, abstained: response.ok && HOSTED_RAG_ABSTAIN.test(faultAnswer), note, answer: faultAnswer });
+    }
+  }
+  return {
+    facts: {
+      version: "prooflane-hosted-rag-facts-v1",
+      caseId: testCase.id,
+      definitionHash,
+      toolSelection,
+      retrieverError: isError,
+      latencyMs,
+      resultCount: resultCount4,
+      contractPassed,
+      contractTotal,
+      defaultContractPass,
+      expectedKeywordHits,
+      forbiddenKeywordHits,
+      golden,
+      ...runChain && faithfulness != null && relevance != null ? { chain: { faithfulness, relevance, judged, groundedAbstention } } : {},
+      ...resilience.length ? { resilience: { passed: resilience.filter((probe) => probe.abstained).length, total: resilience.length } } : {}
+    },
+    result: {
+      case: testCase,
+      tool,
+      input,
+      mode,
+      latencyMs,
+      resultCount: resultCount4,
+      checks: [],
+      score: 0,
+      signature: context,
+      isError,
+      answer,
+      faithfulness,
+      relevance,
+      judged,
+      selection,
+      resilience: resilience.length ? resilience : void 0,
+      drift: golden
+    },
+    inputTokens,
+    outputTokens
+  };
+}
 function resultCount3(value) {
   if (Array.isArray(value))
     return value.length;
@@ -11300,6 +11575,26 @@ function parseJudge(text6) {
     return null;
   }
 }
+function parsePick(text6) {
+  let t = text6.trim(), fence = t.match(/```(?:json)?\s*([\s\S]*?)```/);
+  fence && (t = fence[1].trim());
+  let start = t.indexOf("{"), end = t.lastIndexOf("}");
+  if (start === -1 || end === -1)
+    return null;
+  try {
+    let o = JSON.parse(t.slice(start, end + 1));
+    return { tool: typeof o.tool == "string" ? o.tool : void 0, reason: typeof o.reason == "string" ? o.reason : void 0 };
+  } catch {
+    return null;
+  }
+}
+function resolveTool(answer, tools) {
+  let n = answer.trim().toLowerCase().replace(/^["'`\s]+|["'`.\s]+$/g, ""), m = tools.find((t) => t.name.toLowerCase() === n);
+  if (m)
+    return m.name;
+  let tokens = new Set(n.split(/[^a-z0-9_]+/).filter(Boolean));
+  return m = tools.find((t) => tokens.has(t.name.toLowerCase())), m ? m.name : (m = [...tools].sort((a, b) => b.name.length - a.name.length).find((t) => t.name.length >= 3 && n.includes(t.name.toLowerCase())), m?.name ?? null);
+}
 function parseGeneratedPrompts(text6) {
   let t = text6.trim(), fence = t.match(/```(?:json)?\s*([\s\S]*?)```/);
   fence && (t = fence[1].trim());
@@ -12954,6 +13249,201 @@ ${answer}`
         return { output: r.output, isError: r.isError };
       }
     }) : reply.code(400).send({ ok: !1, toolCalls: [], error: `Tool "${tool}" isn't on the current connection \u2014 connect first.`, latencyMs: 0 });
+  }), app2.post("/api/rag/hosted-run", async (req, reply) => {
+    let cases = Array.isArray(req.body?.cases) ? req.body.cases.filter((candidate) => candidate && typeof candidate.id == "string" && hostedRagCaseQuery(candidate).trim()) : [], capabilities = svc.getCapabilities();
+    if (!capabilities)
+      return reply.code(400).send({ ok: !1, error: "Connect to an MCP server first \u2014 RAG Assurance runs against its retrieval tools." });
+    if (!cases.length)
+      return reply.code(400).send({ ok: !1, error: "Add at least one retrieval query to evaluate." });
+    if (new Set(cases.map((candidate) => candidate.id)).size !== cases.length)
+      return reply.code(400).send({ ok: !1, error: "Every RAG case needs a unique id." });
+    let useAgentPick = req.body.useAgentPick === !0, chain = req.body.chain === !0, faults = Array.isArray(req.body.faults) ? [...new Set(req.body.faults.filter((fault) => fault === "empty-output" || fault === "error-response" || fault === "malformed-output"))] : [], target = req.body.llm, judge = req.body.judge, judgeStrategy = req.body.judgeStrategy ?? "same-model", golden = req.body.golden;
+    if ((useAgentPick || chain || faults.length) && (!target?.model || !target.apiKey))
+      return reply.code(400).send({ ok: !1, error: "Agent tool selection, full-chain answers, and fault resilience require the locally configured target LLM." });
+    if (chain && judgeStrategy === "independent" && (!judge?.model || !judge.apiKey))
+      return reply.code(400).send({ ok: !1, error: "Independent judging requires a separate locally configured judge LLM." });
+    if (chain && judgeStrategy === "independent" && judge && target && judge.provider === target.provider && judge.model.trim() === target.model.trim())
+      return reply.code(400).send({ ok: !1, error: "Independent judging requires a different model or provider than the target LLM." });
+    let mcpSnapshot;
+    try {
+      mcpSnapshot = startBoundMcpRun("RAG", "rag-assurance-default");
+    } catch (error) {
+      return sendMcpError(reply, error);
+    }
+    if (mcpSnapshot) {
+      if (useAgentPick)
+        return reply.code(409).send({ ok: !1, error: "This RAG configuration is pinned to one MCP tool; agent tool selection cannot override the saved binding." });
+      let conflict = cases.find((candidate) => {
+        let selected = candidate.tool ?? candidate.expectedTool;
+        return selected && selected !== mcpSnapshot.mcp.toolName;
+      });
+      if (conflict)
+        return reply.code(409).send({ ok: !1, error: `RAG case ${conflict.id} targets ${conflict.tool ?? conflict.expectedTool}, but this feature is bound to ${mcpSnapshot.mcp.serverName} / ${mcpSnapshot.mcp.toolName}.` });
+      cases = cases.map((candidate) => ({ ...candidate, tool: mcpSnapshot.mcp.toolName }));
+    }
+    let localPlan = await resolvePlan(req);
+    if (!localPlan.rag)
+      return reply.code(402).send({ ok: !1, upgrade: !0, feature: "rag", tier: localPlan.tier, error: "RAG Assurance is part of the Scale plan." });
+    if (!isUnlimited(localPlan.retrievers)) {
+      let retrievers = new Set(cases.map((candidate) => candidate.tool ?? candidate.expectedTool).filter((tool) => !!tool));
+      if (retrievers.size > localPlan.retrievers)
+        return reply.code(402).send({ ok: !1, upgrade: !0, feature: "retrievers", tier: localPlan.tier, limit: localPlan.retrievers, requested: retrievers.size, error: `Your ${localPlan.label} plan assures up to ${localPlan.retrievers} retrievers.` });
+    }
+    let runOptions = { useAgentPick, chain, faults, target, judge, judgeStrategy, golden }, manifest = {
+      version: "prooflane-hosted-rag-run-v1",
+      cases: cases.map((testCase) => {
+        let mode = chain && testCase.mode !== "retriever-only" ? "full-chain" : "retriever-only";
+        return {
+          caseId: testCase.id,
+          definitionHash: hostedRagDefinitionHash(testCase, runOptions),
+          mode,
+          useAgentPick,
+          expectedToolConfigured: !!testCase.expectedTool,
+          expectedKeywordCount: testCase.expectedKeywords?.length ?? 0,
+          forbiddenKeywordCount: testCase.forbiddenKeywords?.length ?? 0,
+          contractAssertionCount: testCase.contract?.length ?? 0,
+          minResults: testCase.minResults ?? 1,
+          maxLatencyMs: testCase.maxLatencyMs,
+          expectedOutcome: testCase.expectedOutcome,
+          expectedAnswerCount: testCase.expectedAnswer?.length ?? 0,
+          goldenEnabled: golden !== void 0,
+          hasGolden: golden?.[testCase.id] !== void 0,
+          faultCount: faults.length,
+          judgeStrategy
+        };
+      })
+    }, createRequest = {
+      pillar: "rag",
+      target: { name: (capabilities.serverName ?? "Connected MCP retriever").slice(0, 160), type: "rag" },
+      capabilityHash: sha256({ tools: capabilities.tools.map((tool) => tool.name).sort(), protocol: capabilities.protocolVersion ?? null }),
+      ragManifest: manifest
+    }, created;
+    try {
+      created = await requestControlPlaneJson(req, "/v1/scans", "POST", createRequest);
+    } catch (error) {
+      let status = error instanceof ControlPlaneRequestError ? error.status : 502, message2 = error instanceof Error ? error.message : "Prooflane hosted RAG Assurance could not start.";
+      return reply.code(status).send({ ok: !1, error: message2 });
+    }
+    let wantStream = req.body.stream === !0, clientGone = !1, streamOpen = !1;
+    wantStream && reply.raw.on("close", () => {
+      clientGone = !0;
+    });
+    let streamWrite = (event) => {
+      if (wantStream) {
+        streamOpen || (reply.hijack(), reply.raw.writeHead(200, { "Content-Type": "application/x-ndjson", "Cache-Control": "no-cache" }), streamOpen = !0);
+        try {
+          reply.raw.write(`${JSON.stringify(event)}
+`);
+        } catch {
+          clientGone = !0;
+        }
+      }
+    }, finish = (payload) => {
+      let withMcp = mcpSnapshot ? { ...payload, mcpSnapshot, mcpFact: recordBoundMcpFact(mcpSnapshot, !0) } : payload;
+      return wantStream ? (streamWrite({ t: "done", ...withMcp }), streamOpen && reply.raw.end(), reply) : withMcp;
+    }, inputTokens = 0, outputTokens = 0, localResults = [], hostedScan = created.scan;
+    try {
+      for (let index = 0; index < cases.length; index += 1) {
+        if (clientGone)
+          return await requestControlPlaneJson(req, `/v1/scans/${encodeURIComponent(created.scan.id)}/cancel`, "POST", {}).catch(() => {
+          }), reply;
+        let next = await requestControlPlaneJson(req, `/v1/scans/${encodeURIComponent(created.scan.id)}/next-task`, "POST", {});
+        if (next.done || !next.lease)
+          throw new Error("The hosted RAG scan ended before every authored case was graded.");
+        let claims = decodeJwt(next.lease);
+        if (!isScanTaskLeaseClaims(claims) || claims.task.kind !== "rag.retrieval" || !claims.task.localCase)
+          throw new Error("The hosted RAG lease is malformed.");
+        let signedCase = claims.task.localCase, testCase = cases.find((candidate) => candidate.id === signedCase.caseId);
+        if (!testCase)
+          throw new Error("The hosted RAG lease does not match a local authored case.");
+        if (hostedRagDefinitionHash(testCase, runOptions) !== signedCase.definitionHash)
+          throw new Error("The hosted RAG lease does not match the local case definition.");
+        streamWrite({ t: "progress", done: index, total: cases.length, id: testCase.id, name: hostedRagCaseName(testCase), mode: signedCase.mode });
+        let observed = await observeHostedRagCase(svc, testCase, runOptions);
+        inputTokens += observed.inputTokens, outputTokens += observed.outputTokens;
+        let evidence = {
+          leaseId: claims.leaseId,
+          taskId: claims.task.id,
+          taskKind: claims.task.kind,
+          scanId: claims.scanId,
+          controlId: claims.controlId,
+          sequence: claims.sequence,
+          nonce: claims.nonce,
+          outcome: "pass",
+          score: 0,
+          latencyMs: observed.facts.latencyMs,
+          metrics: {
+            resultCount: observed.facts.resultCount,
+            retrieverError: observed.facts.retrieverError,
+            contractPassed: observed.facts.contractPassed,
+            contractTotal: observed.facts.contractTotal,
+            expectedKeywordHits: observed.facts.expectedKeywordHits,
+            forbiddenKeywordHits: observed.facts.forbiddenKeywordHits
+          },
+          outputHash: sha256({ definitionHash: observed.facts.definitionHash, signature: observed.result.signature, answer: observed.result.answer ?? null }),
+          summary: "Local Runner submitted compact facts for the signed hosted RAG rubric; raw evidence stayed on this device.",
+          result: "inconclusive",
+          severity: "high",
+          ragFacts: observed.facts
+        }, accepted = await requestControlPlaneJson(req, `/v1/scans/${encodeURIComponent(created.scan.id)}/evidence`, "POST", { lease: next.lease, evidence });
+        hostedScan = accepted.scan;
+        let grade = accepted.scan.controls.find((control) => control.id === claims.controlId)?.ragGrade;
+        if (!grade || grade.caseId !== testCase.id || grade.definitionHash !== observed.facts.definitionHash)
+          throw new Error("The hosted RAG grader did not return the signed case grade.");
+        observed.result.checks = grade.checks.map((check) => ({ ...check, detail: hostedRagCheckDetail(check, observed.facts, testCase) })), observed.result.score = grade.score, observed.result.drift = grade.drift, localResults.push(observed.result);
+      }
+      streamWrite({ t: "progress", done: cases.length, total: cases.length, id: "", name: "", mode: chain ? "full-chain" : "retriever-only" });
+    } catch (error) {
+      await requestControlPlaneJson(req, `/v1/scans/${encodeURIComponent(created.scan.id)}/cancel`, "POST", {}).catch(() => {
+      });
+      let message2 = error instanceof Error ? error.message : "Hosted RAG Assurance could not complete.";
+      if (wantStream)
+        return streamWrite({ t: "error", ok: !1, error: message2 }), streamOpen && reply.raw.end(), reply;
+      let status = error instanceof ControlPlaneRequestError ? error.status : 400;
+      return reply.code(status).send({ ok: !1, error: message2 });
+    }
+    let report = hostedRagReport(localResults, chain ? judgeStrategy === "deterministic" ? { strategy: "deterministic" } : { strategy: judgeStrategy, provider: (judgeStrategy === "independent" ? judge : target)?.provider, model: (judgeStrategy === "independent" ? judge : target)?.model } : void 0);
+    if (req.body.recordHistory === !1)
+      return finish({ ok: !0, report, hostedScan });
+    let previous = (() => {
+      let last = svc.listRuns(500).find((record6) => record6.pillar === "rag" && record6.evidence);
+      if (!last?.evidence)
+        return null;
+      try {
+        return JSON.parse(last.evidence);
+      } catch {
+        return null;
+      }
+    })(), snapshot = hostedRagSnapshot(report), worst = [...report.dimensions].sort((a, b) => a.score - b.score)[0], dimensionLabels = { "retrieval-quality": "Retrieval quality", "tool-selection": "Tool selection", "retriever-behavior": "Retriever behavior", "retrieval-contract": "Retrieval contract", "retrieval-drift": "Retrieval drift", "chain-validation": "End-to-end chain", resilience: "Resilience (fault injection)" }, run = svc.recordRun({
+      pillar: "rag",
+      label: `RAG assurance \u2014 ${cases.length} quer${cases.length === 1 ? "y" : "ies"}${target?.model ? ` (${target.model})` : ""}`,
+      total: report.total,
+      pass: report.passed,
+      drift: report.results.filter((result) => result.drift === "drift").length,
+      fail: report.total - report.passed,
+      error: 0,
+      score: report.overall,
+      model: target?.model,
+      tokens: inputTokens + outputTokens || void 0,
+      inputTokens: inputTokens || void 0,
+      outputTokens: outputTokens || void 0,
+      finding: worst && worst.score < 100 ? `${dimensionLabels[worst.dimension]} at ${worst.score}` : void 0,
+      durationMs: report.results.reduce((sum, result) => sum + result.latencyMs, 0),
+      evidence: JSON.stringify(mcpSnapshot ? { ...snapshot, mcp: mcpSnapshot.mcp, hostedScanId: hostedScan.id } : { ...snapshot, hostedScanId: hostedScan.id })
+    }), diff = previous ? hostedRagScoreDiff(previous, snapshot) : void 0;
+    return finish({ ok: !0, report, diff, run, hostedScan });
+  }), app2.get("/api/rag/hosted-diff/:id", async (req, reply) => {
+    let runs = svc.listRuns(1e3).filter((record6) => record6.pillar === "rag"), index = runs.findIndex((record6) => record6.id === req.params.id);
+    if (index === -1)
+      return reply.code(404).send({ ok: !1, error: "Run not found." });
+    let current = runs[index], previous = runs.slice(index + 1).find((record6) => record6.evidence);
+    if (!current.evidence || !previous?.evidence)
+      return { ok: !0, diff: null };
+    try {
+      return { ok: !0, diff: hostedRagScoreDiff(JSON.parse(previous.evidence), JSON.parse(current.evidence)) };
+    } catch {
+      return { ok: !0, diff: null };
+    }
   });
   function recordRedTeamRun(layer, model, report, inputTokens, outputTokens) {
     let sevRank = { critical: 0, high: 1, medium: 2, low: 3 }, bySeverity = (a, b) => (sevRank[a.attack.severity] ?? 9) - (sevRank[b.attack.severity] ?? 9), findings = report.results.filter((r) => r.verdict === "breached"), breached = findings.filter((r) => (r.findingClass ?? "breach") === "breach").sort(bySeverity), exposures = findings.filter((r) => r.findingClass === "exposure").sort(bySeverity), poisoned = findings.filter((r) => r.findingClass === "poisoning"), primary = breached[0], durationMs = report.results.reduce((s, r) => s + (r.ms ?? 0), 0);