chainlesschain 0.166.20 → 0.166.22

package/src/lib/skill-outcome-transcript-authority.js

@@ -0,0 +1,252 @@
+import { createHash } from "node:crypto";
+import skillInvocationReceipt from "@chainlesschain/session-core/skill-invocation-receipt";
+
+import {
+  listSessionAuthoritySummaries,
+  readVerifiedProjection,
+} from "../harness/jsonl-session-store.js";
+import { canonicalJson } from "./scheduler-kernel/contract.js";
+
+const { verifySkillInvocationReceipt } = skillInvocationReceipt;
+
+export const SKILL_OUTCOME_AUTHORITY_SCHEMA =
+  "chainlesschain.skill-outcome-transcript-authority/v1";
+export const MAX_OUTCOME_AUTHORITY_SESSIONS = 128;
+export const MAX_OUTCOME_AUTHORITY_RECEIPTS = 10_000;
+
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+const TRANSCRIPT_HASH = /^[a-f0-9]{64}$/u;
+
+function unavailable(message, code = "CC_SKILL_OUTCOME_AUTHORITY_UNAVAILABLE") {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+function verifiedReceipt(receipt) {
+  verifySkillInvocationReceipt(receipt);
+  if (
+    !Array.isArray(receipt.selectedSkillDigests) ||
+    receipt.selectedSkillDigests.length !== 1 ||
+    !DIGEST.test(receipt.selectedSkillDigests[0] || "")
+  ) {
+    throw unavailable("Skill outcome receipt has an invalid selection binding");
+  }
+  if (
+    !Array.isArray(receipt.graderReceipts) ||
+    receipt.graderReceipts.length > 64 ||
+    receipt.graderReceipts.some((digest) => !DIGEST.test(digest || "")) ||
+    (receipt.userCorrectionRef !== null &&
+      (typeof receipt.userCorrectionRef !== "string" ||
+        receipt.userCorrectionRef.length < 1 ||
+        receipt.userCorrectionRef.length > 256)) ||
+    (receipt.attributionEligible === true &&
+      (receipt.attributionStatus !== "complete" ||
+        !Array.isArray(receipt.missingAttribution) ||
+        receipt.missingAttribution.length !== 0))
+  ) {
+    throw unavailable("Skill outcome receipt has an invalid attribution state");
+  }
+  return receipt;
+}
+
+function transcriptReceiptProjection(receipts, capacity) {
+  return {
+    accept(event) {
+      const receipt = event?.data?.skill_invocation_receipt;
+      if (receipt === undefined) return;
+      if (event.type !== "tool_call" || event.data?.tool !== "run_skill") {
+        throw unavailable(
+          "Skill outcome receipt is bound to a non-Skill event",
+        );
+      }
+      if (receipts.length >= capacity) {
+        throw unavailable(
+          "Skill outcome receipt capacity exceeded",
+          "CC_SKILL_OUTCOME_AUTHORITY_CAPACITY",
+        );
+      }
+      receipts.push(verifiedReceipt(receipt));
+    },
+    finish(authority) {
+      if (
+        !authority ||
+        !Number.isSafeInteger(authority.eventCount) ||
+        authority.eventCount < 0 ||
+        (authority.headHash !== null &&
+          !TRANSCRIPT_HASH.test(authority.headHash || ""))
+      ) {
+        throw unavailable("Verified transcript returned invalid authority");
+      }
+      return Object.freeze({
+        headHash: authority.headHash,
+        eventCount: authority.eventCount,
+      });
+    },
+  };
+}
+
+/**
+ * Build bounded outcome metrics only from fully verified canonical transcripts.
+ * Any damaged/blocked selected session or projection failure disables the whole
+ * sample instead of silently ranking from a favorable partial subset.
+ */
+export function buildSkillOutcomeTranscriptAuthority(
+  options = {},
+  dependencies = {},
+) {
+  const maxSessions =
+    options.maxSessions === undefined
+      ? MAX_OUTCOME_AUTHORITY_SESSIONS
+      : Number(options.maxSessions);
+  const maxReceipts =
+    options.maxReceipts === undefined
+      ? MAX_OUTCOME_AUTHORITY_RECEIPTS
+      : Number(options.maxReceipts);
+  if (
+    !Number.isSafeInteger(maxSessions) ||
+    maxSessions < 1 ||
+    maxSessions > MAX_OUTCOME_AUTHORITY_SESSIONS ||
+    !Number.isSafeInteger(maxReceipts) ||
+    maxReceipts < 1 ||
+    maxReceipts > MAX_OUTCOME_AUTHORITY_RECEIPTS
+  ) {
+    throw new TypeError("Skill outcome authority bounds are invalid");
+  }
+  const list =
+    dependencies.listSessionAuthoritySummaries || listSessionAuthoritySummaries;
+  const read = dependencies.readVerifiedProjection || readVerifiedProjection;
+  if (typeof list !== "function" || typeof read !== "function") {
+    throw new TypeError(
+      "Skill outcome transcript authority ports are required",
+    );
+  }
+  const rows = list({ limit: maxSessions });
+  if (!Array.isArray(rows) || rows.length > maxSessions) {
+    throw unavailable(
+      "Skill outcome session authority is invalid or unbounded",
+    );
+  }
+  const sessionIds = new Set();
+  for (const row of rows) {
+    if (
+      !row ||
+      typeof row.id !== "string" ||
+      row.id.length < 1 ||
+      row.id.length > 256 ||
+      sessionIds.has(row.id) ||
+      row._store !== "jsonl" ||
+      row._presence !== "present" ||
+      row._blocked !== false
+    ) {
+      throw unavailable(
+        "Skill outcome session authority contains a blocked row",
+      );
+    }
+    sessionIds.add(row.id);
+  }
+
+  const receipts = [];
+  const sessions = rows.map((row) => {
+    const authority = read(row.id, () =>
+      transcriptReceiptProjection(receipts, maxReceipts),
+    );
+    return Object.freeze({
+      sessionId: row.id,
+      headHash: authority.headHash,
+      eventCount: authority.eventCount,
+    });
+  });
+  const uniqueReceipts = new Map();
+  let duplicateReceiptCount = 0;
+  for (const receipt of receipts) {
+    if (uniqueReceipts.has(receipt.receiptDigest)) {
+      duplicateReceiptCount += 1;
+      continue;
+    }
+    uniqueReceipts.set(receipt.receiptDigest, receipt);
+  }
+
+  const totals = new Map();
+  let attributionEligibleReceiptCount = 0;
+  let outcomeEligibleReceiptCount = 0;
+  for (const receipt of uniqueReceipts.values()) {
+    if (receipt.attributionEligible !== true) continue;
+    attributionEligibleReceiptCount += 1;
+    const hasOutcomeEvidence =
+      receipt.graderReceipts.length > 0 || receipt.userCorrectionRef !== null;
+    if (
+      !hasOutcomeEvidence ||
+      !["completed", "failed"].includes(receipt.executionStatus)
+    ) {
+      continue;
+    }
+    outcomeEligibleReceiptCount += 1;
+    const digest = receipt.selectedSkillDigests[0];
+    const prior = totals.get(digest) || {
+      samples: 0,
+      successes: 0,
+      corrections: 0,
+    };
+    prior.samples += 1;
+    if (receipt.executionStatus === "completed") prior.successes += 1;
+    if (receipt.userCorrectionRef !== null) prior.corrections += 1;
+    totals.set(digest, prior);
+  }
+  const metrics = Object.fromEntries(
+    [...totals.entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([digest, value]) => [
+        digest,
+        Object.freeze({
+          samples: value.samples,
+          successRate: value.successes / value.samples,
+          correctionRate: value.corrections / value.samples,
+        }),
+      ]),
+  );
+  const source = sessions
+    .map(({ sessionId, headHash, eventCount }) => ({
+      sessionId,
+      headHash,
+      eventCount,
+    }))
+    .sort((left, right) => left.sessionId.localeCompare(right.sessionId));
+  const sourceDigest = `sha256:${createHash("sha256")
+    .update(`${SKILL_OUTCOME_AUTHORITY_SCHEMA}\0${canonicalJson(source)}`)
+    .digest("hex")}`;
+  return Object.freeze({
+    schema: SKILL_OUTCOME_AUTHORITY_SCHEMA,
+    status: "verified",
+    metrics: Object.freeze(metrics),
+    evidence: Object.freeze({
+      schema: SKILL_OUTCOME_AUTHORITY_SCHEMA,
+      status: "verified",
+      sourceDigest,
+      selectedSessionCount: sessions.length,
+      receiptCount: receipts.length,
+      uniqueReceiptCount: uniqueReceipts.size,
+      attributionEligibleReceiptCount,
+      outcomeEligibleReceiptCount,
+      duplicateReceiptCount,
+      maxSessions,
+      maxReceipts,
+    }),
+  });
+}
+
+export function unavailableSkillOutcomeTranscriptAuthority(error) {
+  return Object.freeze({
+    schema: SKILL_OUTCOME_AUTHORITY_SCHEMA,
+    status: "unavailable",
+    metrics: null,
+    evidence: Object.freeze({
+      schema: SKILL_OUTCOME_AUTHORITY_SCHEMA,
+      status: "unavailable",
+      code:
+        typeof error?.code === "string" && error.code.startsWith("CC_SKILL_")
+          ? error.code
+          : "CC_SKILL_OUTCOME_AUTHORITY_UNAVAILABLE",
+    }),
+  });
+}

package/src/lib/skill-retrieval-router-benchmark.js

@@ -0,0 +1,135 @@
+import { createHash } from "node:crypto";
+import { performance } from "node:perf_hooks";
+
+import { routeSkillDescriptors } from "./skill-retrieval-router.js";
+
+export const SKILL_RETRIEVAL_BENCHMARK_SCHEMA =
+  "chainlesschain.skill-retrieval-benchmark/v1";
+
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+
+function canonical(value) {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+  return `{${Object.keys(value)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`)
+    .join(",")}}`;
+}
+
+function hash(value) {
+  return `sha256:${createHash("sha256")
+    .update(SKILL_RETRIEVAL_BENCHMARK_SCHEMA)
+    .update("\0")
+    .update(canonical(value))
+    .digest("hex")}`;
+}
+
+function finiteRatio(value, name) {
+  const number = Number(value);
+  if (!Number.isFinite(number) || number < 0 || number > 1)
+    throw new TypeError(`${name} must be between zero and one`);
+  return number;
+}
+
+function normalizeThresholds(input) {
+  const recallAt5 = finiteRatio(input?.recallAt5, "recallAt5");
+  const falseInvocationRate = finiteRatio(
+    input?.falseInvocationRate,
+    "falseInvocationRate",
+  );
+  const p95Ms = Number(input?.p95Ms);
+  if (!Number.isFinite(p95Ms) || p95Ms <= 0)
+    throw new TypeError("p95Ms must be positive");
+  return Object.freeze({ recallAt5, falseInvocationRate, p95Ms });
+}
+
+export function runSkillRetrievalBenchmark({
+  skills,
+  cases,
+  thresholds,
+  clock = () => performance.now(),
+} = {}) {
+  if (
+    !Array.isArray(skills) ||
+    skills.length < 2 ||
+    skills.length > 10_000 ||
+    !Array.isArray(cases) ||
+    cases.length < 1 ||
+    cases.length > 10_000 ||
+    typeof clock !== "function"
+  )
+    throw new TypeError("Skill retrieval benchmark input is invalid");
+  const gate = normalizeThresholds(thresholds);
+  const corpusDigests = new Set(
+    skills.map((skill) => skill.executionIdentity?.contentDigest),
+  );
+  const normalizedCases = cases.map((item, index) => {
+    if (
+      typeof item?.query !== "string" ||
+      item.query.trim() === "" ||
+      !DIGEST.test(item?.expectedDigest ?? "")
+    )
+      throw new TypeError(`benchmark case ${index} is invalid`);
+    if (!corpusDigests.has(item.expectedDigest))
+      throw new TypeError(`benchmark case ${index} is absent from the corpus`);
+    return { query: item.query, expectedDigest: item.expectedDigest };
+  });
+  const plan = Object.freeze({
+    skillCorpusDigest: hash(
+      skills.map((skill) => ({
+        id: skill.id,
+        digest: skill.executionIdentity?.contentDigest ?? null,
+        description: skill.description ?? "",
+      })),
+    ),
+    caseCorpusDigest: hash(normalizedCases),
+    caseCount: normalizedCases.length,
+    thresholds: gate,
+  });
+  const latencies = [];
+  let recalled = 0;
+  let falseInvocations = 0;
+  let abstentions = 0;
+  for (const item of normalizedCases) {
+    const startedAt = Number(clock());
+    const result = routeSkillDescriptors({
+      skills,
+      query: item.query,
+      topK: 5,
+      ambiguityMargin: 0,
+    });
+    const endedAt = Number(clock());
+    if (
+      !Number.isFinite(startedAt) ||
+      !Number.isFinite(endedAt) ||
+      endedAt < startedAt
+    )
+      throw new Error("Skill retrieval benchmark clock is invalid");
+    latencies.push(endedAt - startedAt);
+    if (result.candidates.some(({ digest }) => digest === item.expectedDigest))
+      recalled += 1;
+    if (result.selected === null) abstentions += 1;
+    else if (result.selected.digest !== item.expectedDigest)
+      falseInvocations += 1;
+  }
+  latencies.sort((a, b) => a - b);
+  const recallAt5 = recalled / normalizedCases.length;
+  const falseInvocationRate = falseInvocations / normalizedCases.length;
+  const p95Ms = latencies[Math.max(0, Math.ceil(latencies.length * 0.95) - 1)];
+  const core = {
+    schema: SKILL_RETRIEVAL_BENCHMARK_SCHEMA,
+    plan,
+    metrics: {
+      recallAt5,
+      falseInvocationRate,
+      p95Ms,
+      abstentionRate: abstentions / normalizedCases.length,
+    },
+    passed:
+      recallAt5 >= gate.recallAt5 &&
+      falseInvocationRate < gate.falseInvocationRate &&
+      p95Ms < gate.p95Ms,
+  };
+  return Object.freeze({ ...core, reportDigest: hash(core) });
+}

package/src/lib/skill-retrieval-router.js

@@ -0,0 +1,235 @@
+import { BM25Search } from "./bm25-search.js";
+
+export const SKILL_RETRIEVAL_RESULT_SCHEMA =
+  "chainlesschain.skill-retrieval-result/v1";
+export const MAX_ROUTABLE_SKILLS = 10_000;
+
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+
+function finite(value, fallback = 0) {
+  return Number.isFinite(Number(value)) ? Number(value) : fallback;
+}
+
+function normalize(scores) {
+  const values = [...scores.values()];
+  const max = Math.max(0, ...values);
+  if (max === 0) return new Map([...scores].map(([key]) => [key, 0]));
+  return new Map([...scores].map(([key, value]) => [key, value / max]));
+}
+
+function digestFor(skill) {
+  const value = skill?.executionIdentity?.contentDigest;
+  return DIGEST.test(value || "") ? value : null;
+}
+
+function containsPath(skill, targetPath) {
+  if (!targetPath || !Array.isArray(skill.paths) || skill.paths.length === 0)
+    return true;
+  const normalized = String(targetPath).replaceAll("\\", "/");
+  return skill.paths.some((scope) => {
+    const prefix = String(scope).replaceAll("\\", "/").replace(/\/$/u, "");
+    return normalized === prefix || normalized.startsWith(`${prefix}/`);
+  });
+}
+
+function compatible(skill, target) {
+  if (!target) return { accepted: true, reasons: [] };
+  const reasons = [];
+  if (
+    Array.isArray(skill.os) &&
+    skill.os.length > 0 &&
+    target.os &&
+    !skill.os.includes(target.os)
+  ) {
+    reasons.push(`os:${target.os}`);
+  }
+  if (
+    Array.isArray(target.allowedCapabilities) &&
+    Array.isArray(skill.capabilities)
+  ) {
+    const allowed = new Set(target.allowedCapabilities);
+    const denied = skill.capabilities.filter((value) => !allowed.has(value));
+    if (denied.length > 0) reasons.push(`capability:${denied.join(",")}`);
+  }
+  return { accepted: reasons.length === 0, reasons };
+}
+
+function metricFor(metrics, digest) {
+  const value = metrics?.[digest] ?? metrics?.get?.(digest) ?? null;
+  if (value === null) return { samples: 0, successRate: 0, correctionRate: 0 };
+  const samples = finite(value.samples);
+  const successRate = finite(value.successRate);
+  const correctionRate = finite(value.correctionRate);
+  if (
+    !Number.isSafeInteger(samples) ||
+    samples < 0 ||
+    successRate < 0 ||
+    successRate > 1 ||
+    correctionRate < 0 ||
+    correctionRate > 1
+  ) {
+    throw new TypeError("Skill outcome metrics are invalid");
+  }
+  return { samples, successRate, correctionRate };
+}
+
+export function routeSkillDescriptors({
+  skills,
+  query,
+  namespace = null,
+  tags = [],
+  targetPath = null,
+  target = null,
+  vectorScores = null,
+  outcomeMetrics = null,
+  topK = 5,
+  ambiguityMargin = 0.02,
+} = {}) {
+  if (
+    !Array.isArray(skills) ||
+    skills.length > MAX_ROUTABLE_SKILLS ||
+    typeof query !== "string" ||
+    query.trim().length < 1 ||
+    query.length > 4096 ||
+    !Array.isArray(tags) ||
+    !Number.isSafeInteger(topK) ||
+    topK < 1 ||
+    topK > 64 ||
+    !Number.isFinite(ambiguityMargin) ||
+    ambiguityMargin < 0 ||
+    ambiguityMargin > 1
+  ) {
+    throw new TypeError("Skill retrieval request is invalid or unbounded");
+  }
+  const requiredTags = new Set(
+    tags.map((value) => String(value).toLowerCase()),
+  );
+  const rejected = [];
+  const admitted = [];
+  for (const skill of skills) {
+    const digest = digestFor(skill);
+    const skillTags = new Set(
+      (Array.isArray(skill.tags) ? skill.tags : []).map((value) =>
+        String(value).toLowerCase(),
+      ),
+    );
+    const mismatch = compatible(skill, target);
+    const reasons = [];
+    if (!digest) reasons.push("missing-content-digest");
+    if (namespace !== null && skill.source !== namespace)
+      reasons.push("namespace-mismatch");
+    if ([...requiredTags].some((value) => !skillTags.has(value)))
+      reasons.push("tag-mismatch");
+    if (!containsPath(skill, targetPath)) reasons.push("path-mismatch");
+    reasons.push(...mismatch.reasons);
+    if (reasons.length > 0) {
+      rejected.push({ id: skill.id, digest, reasons });
+      continue;
+    }
+    admitted.push({ skill, digest });
+  }
+
+  const bm25 = new BM25Search({ language: "auto" });
+  bm25.indexDocuments(
+    admitted.map(({ skill, digest }) => ({
+      id: digest,
+      title: `${skill.id} ${skill.displayName || ""} ${(skill.tags || []).join(" ")}`,
+      content: `${skill.description || ""} ${skill.category || ""}`,
+    })),
+  );
+  const bm25Scores = normalize(
+    new Map(
+      bm25
+        .search(query, { topK: admitted.length || 1, threshold: 0 })
+        .map(({ id, score }) => [id, score]),
+    ),
+  );
+  const vector = normalize(
+    new Map(
+      admitted.map(({ digest }) => [
+        digest,
+        Math.max(
+          0,
+          finite(vectorScores?.[digest] ?? vectorScores?.get?.(digest)),
+        ),
+      ]),
+    ),
+  );
+  const vectorAvailable = [...vector.values()].some((value) => value > 0);
+  const candidates = admitted
+    .map(({ skill, digest }) => {
+      const lexicalScore = bm25Scores.get(digest) || 0;
+      const vectorScore = vector.get(digest) || 0;
+      const outcome = metricFor(outcomeMetrics, digest);
+      const outcomeScore =
+        outcome.samples === 0
+          ? 0.5
+          : outcome.successRate * (1 - outcome.correctionRate);
+      const score =
+        lexicalScore * (vectorAvailable ? 0.45 : 0.75) +
+        vectorScore * (vectorAvailable ? 0.35 : 0) +
+        outcomeScore * (vectorAvailable ? 0.2 : 0.25);
+      return {
+        id: skill.id,
+        displayName: skill.displayName || skill.id,
+        namespace: skill.source,
+        version: String(skill.version || "1.0.0"),
+        digest,
+        category: skill.category || "uncategorized",
+        contextCostTokens: Math.ceil(
+          `${skill.id}\n${skill.description || ""}\n${skill.category || ""}`
+            .length / 4,
+        ),
+        score,
+        scores: {
+          lexical: lexicalScore,
+          vector: vectorScore,
+          outcome: outcomeScore,
+        },
+        outcome,
+        reason: `bm25=${lexicalScore.toFixed(3)}, vector=${vectorScore.toFixed(3)}, outcome=${outcomeScore.toFixed(3)}`,
+      };
+    })
+    .filter(({ scores }) => scores.lexical > 0 || scores.vector > 0)
+    .sort(
+      (left, right) =>
+        right.score - left.score || left.digest.localeCompare(right.digest),
+    );
+
+  const conflicts = [];
+  const byName = new Map();
+  for (const candidate of candidates) {
+    const key = candidate.id.toLowerCase();
+    const prior = byName.get(key);
+    if (prior && prior.digest !== candidate.digest) {
+      conflicts.push({
+        type: "same-name-different-version",
+        name: candidate.id,
+        digests: [prior.digest, candidate.digest].sort(),
+      });
+    } else byName.set(key, candidate);
+  }
+  const returned = candidates.slice(0, topK);
+  const ambiguous =
+    returned.length > 1 &&
+    returned[0].score - returned[1].score < ambiguityMargin;
+  if (ambiguous) {
+    conflicts.push({
+      type: "ambiguous-top-score",
+      digests: [returned[0].digest, returned[1].digest],
+      margin: returned[0].score - returned[1].score,
+    });
+  }
+  const topConflict = conflicts.some(({ digests }) =>
+    digests?.includes(returned[0]?.digest),
+  );
+  return Object.freeze({
+    schema: SKILL_RETRIEVAL_RESULT_SCHEMA,
+    query: query.trim(),
+    selected: returned.length > 0 && !topConflict ? returned[0] : null,
+    candidates: Object.freeze(returned.map(Object.freeze)),
+    conflicts: Object.freeze(conflicts.map(Object.freeze)),
+    rejected: Object.freeze(rejected.map(Object.freeze)),
+    vectorAvailable,
+  });
+}