chainlesschain 0.166.69 → 0.166.70

package/src/lib/decision-layer/benchmark.js

@@ -0,0 +1,164 @@
+import { performance } from "node:perf_hooks";
+
+import { decisionDigest } from "./contracts.js";
+
+export const SKILL_DECISION_BENCHMARK_SCHEMA =
+  "chainlesschain.skill-decision-benchmark/v1";
+
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+
+function ratio(value, label) {
+  const number = Number(value);
+  if (!Number.isFinite(number) || number < 0 || number > 1) {
+    throw new TypeError(`${label} must be between zero and one`);
+  }
+  return number;
+}
+
+function normalizeThresholds(input) {
+  const p95Ms = Number(input?.p95Ms);
+  if (!Number.isFinite(p95Ms) || p95Ms <= 0) {
+    throw new TypeError("p95Ms must be positive");
+  }
+  return Object.freeze({
+    acceptedSuggestionErrorRate: ratio(
+      input?.acceptedSuggestionErrorRate,
+      "acceptedSuggestionErrorRate",
+    ),
+    noMatchFalseSuggestionRate: ratio(
+      input?.noMatchFalseSuggestionRate,
+      "noMatchFalseSuggestionRate",
+    ),
+    affirmativeCoverage: ratio(
+      input?.affirmativeCoverage,
+      "affirmativeCoverage",
+    ),
+    p95Ms,
+  });
+}
+
+function normalizeCases(cases) {
+  if (!Array.isArray(cases) || cases.length < 1 || cases.length > 10_000) {
+    throw new TypeError("decision benchmark cases are invalid or unbounded");
+  }
+  return cases.map((item, index) => {
+    if (
+      !item ||
+      typeof item.query !== "string" ||
+      item.query.trim() === "" ||
+      !Array.isArray(item.candidates) ||
+      item.candidates.length < 1 ||
+      !Array.isArray(item.acceptableDigests) ||
+      item.acceptableDigests.some((digest) => !DIGEST.test(digest)) ||
+      new Set(item.acceptableDigests).size !== item.acceptableDigests.length
+    ) {
+      throw new TypeError(`decision benchmark case ${index} is invalid`);
+    }
+    const candidateDigests = new Set(
+      item.candidates.map(({ digest }) => digest),
+    );
+    if (
+      item.acceptableDigests.some((digest) => !candidateDigests.has(digest))
+    ) {
+      throw new TypeError(
+        `decision benchmark case ${index} has truth outside the candidates`,
+      );
+    }
+    return Object.freeze({
+      query: item.query.trim(),
+      candidates: Object.freeze(
+        item.candidates.map((candidate) => Object.freeze({ ...candidate })),
+      ),
+      acceptableDigests: Object.freeze([...item.acceptableDigests]),
+    });
+  });
+}
+
+export async function runSkillDecisionBenchmark({
+  runtime,
+  cases,
+  thresholds,
+  turnIdPrefix = "benchmark",
+  contextRevision = "benchmark-v1",
+  clock = () => performance.now(),
+} = {}) {
+  if (!runtime || typeof runtime.suggest !== "function") {
+    throw new TypeError("decision benchmark requires a runtime");
+  }
+  const normalizedCases = normalizeCases(cases);
+  const gate = normalizeThresholds(thresholds);
+  const latencies = [];
+  let accepted = 0;
+  let acceptedErrors = 0;
+  let positives = 0;
+  let affirmative = 0;
+  let noMatches = 0;
+  let noMatchFalseSuggestions = 0;
+  let abstentions = 0;
+  let unavailable = 0;
+  const rows = [];
+  for (const [index, item] of normalizedCases.entries()) {
+    const start = Number(clock());
+    const result = await runtime.suggest({
+      query: item.query,
+      candidates: item.candidates,
+      turnId: `${turnIdPrefix}-${index}`,
+      contextRevision,
+    });
+    const end = Number(clock());
+    if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) {
+      throw new Error("decision benchmark clock is invalid");
+    }
+    latencies.push(end - start);
+    const isPositive = item.acceptableDigests.length > 0;
+    if (isPositive) positives += 1;
+    else noMatches += 1;
+    if (result.status === "suggestion") {
+      accepted += 1;
+      if (isPositive) affirmative += 1;
+      const correct = item.acceptableDigests.includes(result.selectedDigest);
+      if (!correct) acceptedErrors += 1;
+      if (!isPositive) noMatchFalseSuggestions += 1;
+    } else if (result.status === "abstain") abstentions += 1;
+    else if (result.status === "unavailable") unavailable += 1;
+    rows.push(
+      Object.freeze({
+        index,
+        status: result.status,
+        selectedDigest: result.selectedDigest,
+        acceptableDigests: item.acceptableDigests,
+        latencyMs: end - start,
+      }),
+    );
+  }
+  latencies.sort((left, right) => left - right);
+  const metrics = Object.freeze({
+    acceptedSuggestionErrorRate: acceptedErrors / Math.max(1, accepted),
+    noMatchFalseSuggestionRate:
+      noMatchFalseSuggestions / Math.max(1, noMatches),
+    affirmativeCoverage: affirmative / Math.max(1, positives),
+    abstentionRate: abstentions / normalizedCases.length,
+    unavailableRate: unavailable / normalizedCases.length,
+    p95Ms: latencies[Math.max(0, Math.ceil(latencies.length * 0.95) - 1)],
+  });
+  const core = {
+    schema: SKILL_DECISION_BENCHMARK_SCHEMA,
+    caseCount: normalizedCases.length,
+    caseDigest: decisionDigest(
+      "chainlesschain.skill-decision-benchmark-cases/v1",
+      normalizedCases,
+    ),
+    thresholds: gate,
+    metrics,
+    passed:
+      metrics.acceptedSuggestionErrorRate <= gate.acceptedSuggestionErrorRate &&
+      metrics.noMatchFalseSuggestionRate <= gate.noMatchFalseSuggestionRate &&
+      metrics.affirmativeCoverage >= gate.affirmativeCoverage &&
+      metrics.p95Ms <= gate.p95Ms,
+    rows: Object.freeze(rows),
+  };
+  return Object.freeze({
+    ...core,
+    reportDigest: decisionDigest(SKILL_DECISION_BENCHMARK_SCHEMA, core),
+  });
+}

package/src/lib/decision-layer/contracts.js

@@ -0,0 +1,377 @@
+import { createHash } from "node:crypto";
+import { types as utilTypes } from "node:util";
+
+export const SKILL_DECISION_REQUEST_SCHEMA =
+  "chainlesschain.skill-decision-request/v1";
+export const SKILL_DECISION_RESULT_SCHEMA =
+  "chainlesschain.skill-decision-result/v1";
+export const SKILL_DECISION_OBSERVATION_SCHEMA =
+  "chainlesschain.skill-decision-observation/v1";
+
+export const DECISION_MODES = Object.freeze(["off", "shadow", "suggest"]);
+export const MAX_DECISION_CANDIDATES = 8;
+
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$/u;
+
+function canonical(value) {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+  return `{${Object.keys(value)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`)
+    .join(",")}}`;
+}
+
+export function decisionDigest(domain, value) {
+  return `sha256:${createHash("sha256")
+    .update(domain)
+    .update("\0")
+    .update(canonical(value))
+    .digest("hex")}`;
+}
+
+function plainObject(value, label) {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    utilTypes.isProxy(value) ||
+    ![Object.prototype, null].includes(Object.getPrototypeOf(value))
+  ) {
+    throw new TypeError(`${label} must be a plain object`);
+  }
+  for (const key of Reflect.ownKeys(value)) {
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    if (
+      typeof key !== "string" ||
+      !descriptor?.enumerable ||
+      !("value" in descriptor)
+    ) {
+      throw new TypeError(`${label} contains unsupported properties`);
+    }
+  }
+  return value;
+}
+
+function boundedText(value, label, maximum) {
+  if (
+    typeof value !== "string" ||
+    value.trim() === "" ||
+    value.length > maximum ||
+    /\p{Cc}/u.test(value)
+  ) {
+    throw new TypeError(`${label} is invalid or unbounded`);
+  }
+  return value.trim();
+}
+
+function identifier(value, label) {
+  const normalized = boundedText(value, label, 256);
+  if (!SAFE_ID.test(normalized)) throw new TypeError(`${label} is invalid`);
+  return normalized;
+}
+
+function probability(value, label) {
+  if (
+    typeof value !== "number" ||
+    !Number.isFinite(value) ||
+    value < 0 ||
+    value > 1
+  ) {
+    throw new TypeError(`${label} must be a finite probability`);
+  }
+  return value;
+}
+
+function normalizeCandidate(candidate, index) {
+  plainObject(candidate, `candidates[${index}]`);
+  const digest = candidate.digest;
+  if (!DIGEST.test(digest ?? "")) {
+    throw new TypeError(`candidates[${index}].digest is invalid`);
+  }
+  const tags = Array.isArray(candidate.tags)
+    ? candidate.tags.map((tag, tagIndex) =>
+        boundedText(tag, `candidates[${index}].tags[${tagIndex}]`, 128),
+      )
+    : [];
+  if (tags.length > 32 || new Set(tags).size !== tags.length) {
+    throw new TypeError(`candidates[${index}].tags are invalid or unbounded`);
+  }
+  return Object.freeze({
+    candidateId: `c${index + 1}`,
+    id: identifier(candidate.id, `candidates[${index}].id`),
+    displayName: boundedText(
+      candidate.displayName || candidate.id,
+      `candidates[${index}].displayName`,
+      512,
+    ),
+    description: boundedText(
+      candidate.description || "(no description)",
+      `candidates[${index}].description`,
+      16_384,
+    ),
+    category: boundedText(
+      candidate.category || "uncategorized",
+      `candidates[${index}].category`,
+      128,
+    ),
+    digest,
+    tags: Object.freeze(tags),
+  });
+}
+
+export function normalizeDecisionMode(value = "off") {
+  if (!DECISION_MODES.includes(value)) {
+    throw new TypeError("decision mode must be off, shadow, or suggest");
+  }
+  return value;
+}
+
+export function createSkillDecisionRequest({
+  decisionId,
+  tenantId,
+  sessionId,
+  turnId,
+  query,
+  candidates,
+  policyDigest,
+  contextRevision,
+} = {}) {
+  const normalizedCandidates = Array.isArray(candidates)
+    ? candidates.map(normalizeCandidate)
+    : null;
+  if (
+    !normalizedCandidates ||
+    normalizedCandidates.length < 1 ||
+    normalizedCandidates.length > MAX_DECISION_CANDIDATES
+  ) {
+    throw new TypeError("decision candidates are invalid or unbounded");
+  }
+  if (
+    new Set(normalizedCandidates.map(({ digest }) => digest)).size !==
+    normalizedCandidates.length
+  ) {
+    throw new TypeError("decision candidates require unique content digests");
+  }
+  const binding = Object.freeze({
+    tenantId: identifier(tenantId, "tenantId"),
+    sessionId: identifier(sessionId, "sessionId"),
+    turnId: identifier(turnId, "turnId"),
+    policyDigest: DIGEST.test(policyDigest ?? "")
+      ? policyDigest
+      : decisionDigest("chainlesschain.skill-decision-policy/v1", {
+          version: 1,
+        }),
+    contextRevision: identifier(
+      contextRevision || "current",
+      "contextRevision",
+    ),
+  });
+  const candidateSetDigest = decisionDigest(
+    "chainlesschain.skill-decision-candidates/v1",
+    normalizedCandidates,
+  );
+  const state = Object.freeze({
+    task: boundedText(query, "query", 4096),
+    candidates: Object.freeze(
+      normalizedCandidates.map((candidate) =>
+        Object.freeze({
+          candidateId: candidate.candidateId,
+          name: candidate.displayName,
+          description: candidate.description,
+          category: candidate.category,
+          tags: candidate.tags,
+        }),
+      ),
+    ),
+  });
+  if (Buffer.byteLength(canonical(state), "utf8") > 96 * 1024) {
+    throw new TypeError("decision state is too large");
+  }
+  const criteria = Object.fromEntries(
+    normalizedCandidates.map((candidate) => [
+      candidate.candidateId,
+      `Use ${candidate.displayName}: ${candidate.description}`,
+    ]),
+  );
+  criteria.none = "None of the listed skills is suitable for this task";
+  const questions = {
+    needs_skill: {
+      type: "noul",
+      instructions:
+        "Does this task need one of the listed skills to be handled well?",
+    },
+    best_skill: {
+      type: "choice",
+      instructions:
+        "Which listed skill is the best fit for the task? Choose none when every listed skill is unsuitable.",
+      criteria,
+    },
+  };
+  for (const candidate of normalizedCandidates) {
+    questions[`fits_${candidate.candidateId}`] = {
+      type: "noul",
+      instructions: `Is ${candidate.candidateId} suitable for the task and its stated boundaries?`,
+    };
+  }
+  const core = {
+    schema: SKILL_DECISION_REQUEST_SCHEMA,
+    decisionId: identifier(decisionId, "decisionId"),
+    purpose: "skill-routing",
+    binding,
+    candidateSetDigest,
+    candidates: Object.freeze(normalizedCandidates),
+    payload: Object.freeze({ state, questions: Object.freeze(questions) }),
+  };
+  return Object.freeze({
+    ...core,
+    requestDigest: decisionDigest(SKILL_DECISION_REQUEST_SCHEMA, core),
+  });
+}
+
+function normalizeNoul(answer, label) {
+  plainObject(answer, label);
+  if (answer.type !== undefined && answer.type !== "noul") {
+    throw new TypeError(`${label} has the wrong answer type`);
+  }
+  return Object.freeze({
+    type: "noul",
+    noul: probability(answer.noul, `${label}.noul`),
+  });
+}
+
+function normalizeChoice(answer, allowed, label) {
+  plainObject(answer, label);
+  if (answer.type !== undefined && answer.type !== "choice") {
+    throw new TypeError(`${label} has the wrong answer type`);
+  }
+  if (!allowed.has(answer.choice)) {
+    throw new TypeError(`${label}.choice is outside the candidate set`);
+  }
+  plainObject(answer.probabilities, `${label}.probabilities`);
+  if (
+    Object.keys(answer.probabilities).length !== allowed.size ||
+    Object.keys(answer.probabilities).some((key) => !allowed.has(key))
+  ) {
+    throw new TypeError(`${label}.probabilities are incomplete`);
+  }
+  const probabilities = Object.fromEntries(
+    [...allowed].map((key) => [
+      key,
+      probability(answer.probabilities[key], `${label}.probabilities.${key}`),
+    ]),
+  );
+  const sum = Object.values(probabilities).reduce(
+    (total, value) => total + value,
+    0,
+  );
+  if (Math.abs(sum - 1) > 0.02) {
+    throw new TypeError(`${label}.probabilities do not sum to one`);
+  }
+  return Object.freeze({
+    type: "choice",
+    choice: answer.choice,
+    confidence: probability(answer.confidence, `${label}.confidence`),
+    probabilities: Object.freeze(probabilities),
+  });
+}
+
+export function normalizeSkillDecisionProviderResult(request, value) {
+  plainObject(value, "decision provider result");
+  plainObject(value.answers, "decision provider result.answers");
+  const expectedKeys = [
+    "needs_skill",
+    "best_skill",
+    ...request.candidates.map(({ candidateId }) => `fits_${candidateId}`),
+  ];
+  if (
+    Object.keys(value.answers).length !== expectedKeys.length ||
+    expectedKeys.some((key) => !Object.hasOwn(value.answers, key))
+  ) {
+    throw new TypeError("decision provider answers are incomplete");
+  }
+  const allowed = new Set([
+    ...request.candidates.map(({ candidateId }) => candidateId),
+    "none",
+  ]);
+  const answers = {
+    needs_skill: normalizeNoul(
+      value.answers.needs_skill,
+      "answers.needs_skill",
+    ),
+    best_skill: normalizeChoice(
+      value.answers.best_skill,
+      allowed,
+      "answers.best_skill",
+    ),
+  };
+  for (const candidate of request.candidates) {
+    const key = `fits_${candidate.candidateId}`;
+    answers[key] = normalizeNoul(value.answers[key], `answers.${key}`);
+  }
+  const core = {
+    schema: SKILL_DECISION_RESULT_SCHEMA,
+    decisionId: request.decisionId,
+    requestDigest: request.requestDigest,
+    candidateSetDigest: request.candidateSetDigest,
+    provider: boundedText(value.provider, "provider", 160),
+    model: boundedText(value.model, "model", 160),
+    answers: Object.freeze(answers),
+  };
+  return Object.freeze({
+    ...core,
+    usage:
+      value.usage && typeof value.usage === "object"
+        ? Object.freeze({ ...value.usage })
+        : null,
+    resultDigest: decisionDigest(SKILL_DECISION_RESULT_SCHEMA, core),
+  });
+}
+
+export function resolveSkillDecision(request, result, thresholds = {}) {
+  const needsSkill = thresholds.needsSkill ?? 0.5;
+  const candidateFit = thresholds.candidateFit ?? 0.5;
+  const choiceConfidence = thresholds.choiceConfidence ?? 0.5;
+  for (const [name, value] of Object.entries({
+    needsSkill,
+    candidateFit,
+    choiceConfidence,
+  })) {
+    probability(value, `thresholds.${name}`);
+  }
+  const choice = result.answers.best_skill;
+  let status = "suggestion";
+  let reasonCode = "accepted";
+  let selected = null;
+  if (result.answers.needs_skill.noul < needsSkill) {
+    status = "no-match";
+    reasonCode = "needs-skill-below-threshold";
+  } else if (choice.choice === "none") {
+    status = "no-match";
+    reasonCode = "choice-none";
+  } else if (choice.confidence < choiceConfidence) {
+    status = "abstain";
+    reasonCode = "choice-confidence-below-threshold";
+  } else {
+    selected = request.candidates.find(
+      ({ candidateId }) => candidateId === choice.choice,
+    );
+    const fit = result.answers[`fits_${choice.choice}`].noul;
+    if (fit < candidateFit) {
+      status = "abstain";
+      reasonCode = "candidate-fit-below-threshold";
+      selected = null;
+    }
+  }
+  return Object.freeze({
+    status,
+    reasonCode,
+    selectedCandidateId: selected?.candidateId ?? null,
+    selectedDigest: selected?.digest ?? null,
+    selectedSkillId: selected?.id ?? null,
+    needsSkillProbability: result.answers.needs_skill.noul,
+    choiceConfidence: choice.confidence,
+    resultDigest: result.resultDigest,
+  });
+}

package/src/lib/decision-layer/provider-authority.js

@@ -0,0 +1,44 @@
+import { types as utilTypes } from "node:util";
+
+export const DECISION_PROVIDER_AUTHORITY_SCHEMA =
+  "chainlesschain.decision-provider-authority/v1";
+
+const AUTHORITIES = new WeakSet();
+
+function boundedText(value, label) {
+  if (
+    typeof value !== "string" ||
+    value.trim() === "" ||
+    value.length > 160 ||
+    /\p{Cc}/u.test(value)
+  ) {
+    throw new TypeError(`${label} is invalid or unbounded`);
+  }
+  return value.trim();
+}
+
+export function openDecisionProviderAuthority({
+  provider,
+  model,
+  decide,
+} = {}) {
+  if (typeof decide !== "function" || utilTypes.isProxy(decide)) {
+    throw new TypeError("decision provider decide() is required");
+  }
+  const operation = (...args) => Reflect.apply(decide, undefined, args);
+  const authority = Object.freeze({
+    schema: DECISION_PROVIDER_AUTHORITY_SCHEMA,
+    provider: boundedText(provider, "provider"),
+    model: boundedText(model, "model"),
+    decide: operation,
+  });
+  AUTHORITIES.add(authority);
+  return authority;
+}
+
+export function captureDecisionProviderAuthority(value) {
+  if (!AUTHORITIES.has(value)) {
+    throw new TypeError("a branded decision provider authority is required");
+  }
+  return value;
+}