@inneranimalmedia/agentsam-sdk 2.6.2 → 2.6.4

package/src/sam/decision/evaluators/semantic.js

@@ -0,0 +1,149 @@
+/**
+ * Semantic evaluator — optional LLM path via existing AgentSam model routing.
+ * Numbers from this evaluator are confidence_estimate / support only —
+ * NEVER labeled calibrated_probability without calibration_id.
+ */
+
+import { buildAnswer, concentrationFromScores, clamp01, softmax } from '../confidence.js';
+
+export function createSemanticEvaluator(options = {}) {
+  const complete = options.complete;
+
+  return Object.freeze({
+    kind: 'semantic',
+    version: '1',
+
+    supports(question) {
+      return question?.evaluator_hint === 'semantic';
+    },
+
+    async evaluate(state, questions) {
+      if (typeof complete !== 'function') {
+        const err = new Error('semantic_evaluator_unconfigured');
+        err.code = 'evaluator_unavailable';
+        throw err;
+      }
+
+      /** @type {Record<string, object>} */
+      const answers = {};
+      for (const q of questions) {
+        const prompt = buildPrompt(state, q);
+        const raw = await complete({ prompt, question: q, state });
+        answers[q.id] = normalizeSemanticAnswer(q, raw);
+      }
+      return {
+        ok: true,
+        evaluator: { kind: 'semantic', version: '1' },
+        answers,
+        model_used: true,
+        deterministic: false,
+      };
+    },
+  });
+}
+
+function buildPrompt(state, question) {
+  return {
+    question_id: question.id,
+    type: question.type,
+    instructions: question.instructions,
+    criteria: question.criteria,
+    options: question.options,
+    levels: question.levels,
+    state_facts: state.facts || {},
+    intent: state.intent,
+  };
+}
+
+function normalizeSemanticAnswer(question, raw) {
+  const warnings = ['semantic_values_are_estimates_not_calibrated_probabilities'];
+  if (!raw || typeof raw !== 'object') {
+    const err = new Error('semantic_answer_invalid');
+    err.code = 'evaluator_failed';
+    throw err;
+  }
+
+  if (question.type === 'choose') {
+    const value = String(raw.value ?? raw.choice ?? '');
+    if (!question.options.includes(value)) {
+      const err = new Error(`semantic_choice_not_in_options:${value}`);
+      err.code = 'evaluator_failed';
+      throw err;
+    }
+    const scores = raw.scores && typeof raw.scores === 'object'
+      ? Object.fromEntries(question.options.map((o) => [o, Number(raw.scores[o]) || 0]))
+      : Object.fromEntries(question.options.map((o) => [o, o === value ? 1 : 0]));
+    return buildAnswer({
+      type: 'choose',
+      question_id: question.id,
+      value,
+      scores,
+      confidence_estimate: Number.isFinite(Number(raw.confidence_estimate))
+        ? clamp01(Number(raw.confidence_estimate))
+        : concentrationFromScores(scores),
+      // Explicitly omit probabilities unless caller marks probabilistic=true AND supplies dist
+      probabilities: raw.probabilistic === true && raw.probabilities
+        ? raw.probabilities
+        : null,
+      evaluator: { kind: 'semantic', version: '1' },
+      warnings,
+      evidence: Array.isArray(raw.evidence) ? raw.evidence : [],
+    });
+  }
+
+  if (question.type === 'score') {
+    let value = Number(raw.value);
+    const scores = raw.scores && typeof raw.scores === 'object'
+      ? Object.fromEntries(question.levels.map((l) => [String(l.index), Number(raw.scores[l.index] ?? raw.scores[String(l.index)]) || 0]))
+      : null;
+    if (raw.probabilistic === true && raw.probabilities) {
+      const probs = Object.fromEntries(
+        question.levels.map((l) => [String(l.index), Number(raw.probabilities[l.index] ?? raw.probabilities[String(l.index)]) || 0]),
+      );
+      value = question.levels.reduce((n, l) => n + l.index * (probs[String(l.index)] || 0), 0);
+      return buildAnswer({
+        type: 'score',
+        question_id: question.id,
+        value,
+        levels: Object.fromEntries(question.levels.map((l) => [String(l.index), l.label])),
+        scores: scores || probs,
+        probabilities: probs,
+        confidence_estimate: concentrationFromScores(probs),
+        evaluator: { kind: 'semantic', version: '1' },
+        warnings,
+      });
+    }
+    if (!Number.isFinite(value)) {
+      const err = new Error('semantic_score_invalid');
+      err.code = 'evaluator_failed';
+      throw err;
+    }
+    return buildAnswer({
+      type: 'score',
+      question_id: question.id,
+      value,
+      levels: Object.fromEntries(question.levels.map((l) => [String(l.index), l.label])),
+      scores: scores || softmax(Object.fromEntries(question.levels.map((l) => [String(l.index), -Math.abs(l.index - value)]))),
+      confidence_estimate: Number.isFinite(Number(raw.confidence_estimate))
+        ? clamp01(Number(raw.confidence_estimate))
+        : 0.5,
+      evaluator: { kind: 'semantic', version: '1' },
+      warnings,
+    });
+  }
+
+  // check
+  const support = clamp01(Number(raw.support ?? raw.confidence_estimate ?? (raw.value ? 0.8 : 0.2)));
+  return buildAnswer({
+    type: 'check',
+    question_id: question.id,
+    value: Boolean(raw.value ?? support >= 0.5),
+    support,
+    evaluator: { kind: 'semantic', version: '1' },
+    warnings,
+    evidence: Array.isArray(raw.evidence) ? raw.evidence : [],
+  });
+}
+
+/** Default unconfigured instance — fails closed. */
+export const SEMANTIC_EVALUATOR = createSemanticEvaluator();

package/src/sam/decision/hierarchical.js

@@ -0,0 +1,61 @@
+/**
+ * Hierarchical choose + beam retention for large taxonomies (skills, tools, errors).
+ * When top options are close, keep multiple paths alive instead of greedy commit.
+ */
+
+/**
+ * @param {Array<{ id: string, score: number, parent?: string|null }>} candidates
+ * @param {object} [opts]
+ * @param {number} [opts.beamWidth]
+ * @param {number} [opts.closeMargin]  keep peers within this absolute score of the best
+ */
+export function beamRetain(candidates, opts = {}) {
+  const beamWidth = opts.beamWidth ?? 3;
+  const closeMargin = opts.closeMargin ?? 0.08;
+  const sorted = [...candidates].sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+  if (!sorted.length) return [];
+  const best = sorted[0].score;
+  const close = sorted.filter((c) => best - c.score <= closeMargin);
+  const beam = (close.length > 1 ? close : sorted).slice(0, beamWidth);
+  return beam;
+}
+
+/**
+ * Walk a taxonomy level-by-level with beam retention.
+ * @param {object} opts
+ * @param {Array<{ id: string, parent: string|null, score: number }>} opts.nodes
+ * @param {number} [opts.beamWidth]
+ * @param {number} [opts.closeMargin]
+ */
+export function hierarchicalChoose(opts = {}) {
+  const nodes = Array.isArray(opts.nodes) ? opts.nodes : [];
+  const roots = nodes.filter((n) => n.parent == null || n.parent === '');
+  let beam = beamRetain(roots, opts);
+  const path = [];
+  while (beam.length) {
+    // If multiple remain close, surface beam instead of committing.
+    if (beam.length > 1) {
+      return {
+        ok: true,
+        committed: false,
+        beam: beam.map((b) => b.id),
+        path: path.map((p) => p.id),
+        reason: 'close_candidates_retained',
+      };
+    }
+    const chosen = beam[0];
+    path.push(chosen);
+    const children = nodes.filter((n) => n.parent === chosen.id);
+    if (!children.length) {
+      return {
+        ok: true,
+        committed: true,
+        value: chosen.id,
+        path: path.map((p) => p.id),
+        beam: [chosen.id],
+      };
+    }
+    beam = beamRetain(children, opts);
+  }
+  return { ok: false, committed: false, beam: [], path: [], reason: 'empty_taxonomy' };
+}

package/src/sam/decision/index.js

@@ -0,0 +1,53 @@
+/**
+ * SAM decision module — choose / score / check + batch evaluate.
+ */
+
+export {
+  DECISION_STATE_SCHEMA,
+  DECISION_QUESTION_SCHEMA,
+  DECISION_ANSWER_SCHEMA,
+  DECISION_EVALUATION_SCHEMA,
+  DECISION_RECEIPT_SCHEMA,
+  DECISION_OUTCOME_SCHEMA,
+  DECISION_CALIBRATION_SCHEMA,
+} from './types.js';
+
+export { buildDecisionState, hashDecisionState, projectState } from './state.js';
+export {
+  defineChoose,
+  defineScore,
+  defineCheck,
+  validateQuestion,
+  validateQuestionBatch,
+} from './validate.js';
+export {
+  concentrationFromScores,
+  softmax,
+  expectedScore,
+  assertProbabilitiesSumToOne,
+  buildAnswer,
+  clamp01,
+} from './confidence.js';
+export { evaluate, evaluateDependentStages } from './evaluate.js';
+export { applyDecisionPolicy, composeUtility } from './policy.js';
+export { createDecisionReceipt, appendOutcome, attachAction, OUTCOME_DISPOSITIONS } from './receipt.js';
+export {
+  brierScore,
+  logLoss,
+  expectedCalibrationError,
+  exportCalibrationDataset,
+} from './calibration.js';
+export {
+  QUESTION_REGISTRY,
+  getRegisteredQuestion,
+  listRegisteredQuestions,
+} from './questions.js';
+export {
+  selectEvaluator,
+  DETERMINISTIC_EVALUATOR,
+  HEURISTIC_EVALUATOR,
+  SEMANTIC_EVALUATOR,
+  DEFAULT_CHAIN,
+} from './evaluators/select.js';
+export { createSemanticEvaluator } from './evaluators/semantic.js';
+export { hierarchicalChoose, beamRetain } from './hierarchical.js';

package/src/sam/decision/policy.js

@@ -0,0 +1,86 @@
+/**
+ * Policy owns permission — scores/confidence never grant authority alone.
+ */
+
+/**
+ * @param {object} opts
+ * @param {Record<string, object>} opts.answers
+ * @param {object} [opts.policy]
+ * @param {object} [opts.state]
+ */
+export function applyDecisionPolicy(opts = {}) {
+  const answers = opts.answers || {};
+  const policy = opts.policy || {};
+  const state = opts.state || {};
+  const required = [];
+  const denied = [];
+  const notes = [];
+
+  // Hard policy: production writes always require approval regardless of check support.
+  if (
+    policy.production_database_write === 'ALWAYS_REQUIRE_APPROVAL'
+    || state.constraints?.production_write
+    || state.facts?.production_write
+  ) {
+    required.push('user_approval');
+    notes.push('policy:production_write_always_requires_approval');
+  }
+
+  const approvalAnswer = answers.requires_approval;
+  if (approvalAnswer?.type === 'check' && approvalAnswer.value === true) {
+    required.push('user_approval');
+  }
+
+  // High confidence NEVER authorizes by itself.
+  for (const [id, answer] of Object.entries(answers)) {
+    if (
+      answer?.confidence_estimate != null
+      && answer.confidence_estimate >= 0.95
+      && policy.confidence_grants_authority
+    ) {
+      notes.push(`ignored_confidence_authority_claim:${id}`);
+    }
+  }
+
+  if (policy.deny_sandbox && answers.terminal_lane?.value === 'sandbox') {
+    denied.push('terminal_lane:sandbox');
+  }
+
+  const allowed = denied.length === 0;
+  return {
+    allowed,
+    require_approval: required.includes('user_approval'),
+    required: [...new Set(required)],
+    denied,
+    notes,
+    // Explicit: confidence is not permission
+    authorization_source: 'policy',
+  };
+}
+
+/**
+ * Composite utility from separate score/check answers — weights live in code.
+ * @param {object} parts
+ * @param {object} [weights]
+ */
+export function composeUtility(parts = {}, weights = {}) {
+  const w = {
+    goal_progress: weights.goal_progress ?? 0.4,
+    reversibility: weights.reversibility ?? 0.15,
+    risk: weights.risk ?? 0.25,
+    cost: weights.cost ?? 0.1,
+    confidence: weights.confidence ?? 0.1,
+  };
+  const goal = Number(parts.goal_progress ?? 0);
+  const rev = Number(parts.reversibility ?? 0);
+  const risk = Number(parts.risk ?? 0);
+  const cost = Number(parts.cost ?? 0);
+  const conf = Number(parts.confidence ?? 0);
+  return (
+    goal * w.goal_progress
+    + rev * w.reversibility
+    + conf * w.confidence
+    - risk * w.risk
+    - cost * w.cost
+  );
+}

package/src/sam/decision/questions.js

@@ -0,0 +1,120 @@
+/**
+ * Initial AgentSam decision question registry.
+ */
+
+import { defineChoose, defineScore, defineCheck } from './validate.js';
+
+export const QUESTION_REGISTRY = Object.freeze({
+  terminal_lane: defineChoose({
+    id: 'terminal_lane',
+    version: 1,
+    evaluator_hint: 'deterministic',
+    instructions: {
+      question: 'Which execution lane best fits this task?',
+      focus: 'Use isolation, repository locality, availability and cost.',
+    },
+    criteria: {
+      local: {
+        use_when: ['task requires unsynced local files', 'local terminal healthy'],
+        avoid_when: ['strong isolation required'],
+      },
+      remote: {
+        use_when: ['shared remote environment needed'],
+        avoid_when: ['requires local-only paths'],
+      },
+      sandbox: {
+        use_when: ['code is untrusted', 'strong isolation is required'],
+        avoid_when: ['task requires direct access to unsynced local files'],
+      },
+    },
+    options: ['local', 'remote', 'sandbox'],
+  }),
+
+  retrieval_mode: defineChoose({
+    id: 'retrieval_mode',
+    version: 1,
+    evaluator_hint: 'deterministic',
+    instructions: 'Which retrieval corpus should SAM query first?',
+    options: ['ast', 'lexical', 'semantic', 'memory', 'web', 'none'],
+    criteria: {
+      ast: { use_when: ['symbol / import / call-graph questions'] },
+      lexical: { use_when: ['exact string / path lookup'] },
+      semantic: { use_when: ['architectural concept questions'] },
+      memory: { use_when: ['historical decisions'] },
+      web: { use_when: ['fresh provider docs'] },
+      none: { use_when: ['no retrieval needed'] },
+    },
+  }),
+
+  change_risk: defineScore({
+    id: 'change_risk',
+    version: 1,
+    evaluator_hint: 'deterministic',
+    instructions: 'How much operational risk does this proposed change carry?',
+    levels: [
+      { index: 0, label: 'Local, reversible change with no runtime impact' },
+      { index: 1, label: 'Bounded implementation change with focused verification' },
+      { index: 2, label: 'Cross-package change with known consumers' },
+      { index: 3, label: 'Runtime/schema/security boundary change' },
+      { index: 4, label: 'High-blast-radius or production-sensitive change' },
+    ],
+  }),
+
+  requires_approval: defineCheck({
+    id: 'requires_approval',
+    version: 1,
+    evaluator_hint: 'deterministic',
+    instructions: {
+      question: 'Does this action require explicit user approval?',
+      inspect: ['action.reversibility', 'action.production_effect', 'policy'],
+    },
+  }),
+
+  security_review_required: defineCheck({
+    id: 'security_review_required',
+    version: 1,
+    evaluator_hint: 'deterministic',
+    instructions: {
+      question: 'Does this change require a security review before merge?',
+      inspect: ['security_boundary', 'auth_change', 'secret_handling'],
+    },
+  }),
+
+  verification_scope: defineChoose({
+    id: 'verification_scope',
+    version: 1,
+    evaluator_hint: 'deterministic',
+    instructions: 'How broad should verification be for this change?',
+    options: ['focused', 'package', 'workspace', 'repository'],
+  }),
+});
+
+/**
+ * @param {string} id
+ * @param {object} [overrides] e.g. dynamic options for skill_candidate
+ */
+export function getRegisteredQuestion(id, overrides = {}) {
+  if (id === 'skill_candidate') {
+    const options = Array.isArray(overrides.options) ? overrides.options : [];
+    return defineChoose({
+      id: 'skill_candidate',
+      version: 1,
+      evaluator_hint: overrides.evaluator_hint || 'deterministic',
+      instructions: overrides.instructions || 'Which skill best fits this task from the retrieved candidates?',
+      options,
+      criteria: overrides.criteria || null,
+    });
+  }
+  const base = QUESTION_REGISTRY[id];
+  if (!base) {
+    const err = new Error(`unknown_question:${id}`);
+    err.code = 'unknown_question';
+    throw err;
+  }
+  if (!Object.keys(overrides).length) return base;
+  return { ...base, ...overrides, id: base.id, type: base.type };
+}
+
+export function listRegisteredQuestions() {
+  return Object.keys(QUESTION_REGISTRY);
+}

package/src/sam/decision/receipt.js

@@ -0,0 +1,148 @@
+/**
+ * Decision receipts + outcome reconciliation + run/step/action linkage.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { DECISION_RECEIPT_SCHEMA, DECISION_OUTCOME_SCHEMA } from './types.js';
+import { hashDecisionState } from './state.js';
+
+export const OUTCOME_DISPOSITIONS = Object.freeze([
+  'accepted',
+  'rejected',
+  'overridden',
+  'corrected_to',
+  'retry_required',
+  'verification_failed',
+  'user_intervened',
+  'success',
+  'failure',
+]);
+
+/**
+ * @param {object} opts
+ */
+export function createDecisionReceipt(opts = {}) {
+  const decision_id = opts.decision_id || `dec_${randomUUID().replace(/-/g, '').slice(0, 20)}`;
+  return {
+    schema: DECISION_RECEIPT_SCHEMA,
+    decision_id,
+    run_id: opts.run_id ?? null,
+    step_id: opts.step_id ?? null,
+    action_id: opts.action_id ?? null,
+    outcome_id: null,
+    question_contract: opts.question_contract || null,
+    question_ids: Array.isArray(opts.question_ids) ? opts.question_ids : [],
+    question_versions: opts.question_versions || {},
+    evaluator: opts.evaluator || null,
+    evaluators: opts.evaluators || [],
+    state_hash: opts.state_hash || (opts.state ? hashDecisionState(opts.state) : null),
+    answers: opts.answers || {},
+    policy_result: opts.policy_result || null,
+    evidence: Array.isArray(opts.evidence) ? opts.evidence : [],
+    action: opts.action ?? null,
+    outcome: null,
+    created_at: new Date().toISOString(),
+    why: opts.why || buildWhy(opts.answers, opts.policy_result),
+  };
+}
+
+/**
+ * Attach a consequential outcome to a receipt (pass/fail + human corrections).
+ * Passive /why with no correction is NOT treated as a strong correctness label.
+ *
+ * @param {object} receipt
+ * @param {object} outcome
+ */
+export function appendOutcome(receipt, outcome) {
+  if (!receipt || receipt.schema !== DECISION_RECEIPT_SCHEMA) {
+    const err = new Error('invalid_receipt');
+    err.code = 'invalid_receipt';
+    throw err;
+  }
+  const disposition = normalizeDisposition(outcome);
+  const outcome_id = outcome.outcome_id || `out_${randomUUID().replace(/-/g, '').slice(0, 16)}`;
+  const normalized = {
+    schema: DECISION_OUTCOME_SCHEMA,
+    outcome_id,
+    success: outcome.success != null ? Boolean(outcome.success) : disposition === 'accepted' || disposition === 'success',
+    disposition,
+    human_corrected: Boolean(outcome.human_corrected || disposition === 'overridden' || disposition === 'corrected_to' || disposition === 'user_intervened'),
+    corrected_to: outcome.corrected_to ?? null,
+    user_override: Boolean(outcome.user_override),
+    actual_class: outcome.actual_class ?? null,
+    duration_ms: Number.isFinite(Number(outcome.duration_ms)) ? Number(outcome.duration_ms) : null,
+    retries: Number.isInteger(outcome.retries) ? outcome.retries : null,
+    verification: Array.isArray(outcome.verification) ? outcome.verification : [],
+    // Passive acceptance is weak evidence — do not treat as calibrated label
+    label_strength: outcome.label_strength
+      || (outcome.human_corrected || disposition === 'corrected_to' || disposition === 'overridden'
+        ? 'supervised'
+        : disposition === 'accepted' && outcome.explicit_accept
+          ? 'explicit_accept'
+          : 'weak_passive'),
+    recorded_at: new Date().toISOString(),
+  };
+  return {
+    ...receipt,
+    action_id: outcome.action_id ?? receipt.action_id,
+    outcome_id,
+    outcome: normalized,
+    updated_at: normalized.recorded_at,
+  };
+}
+
+/**
+ * Link an executed action onto a receipt before outcome is known.
+ */
+export function attachAction(receipt, action = {}) {
+  if (!receipt || receipt.schema !== DECISION_RECEIPT_SCHEMA) {
+    const err = new Error('invalid_receipt');
+    err.code = 'invalid_receipt';
+    throw err;
+  }
+  const action_id = action.action_id || `act_${randomUUID().replace(/-/g, '').slice(0, 16)}`;
+  return {
+    ...receipt,
+    action_id,
+    action: {
+      action_id,
+      kind: action.kind || null,
+      summary: action.summary || null,
+      started_at: action.started_at || new Date().toISOString(),
+      ...action,
+    },
+    updated_at: new Date().toISOString(),
+  };
+}
+
+function normalizeDisposition(outcome = {}) {
+  if (outcome.disposition && OUTCOME_DISPOSITIONS.includes(outcome.disposition)) {
+    return outcome.disposition;
+  }
+  if (outcome.human_corrected && outcome.corrected_to != null) return 'corrected_to';
+  if (outcome.user_override) return 'overridden';
+  if (outcome.success === true) return 'accepted';
+  if (outcome.success === false) return 'failure';
+  return 'accepted';
+}
+
+function buildWhy(answers = {}, policy = null) {
+  const decisions = Object.entries(answers).map(([id, a]) => ({
+    id,
+    type: a.type,
+    value: a.value,
+    support: a.support ?? null,
+    confidence_estimate: a.confidence_estimate ?? null,
+  }));
+  return {
+    surface: 'agentsam.why.v1',
+    decisions,
+    policy: policy
+      ? {
+          allowed: policy.allowed,
+          require_approval: policy.require_approval,
+          authorization_source: policy.authorization_source,
+        }
+      : null,
+  };
+}