@inneranimalmedia/agentsam-sdk 2.6.3 → 2.6.5

package/test/integration/sam-decision.test.mjs

@@ -0,0 +1,584 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import {
+  buildDecisionState,
+  defineChoose,
+  defineScore,
+  defineCheck,
+  evaluate,
+  evaluateDependentStages,
+  applyDecisionPolicy,
+  composeUtility,
+  createDecisionReceipt,
+  appendOutcome,
+  QUESTION_REGISTRY,
+  getRegisteredQuestion,
+  brierScore,
+  logLoss,
+  expectedCalibrationError,
+  hierarchicalChoose,
+  beamRetain,
+  createSemanticEvaluator,
+  DECISION_STATE_SCHEMA,
+  DECISION_RECEIPT_SCHEMA,
+} from '../../src/sam/decision/index.js';
+import {
+  concentrationFromScores,
+  expectedScore,
+  softmax,
+  assertProbabilitiesSumToOne,
+  buildAnswer,
+} from '../../src/sam/decision/confidence.js';
+import { validateQuestion } from '../../src/sam/decision/validate.js';
+import { selectEvaluator, DETERMINISTIC_EVALUATOR, SEMANTIC_EVALUATOR } from '../../src/sam/decision/evaluators/select.js';
+import { AgentSamClient, ensureSeedOperations, resetSeedFlagForTests, clearSamRegistryForTests } from '../../src/sam/index.js';
+import { planAStar, planGoap } from '../../src/sam/planning/index.js';
+
+describe('SAM decision state', () => {
+  it('builds a versioned state contract', () => {
+    const state = buildDecisionState({
+      intent: { type: 'symbol_lookup' },
+      repository: { root: '/tmp/repo', dirty: { value: true, source: 'git.status' } },
+      facts: { requires_local_files: true },
+    });
+    assert.equal(state.schema, DECISION_STATE_SCHEMA);
+    assert.equal(state.repository.root, '/tmp/repo');
+    assert.equal(state.facts.requires_local_files, true);
+  });
+});
+
+describe('SAM question contracts', () => {
+  it('validates choose with finite options', () => {
+    const q = defineChoose({
+      id: 'lane',
+      options: ['local', 'remote'],
+      criteria: { local: { use_when: ['x'] }, remote: {} },
+    });
+    assert.equal(q.type, 'choose');
+    assert.deepEqual(q.options, ['local', 'remote']);
+  });
+
+  it('fails closed on choose without options', () => {
+    assert.throws(
+      () => defineChoose({ id: 'empty', options: [] }),
+      (err) => err.code === 'choose_options_required',
+    );
+  });
+
+  it('validates score with ordered levels', () => {
+    const q = defineScore({
+      id: 'risk',
+      levels: [
+        { index: 0, label: 'low' },
+        { index: 1, label: 'mid' },
+        { index: 2, label: 'high' },
+      ],
+    });
+    assert.equal(q.type, 'score');
+    assert.equal(q.levels.length, 3);
+  });
+
+  it('rejects duplicate score level indexes', () => {
+    assert.throws(
+      () => defineScore({
+        id: 'bad',
+        levels: [{ index: 1, label: 'a' }, { index: 1, label: 'b' }],
+      }),
+      (err) => err.code === 'score_levels_unordered',
+    );
+  });
+
+  it('validates check yes/no contract', () => {
+    const q = defineCheck({
+      id: 'needs_approval',
+      instructions: { question: 'Approve?', inspect: ['policy'] },
+    });
+    assert.equal(q.type, 'check');
+  });
+
+  it('keeps registry question ids stable', () => {
+    assert.ok(QUESTION_REGISTRY.terminal_lane);
+    assert.equal(getRegisteredQuestion('terminal_lane').id, 'terminal_lane');
+    assert.equal(getRegisteredQuestion('change_risk').type, 'score');
+  });
+
+  it('accepts structured string/object/array criteria', () => {
+    validateQuestion({
+      id: 'x',
+      type: 'choose',
+      options: ['a'],
+      criteria: { a: { use_when: ['one'] } },
+      instructions: ['step'],
+    });
+    validateQuestion({
+      id: 'y',
+      type: 'choose',
+      options: ['a'],
+      criteria: 'prose ok',
+      instructions: 'string ok',
+    });
+  });
+
+  it('fails closed on malformed criteria', () => {
+    assert.throws(
+      () => validateQuestion({ id: 'z', type: 'choose', options: ['a'], criteria: 42 }),
+      (err) => err.code === 'invalid_criteria',
+    );
+  });
+});
+
+describe('SAM batch evaluate independence', () => {
+  const baseState = {
+    intent: { type: 'architecture concept review' },
+    runtime: { terminal_lanes: { local: 'healthy', remote: 'healthy', sandbox: 'healthy' } },
+    constraints: { requires_local_files: true },
+    code: { affected_packages: ['agentsam-sdk', 'identity'] },
+    facts: { change_risk_hint: 1 },
+  };
+
+  it('one state serves multiple independent questions', async () => {
+    const result = await evaluate({
+      state: baseState,
+      questions: [
+        'terminal_lane',
+        'retrieval_mode',
+        'change_risk',
+        'requires_approval',
+        'security_review_required',
+        'verification_scope',
+      ],
+    });
+    assert.equal(result.ok, true);
+    assert.equal(result.answers.terminal_lane.value, 'local');
+    assert.equal(result.answers.retrieval_mode.value, 'semantic');
+    assert.equal(result.answers.change_risk.type, 'score');
+    assert.equal(result.answers.requires_approval.type, 'check');
+    assert.ok(result.receipt?.why?.decisions?.length >= 6);
+  });
+
+  it('deterministic batch equals separate deterministic evaluations', async () => {
+    const ids = ['terminal_lane', 'change_risk', 'requires_approval'];
+    const batch = await evaluate({ state: baseState, questions: ids, receipt: false });
+    for (const id of ids) {
+      const solo = await evaluate({ state: baseState, questions: [id], receipt: false });
+      assert.equal(solo.answers[id].value, batch.answers[id].value);
+      assert.deepEqual(solo.answers[id].scores, batch.answers[id].scores);
+    }
+  });
+
+  it('questions do not secretly consume previous answers', async () => {
+    let seenSibling = false;
+    const spy = {
+      kind: 'deterministic',
+      version: 'spy',
+      supports: () => true,
+      async evaluate(state, questions) {
+        const answers = {};
+        for (const q of questions) {
+          // Prove evaluator only sees state — no answer bag passed in.
+          assert.ok(state.schema);
+          answers[q.id] = buildAnswer({
+            type: 'check',
+            question_id: q.id,
+            value: false,
+            support: 0.1,
+            evaluator: { kind: 'deterministic', version: 'spy' },
+          });
+        }
+        return { ok: true, answers, model_used: false };
+      },
+    };
+    const result = await evaluate({
+      state: baseState,
+      questions: [
+        defineCheck({ id: 'a', evaluator_hint: 'deterministic' }),
+        defineCheck({ id: 'b', evaluator_hint: 'deterministic' }),
+      ],
+      evaluators: [spy],
+      receipt: false,
+    });
+    assert.equal(seenSibling, false);
+    assert.ok(result.answers.a);
+    assert.ok(result.answers.b);
+  });
+});
+
+describe('SAM evaluators', () => {
+  it('prefers deterministic when facts are available', () => {
+    const q = getRegisteredQuestion('terminal_lane');
+    const ev = selectEvaluator(q);
+    assert.equal(ev.kind, 'deterministic');
+  });
+
+  it('does not silently fall back to semantic', () => {
+    const q = defineChoose({
+      id: 'unknown_thing',
+      options: ['a', 'b'],
+      // no evaluator_hint — must not pick semantic
+    });
+    assert.throws(
+      () => selectEvaluator(q, [SEMANTIC_EVALUATOR]),
+      (err) => err.code === 'evaluator_unsupported',
+    );
+    assert.throws(
+      () => selectEvaluator(q, [DETERMINISTIC_EVALUATOR, SEMANTIC_EVALUATOR]),
+      (err) => err.code === 'evaluator_unsupported',
+    );
+  });
+
+  it('unsupported evaluator fails clearly', async () => {
+    await assert.rejects(
+      () => evaluate({
+        state: {},
+        questions: [defineChoose({ id: 'nope', options: ['x'], evaluator_hint: 'deterministic' })],
+      }),
+      (err) => err.code === 'evaluator_unsupported' || err.code === 'evaluation_incomplete',
+    );
+  });
+
+  it('semantic evaluator is not used as silent fallback and fails closed when unconfigured', async () => {
+    await assert.rejects(
+      () => evaluate({
+        state: {},
+        questions: [defineCheck({ id: 'semantic_only', evaluator_hint: 'semantic' })],
+      }),
+      (err) => err.code === 'evaluator_unavailable',
+    );
+  });
+
+  it('semantic numbers are labeled estimates, not calibrated probabilities', async () => {
+    const semantic = createSemanticEvaluator({
+      complete: async () => ({ value: true, support: 0.87 }),
+    });
+    const result = await evaluate({
+      state: {},
+      questions: [defineCheck({ id: 'sem_check', evaluator_hint: 'semantic' })],
+      evaluators: [semantic],
+      receipt: false,
+    });
+    const a = result.answers.sem_check;
+    assert.equal(a.support, 0.87);
+    assert.equal(a.calibrated_probability, null);
+    assert.ok(a.warnings.some((w) => /not_calibrated/.test(w)));
+  });
+});
+
+describe('SAM confidence contract', () => {
+  it('documents concentration as distribution shape, not P(correct)', () => {
+    const concentrated = concentrationFromScores({ a: 10, b: 0.1, c: 0.1 });
+    const uniform = concentrationFromScores({ a: 1, b: 1, c: 1 });
+    assert.ok(concentrated > uniform);
+    assert.ok(concentrated > 0.5);
+  });
+
+  it('probability distributions sum to 1 within tolerance', () => {
+    const probs = softmax({ a: 1, b: 2, c: 3 });
+    assert.equal(assertProbabilitiesSumToOne(probs), true);
+  });
+
+  it('score expected value is probability-weighted level position', () => {
+    const levels = [{ index: 0 }, { index: 1 }, { index: 2 }];
+    const value = expectedScore(levels, { 0: 0.2, 1: 0.5, 2: 0.3 });
+    assert.ok(Math.abs(value - 1.1) < 1e-9);
+  });
+
+  it('calibrated_probability requires calibration_id', () => {
+    assert.throws(
+      () => buildAnswer({
+        type: 'check',
+        question_id: 'x',
+        value: true,
+        calibrated_probability: 0.9,
+      }),
+      (err) => err.code === 'calibration_required',
+    );
+  });
+
+  it('maps probability_true without calibration to support with warning', () => {
+    const a = buildAnswer({
+      type: 'check',
+      question_id: 'x',
+      value: true,
+      probability_true: 0.77,
+    });
+    assert.equal(a.support, 0.77);
+    assert.ok(a.warnings.includes('probability_true_mapped_to_support_without_calibration'));
+  });
+});
+
+describe('SAM policy owns permission', () => {
+  it('policy overrides model confidence / low approval support', () => {
+    const answers = {
+      requires_approval: buildAnswer({
+        type: 'check',
+        question_id: 'requires_approval',
+        value: false,
+        support: 0.08,
+      }),
+      terminal_lane: buildAnswer({
+        type: 'choose',
+        question_id: 'terminal_lane',
+        value: 'local',
+        scores: { local: 5 },
+        confidence_estimate: 0.99,
+      }),
+    };
+    const policy = applyDecisionPolicy({
+      answers,
+      policy: { production_database_write: 'ALWAYS_REQUIRE_APPROVAL', confidence_grants_authority: true },
+      state: { facts: { production_write: true } },
+    });
+    assert.equal(policy.require_approval, true);
+    assert.equal(policy.authorization_source, 'policy');
+    assert.ok(policy.notes.some((n) => /ignored_confidence/.test(n)));
+  });
+
+  it('composeUtility keeps weights in code', () => {
+    const u = composeUtility(
+      { goal_progress: 1, reversibility: 1, risk: 0.5, cost: 0.2, confidence: 0.8 },
+      { goal_progress: 0.5, risk: 0.3 },
+    );
+    assert.ok(Number.isFinite(u));
+  });
+});
+
+describe('SAM receipts + calibration', () => {
+  it('receipt records evaluator and question versions', async () => {
+    const result = await evaluate({
+      state: { constraints: { requires_local_files: true }, runtime: { terminal_lanes: { local: true } } },
+      questions: ['terminal_lane'],
+    });
+    assert.equal(result.receipt.schema, DECISION_RECEIPT_SCHEMA);
+    assert.equal(result.receipt.question_versions.terminal_lane, 1);
+    assert.ok(result.receipt.evaluators?.length || result.receipt.evaluator);
+    assert.ok(result.receipt.why);
+  });
+
+  it('outcome can be appended', () => {
+    const receipt = createDecisionReceipt({
+      question_ids: ['requires_approval'],
+      question_versions: { requires_approval: 1 },
+      answers: {
+        requires_approval: buildAnswer({
+          type: 'check',
+          question_id: 'requires_approval',
+          value: true,
+          support: 0.9,
+        }),
+      },
+      evaluator: { kind: 'deterministic', version: '1' },
+    });
+    const withOutcome = appendOutcome(receipt, {
+      success: true,
+      human_corrected: false,
+      verification: ['policy_ok'],
+    });
+    assert.equal(withOutcome.outcome.success, true);
+    assert.equal(withOutcome.outcome.schema, 'agentsam.decision-outcome.v1');
+  });
+
+  it('calibration metrics have known fixture values', () => {
+    // Perfect calibration: always predict 1 when y=1 and 0 when y=0
+    const perfect = [
+      { p: 1, y: 1 },
+      { p: 0, y: 0 },
+      { p: 1, y: 1 },
+      { p: 0, y: 0 },
+    ];
+    assert.equal(brierScore(perfect), 0);
+    assert.ok(logLoss(perfect) < 1e-10);
+    assert.equal(expectedCalibrationError(perfect, 10).ece, 0);
+
+    // Classic Brier: one miss at p=0.5 → (0.5-1)^2 = 0.25
+    assert.equal(brierScore([{ p: 0.5, y: 1 }]), 0.25);
+  });
+});
+
+describe('SAM dependent stages + hierarchical beam', () => {
+  it('dependent-stage workflow requires explicit second state', async () => {
+    await assert.rejects(
+      () => evaluateDependentStages(
+        { state: { facts: { 'choose.stage': 'x' } }, questions: [] },
+        null,
+        {},
+      ),
+      (err) => err.code === 'invalid_dependent_stage' || err.code === 'empty_batch',
+    );
+
+    const stage1 = await evaluateDependentStages(
+      {
+        state: {
+          facts: { 'choose.pick': 'ast' },
+          intent: 'symbol lookup',
+          runtime: { terminal_lanes: { local: true } },
+        },
+        questions: ['retrieval_mode'],
+        receipt: false,
+      },
+      async (first) => ({
+        facts: { richer: true, 'choose.skill_candidate': 'mcp-oauth' },
+        intent: { text: 'audit the OAuth migration' },
+        skills: {
+          'mcp-oauth': { tags: ['oauth', 'mcp'] },
+          security: { tags: ['security'] },
+        },
+        retrieval: first.answers.retrieval_mode.value,
+      }),
+      {
+        questions: [
+          getRegisteredQuestion('skill_candidate', {
+            options: ['mcp-oauth', 'security', 'platform-context', 'deploy'],
+            criteria: {
+              'mcp-oauth': { tags: ['oauth', 'mcp', 'migration'] },
+              security: { tags: ['security'] },
+              'platform-context': { tags: ['platform'] },
+              deploy: { tags: ['deploy'] },
+            },
+          }),
+        ],
+        receipt: false,
+      },
+    );
+    assert.equal(stage1.stage1.answers.retrieval_mode.type, 'choose');
+    assert.equal(stage1.stage2.answers.skill_candidate.value, 'mcp-oauth');
+  });
+
+  it('beam/hierarchical choice retains multiple paths when close', () => {
+    const beam = beamRetain(
+      [
+        { id: 'a', score: 0.91 },
+        { id: 'b', score: 0.90 },
+        { id: 'c', score: 0.50 },
+      ],
+      { beamWidth: 3, closeMargin: 0.08 },
+    );
+    assert.deepEqual(beam.map((b) => b.id), ['a', 'b']);
+
+    const hier = hierarchicalChoose({
+      nodes: [
+        { id: 'security', parent: null, score: 0.8 },
+        { id: 'platform', parent: null, score: 0.79 },
+        { id: 'oauth', parent: 'security', score: 0.9 },
+      ],
+      closeMargin: 0.05,
+      beamWidth: 3,
+    });
+    assert.equal(hier.committed, false);
+    assert.ok(hier.beam.includes('security'));
+    assert.ok(hier.beam.includes('platform'));
+  });
+});
+
+describe('SAM real use cases', () => {
+  it('terminal lane: local files + healthy local → local', async () => {
+    const result = await evaluate({
+      state: {
+        constraints: { requires_local_files: true },
+        runtime: { terminal_lanes: { local: 'healthy', remote: 'healthy', sandbox: 'healthy' } },
+      },
+      questions: ['terminal_lane'],
+    });
+    assert.equal(result.answers.terminal_lane.value, 'local');
+    assert.ok(!('probabilities' in result.answers.terminal_lane) || result.answers.terminal_lane.probabilities == null);
+  });
+
+  it('cross-cutting change fan-out', async () => {
+    const result = await evaluate({
+      state: {
+        code: { affected_packages: ['a', 'b', 'c', 'd'] },
+        facts: { security_boundary: true, auth_change: true, production_write: true },
+        runtime: { terminal_lanes: { local: true, remote: true, sandbox: true } },
+        constraints: { isolation_required: true },
+      },
+      questions: [
+        'change_risk',
+        'requires_approval',
+        'terminal_lane',
+        'security_review_required',
+        'verification_scope',
+      ],
+      policy: { production_database_write: 'ALWAYS_REQUIRE_APPROVAL' },
+    });
+    assert.ok(result.answers.change_risk.value >= 3);
+    assert.equal(result.answers.requires_approval.value, true);
+    assert.equal(result.answers.terminal_lane.value, 'sandbox');
+    assert.equal(result.answers.security_review_required.value, true);
+    assert.equal(result.policy_result.require_approval, true);
+  });
+
+  it('skill selection for OAuth migration', async () => {
+    const result = await evaluate({
+      state: {
+        intent: { text: 'audit the OAuth migration for MCP routes' },
+        facts: { intent_tags: ['oauth', 'mcp', 'migration'] },
+      },
+      questions: [
+        getRegisteredQuestion('skill_candidate', {
+          options: ['mcp-oauth', 'security', 'platform-context', 'deploy'],
+          criteria: {
+            'mcp-oauth': { tags: ['oauth', 'mcp'], description: 'MCP OAuth flows' },
+            security: { tags: ['security', 'audit'] },
+            'platform-context': { tags: ['platform'] },
+            deploy: { tags: ['deploy'] },
+          },
+        }),
+      ],
+    });
+    assert.equal(result.answers.skill_candidate.value, 'mcp-oauth');
+  });
+});
+
+describe('SAM decision.evaluate operation', () => {
+  it('is registered and invokable via AgentSamClient', async () => {
+    clearSamRegistryForTests();
+    resetSeedFlagForTests();
+    ensureSeedOperations();
+    const sam = new AgentSamClient({ autoSeed: true });
+    const result = await sam.decision.evaluate({
+      state: {
+        constraints: { requires_local_files: true },
+        runtime: { terminal_lanes: { local: true } },
+      },
+      questions: ['terminal_lane'],
+    });
+    assert.equal(result.ok, true);
+    assert.equal(result.data.answers.terminal_lane.value, 'local');
+  });
+});
+
+describe('SAM A*/GOAP remain green alongside decision', () => {
+  it('A* still finds cheapest path', () => {
+    const edges = {
+      start: [
+        { action: 'via_a', nextState: 'a', cost: 1 },
+        { action: 'direct', nextState: 'goal', cost: 10 },
+      ],
+      a: [{ action: 'finish', nextState: 'goal', cost: 1 }],
+      goal: [],
+    };
+    const result = planAStar({
+      initialState: 'start',
+      isGoal: (s) => s === 'goal',
+      expand: (s) => edges[s] || [],
+      stateKey: (s) => String(s),
+    });
+    assert.equal(result.total_cost, 2);
+  });
+
+  it('GOAP still plans', () => {
+    const result = planGoap({
+      initialState: { have: false },
+      goal: { have: true },
+      actions: [
+        {
+          id: 'get',
+          cost: 1,
+          preconditions: { have: false },
+          effects: { have: true },
+        },
+      ],
+    });
+    assert.equal(result.ok, true);
+    assert.ok(result.plan.length >= 1);
+  });
+});

package/test/integration/sam-kernel.test.mjs

@@ -0,0 +1,99 @@
+import assert from 'node:assert/strict';
+import { describe, it, before } from 'node:test';
+import {
+  AgentSamClient,
+  defineSamOperation,
+  ensureSeedOperations,
+  getSamOperation,
+  listSamOperations,
+  SAM_EXPANSION,
+  SAM_RESULT_SCHEMA,
+} from '../../src/sam/index.js';
+
+describe('SAM kernel seed', () => {
+  before(() => {
+    ensureSeedOperations();
+  });
+
+  it('names Systematic Autonomous Machinery', () => {
+    assert.equal(SAM_EXPANSION, 'Systematic Autonomous Machinery');
+    assert.equal(SAM_RESULT_SCHEMA, 'agentsam.result.v1');
+  });
+
+  it('registers seed operations including planning', () => {
+    const ids = listSamOperations().map((o) => o.id);
+    for (const id of [
+      'repository.inspect',
+      'brand.scan',
+      'security.scan',
+      'terminal.exec',
+      'cad.blender.inspect',
+      'codebaseindex.ingest',
+      'planning.astar',
+      'planning.goap',
+    ]) {
+      assert.ok(ids.includes(id), `missing ${id}`);
+    }
+    assert.equal(getSamOperation('repository.inspect')?.execution.model, 'never');
+    assert.equal(getSamOperation('planning.astar')?.execution.model, 'never');
+    assert.equal(getSamOperation('codebaseindex.ingest')?.execution.model, 'optional');
+  });
+
+  it('defineSamOperation rejects incomplete defs', () => {
+    assert.throws(() => defineSamOperation({ id: 'x.y', handler: () => {} }), /module and action/);
+  });
+
+  it('describe returns metadata without executing', async () => {
+    const sam = new AgentSamClient();
+    const info = await sam.describe('security.scan');
+    assert.equal(info.ok, true);
+    assert.equal(info.id, 'security.scan');
+    assert.equal(info.execution.model, 'never');
+    assert.equal(info.risk, 'read_only');
+  });
+
+  it('discover finds blender inspect by query', async () => {
+    const sam = new AgentSamClient();
+    const found = await sam.discover({ query: 'blender inspect' });
+    assert.equal(found.ok, true);
+    assert.ok(found.operations.some((c) => c.id === 'cad.blender.inspect'));
+  });
+
+  it('invoke repository.inspect returns SamResult envelope', async () => {
+    const sam = new AgentSamClient({ cwd: process.cwd() });
+    const result = await sam.invoke('repository.inspect', { root: process.cwd() });
+    assert.equal(result.schema, SAM_RESULT_SCHEMA);
+    assert.equal(result.operation, 'repository.inspect');
+    assert.equal(result.ok, true);
+    assert.ok(result.receipt?.id?.startsWith('samr_'));
+    assert.equal(result.receipt.execution.model_used, false);
+    assert.equal(result.receipt.execution.deterministic, true);
+    assert.ok(result.data?.snapshot || result.data?.merkle_root);
+  });
+
+  it('domain projection matches invoke', async () => {
+    const sam = new AgentSamClient();
+    const viaModule = await sam.security.scan({ root: process.cwd(), offline: true });
+    assert.equal(viaModule.ok, true);
+    assert.equal(viaModule.operation, 'security.scan');
+    assert.equal(viaModule.data?.scanner, 'agentsam-sca');
+  });
+
+  it('invoke unknown operation fails cleanly', async () => {
+    const sam = new AgentSamClient();
+    const result = await sam.invoke('no.such.op', {});
+    assert.equal(result.ok, false);
+    assert.equal(result.error?.code, 'sam_operation_not_found');
+  });
+
+  it('terminal.exec via invoke', async () => {
+    const sam = new AgentSamClient();
+    const result = await sam.invoke('terminal.exec', {
+      command: 'node',
+      args: ['-e', 'process.stdout.write("ok")'],
+      cwd: process.cwd(),
+    });
+    assert.equal(result.ok, true);
+    assert.match(String(result.data?.stdout || ''), /ok/);
+  });
+});