@tangle-network/starter-foundry 0.7.3 → 0.13.2

package/registry/families/agent-research-harness-ts/files/src/research/proposer.ts

@@ -0,0 +1,215 @@
+/**
+ * Proposer — LLM-assisted hypothesis drafting via `runProposeReview`.
+ *
+ * Given a goal description + a snapshot of recent run state, returns a
+ * batch of candidate `Hypothesis` records the operator can append to
+ * `hypotheses/queue.json`. The proposer never writes to the queue
+ * itself — the human stays in the loop on what gets tested.
+ *
+ * The propose / verify / review primitive from agent-eval drives the
+ * inner loop: each shot proposes hypotheses, the verifier checks shape +
+ * coverage (categories, count, distinct ids), the reviewer either
+ * accepts or asks for a revision. Default cap: 3 shots.
+ */
+
+import {
+  runProposeReview,
+  type ProposeReviewReport,
+  type Verification,
+} from '@tangle-network/agent-eval'
+
+import type { Hypothesis, HypothesisCategory } from './types.js'
+
+export interface ProposerInput {
+  /** What the operator wants the harness to explore. */
+  goal: string
+  /** Number of hypotheses to generate. */
+  count?: number
+  /** Categories the proposer should cover (priority order). */
+  categories?: HypothesisCategory[]
+  /**
+   * LLM call hook. Returns a JSON-shaped object the proposer parses into
+   * `Hypothesis[]`. Consumers wire this to router.tangle.tools or any
+   * compatible inference endpoint.
+   */
+  callJson: (req: { system: string; user: string }) => Promise<unknown>
+  /** Existing queue ids to avoid duplicates. */
+  existingIds?: readonly string[]
+  maxShots?: number
+}
+
+export interface ProposerOutput {
+  hypotheses: Hypothesis[]
+  report: ProposeReviewReport<ProposerState, ProposerSummary>
+}
+
+interface ProposerState {
+  draft: Hypothesis[]
+  attempt: number
+}
+
+interface ProposerSummary {
+  rejectedCount: number
+  notes: string[]
+}
+
+const SYSTEM_PROMPT = `You are a research-program lead drafting hypotheses for a research-harness that runs prompt + steering optimization on @tangle-network/agent-eval. Output STRICT JSON of shape:
+
+{ "hypotheses": Hypothesis[] }
+
+Each Hypothesis must have:
+- id: kebab-case, unique, descriptive
+- name: short human label
+- rationale: WHY this should work + what evidence supports it
+- category: one of "bug-fix" | "architectural" | "efficiency" | "parameter-tuning"
+- expected_impact: e.g. "pass rate: +5-10pp" or "cost: -20%"
+- risk: what could regress
+- priority: 1 | 2 | 3
+- treatment: object describing the change (config delta, prompt delta, code change description)
+
+Rules:
+- Bug fixes first, then architectural, then efficiency, then parameter-tuning
+- No two hypotheses with the same id
+- No hypotheses that "tune to" a specific scenario id
+- Prefer architectural over parameter tuning when impact is comparable`
+
+function isCategory(value: unknown): value is HypothesisCategory {
+  return (
+    value === 'bug-fix' ||
+    value === 'architectural' ||
+    value === 'efficiency' ||
+    value === 'parameter-tuning'
+  )
+}
+
+function isPriority(value: unknown): value is 1 | 2 | 3 {
+  return value === 1 || value === 2 || value === 3
+}
+
+function parseHypotheses(raw: unknown): { hypotheses: Hypothesis[]; notes: string[] } {
+  const notes: string[] = []
+  if (raw === null || typeof raw !== 'object') {
+    return { hypotheses: [], notes: ['response was not a JSON object'] }
+  }
+  const root = raw as Record<string, unknown>
+  const list = root.hypotheses
+  if (!Array.isArray(list)) {
+    return { hypotheses: [], notes: ['response.hypotheses was not an array'] }
+  }
+  const out: Hypothesis[] = []
+  list.forEach((entry, idx) => {
+    if (entry === null || typeof entry !== 'object') {
+      notes.push(`item[${idx}] not an object`)
+      return
+    }
+    const e = entry as Record<string, unknown>
+    if (
+      typeof e.id !== 'string' ||
+      typeof e.name !== 'string' ||
+      typeof e.rationale !== 'string' ||
+      typeof e.expected_impact !== 'string' ||
+      !isCategory(e.category) ||
+      !isPriority(e.priority) ||
+      typeof e.treatment !== 'object' ||
+      e.treatment === null
+    ) {
+      notes.push(`item[${idx}] failed shape check`)
+      return
+    }
+    const hypothesis: Hypothesis = {
+      id: e.id,
+      name: e.name,
+      rationale: e.rationale,
+      category: e.category,
+      expected_impact: e.expected_impact,
+      priority: e.priority,
+      treatment: e.treatment as Record<string, unknown>,
+    }
+    if (typeof e.risk === 'string') hypothesis.risk = e.risk
+    out.push(hypothesis)
+  })
+  return { hypotheses: out, notes }
+}
+
+export async function propose(input: ProposerInput): Promise<ProposerOutput> {
+  const count = input.count ?? 5
+  const categories = input.categories ?? [
+    'bug-fix',
+    'architectural',
+    'efficiency',
+    'parameter-tuning',
+  ]
+  const existingIds = new Set(input.existingIds ?? [])
+  const maxShots = input.maxShots ?? 3
+
+  const report = await runProposeReview<ProposerState, ProposerSummary>({
+    goal: input.goal,
+    initialState: { draft: [], attempt: 0 },
+    maxShots,
+    propose: async ({ shot, goal, priorReview }) => {
+      const userPrompt = [
+        `Goal: ${goal}`,
+        `Generate ${count} hypotheses covering categories: ${categories.join(', ')}.`,
+        existingIds.size > 0
+          ? `Avoid these ids (already queued): ${[...existingIds].join(', ')}`
+          : 'No existing ids to avoid.',
+        priorReview
+          ? `Reviewer feedback: ${priorReview.nextShotInstruction}`
+          : 'This is the first attempt.',
+        'Return JSON only — no commentary.',
+      ].join('\n\n')
+      const raw = await input.callJson({ system: SYSTEM_PROMPT, user: userPrompt })
+      const { hypotheses, notes } = parseHypotheses(raw)
+      return {
+        state: { draft: hypotheses, attempt: shot },
+        traceSummary: { rejectedCount: notes.length, notes },
+      }
+    },
+    verify: async (state): Promise<Verification> => {
+      const issues: string[] = []
+      if (state.draft.length === 0) issues.push('no hypotheses parsed')
+      const ids = new Set<string>()
+      for (const h of state.draft) {
+        if (ids.has(h.id)) issues.push(`duplicate id ${h.id}`)
+        ids.add(h.id)
+        if (existingIds.has(h.id)) issues.push(`id ${h.id} already in queue`)
+      }
+      const haveCategories = new Set(state.draft.map((h) => h.category))
+      const requiredCategories = categories.filter((c) => c !== 'parameter-tuning')
+      for (const c of requiredCategories) {
+        if (!haveCategories.has(c)) issues.push(`category ${c} missing`)
+      }
+      const pass = issues.length === 0 && state.draft.length >= Math.min(count, 3)
+      return {
+        pass,
+        score: pass ? 1 : 0,
+        details: { issues, count: state.draft.length },
+      }
+    },
+    review: async ({ verification, state }) => {
+      const issues =
+        (verification.details as { issues?: string[] } | undefined)?.issues ?? []
+      if (verification.pass) {
+        return {
+          observations: `${state.draft.length} hypotheses parsed, all categories covered`,
+          diagnosis: 'shape valid',
+          nextShotInstruction: 'no further changes',
+          shouldContinue: false,
+          confidence: 0.9,
+        }
+      }
+      return {
+        observations: `verifier rejected: ${issues.join('; ')}`,
+        diagnosis: 'shape invalid or coverage insufficient',
+        nextShotInstruction: `Fix these issues: ${issues.join('; ')}. Return ${count} hypotheses minimum, all required categories covered, no duplicate or pre-existing ids.`,
+        shouldContinue: true,
+        confidence: 0.4,
+      }
+    },
+  })
+
+  return {
+    hypotheses: report.finalState.draft,
+    report,
+  }
+}

package/registry/families/agent-research-harness-ts/files/src/research/runner.ts

@@ -0,0 +1,253 @@
+/**
+ * Runner — orchestrates the hypothesis loop end-to-end.
+ *
+ * Responsibilities:
+ *   1. Load `hypotheses/queue.json`.
+ *   2. Drive the screener (1 rep / hypothesis).
+ *   3. Drive the validator (5 reps / passed-floor hypotheses) with
+ *      bootstrap-CI gates.
+ *   4. Persist artifacts to `research-results/<runId>/`.
+ *   5. (Optional) Compose `PairwiseSteeringOptimizer` from agent-eval for the
+ *      steering-bundle variant pathway when the operator passes
+ *      `--steering`.
+ *
+ * The runner is treatment-agnostic. It accepts a `ScenarioRunner` from
+ * the consumer and forwards each `Hypothesis` to it; how the treatment
+ * is applied is the consumer's call (config delta, prompt swap, code
+ * patch — the runner doesn't care).
+ */
+
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+import {
+  PairwiseSteeringOptimizer,
+  type SteeringOptimizationResult,
+  type SteeringOptimizationRow,
+  type SteeringOptimizerConfig,
+} from '@tangle-network/agent-eval'
+import {
+  analyzeRuns,
+  type AnalyzeRunsOptions,
+  type InsightReport,
+} from '@tangle-network/agent-eval/contract'
+
+import { screen, type ScreenerOptions } from './screener.js'
+import { validate, type ValidatorOptions } from './validator.js'
+import type {
+  Hypothesis,
+  HypothesisQueue,
+  ScenarioRunner,
+  ScreenerReport,
+  ValidatorReport,
+} from './types.js'
+
+export interface SweepOptions {
+  queuePath: string
+  resultsDir: string
+  runner: ScenarioRunner
+  screenerFloorTolerance?: number
+  validatorReps?: number
+  validatorAlpha?: number
+  validatorIterations?: number
+  validatorEffectFloor?: number
+  validatorSeed?: number
+  runId?: string
+  now?: () => Date
+}
+
+export interface SweepReport {
+  runId: string
+  queueSize: number
+  screen: ScreenerReport
+  validate: ValidatorReport | null
+  scorecardPath: string
+}
+
+export function loadQueue(queuePath: string): HypothesisQueue {
+  const raw = readFileSync(queuePath, 'utf8')
+  const parsed = JSON.parse(raw) as unknown
+  if (parsed === null || typeof parsed !== 'object') {
+    throw new Error(`hypothesis queue at ${queuePath} is not a JSON object`)
+  }
+  const root = parsed as Record<string, unknown>
+  const list = root.hypotheses
+  if (!Array.isArray(list)) {
+    throw new Error(`hypothesis queue at ${queuePath} missing 'hypotheses' array`)
+  }
+  // Trust the file shape minimally — full schema check belongs to the proposer.
+  return { hypotheses: list as Hypothesis[] }
+}
+
+function ensureDir(path: string): void {
+  mkdirSync(path, { recursive: true })
+}
+
+function writeJson(path: string, value: unknown): void {
+  ensureDir(dirname(path))
+  writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, 'utf8')
+}
+
+export async function runScreen(args: {
+  queuePath: string
+  resultsDir: string
+  runner: ScenarioRunner
+  floorTolerance?: number
+  runId?: string
+  now?: () => Date
+}): Promise<{ report: ScreenerReport; path: string }> {
+  const queue = loadQueue(args.queuePath)
+  const options: ScreenerOptions = {
+    hypotheses: queue.hypotheses,
+    runner: args.runner,
+  }
+  if (args.floorTolerance !== undefined) options.floorTolerance = args.floorTolerance
+  if (args.runId !== undefined) options.runId = args.runId
+  if (args.now !== undefined) options.now = args.now
+  const report = await screen(options)
+  const path = join(args.resultsDir, report.runId, 'screen.json')
+  writeJson(path, report)
+  return { report, path }
+}
+
+export async function runValidate(args: {
+  queuePath: string
+  resultsDir: string
+  runner: ScenarioRunner
+  candidateIds: readonly string[]
+  reps?: number
+  alpha?: number
+  iterations?: number
+  effectFloor?: number
+  seed?: number
+  runId?: string
+  now?: () => Date
+}): Promise<{ report: ValidatorReport; path: string }> {
+  const queue = loadQueue(args.queuePath)
+  const candidates = new Set(args.candidateIds)
+  const filtered = queue.hypotheses.filter((h) => candidates.has(h.id))
+  if (filtered.length === 0) {
+    throw new Error(
+      `validator received empty candidate set; refusing to run. queue=${queue.hypotheses.length} candidateIds=${args.candidateIds.length}`,
+    )
+  }
+  const options: ValidatorOptions = {
+    hypotheses: filtered,
+    runner: args.runner,
+  }
+  if (args.reps !== undefined) options.reps = args.reps
+  if (args.alpha !== undefined) options.alpha = args.alpha
+  if (args.iterations !== undefined) options.iterations = args.iterations
+  if (args.effectFloor !== undefined) options.effectFloor = args.effectFloor
+  if (args.seed !== undefined) options.seed = args.seed
+  if (args.runId !== undefined) options.runId = args.runId
+  if (args.now !== undefined) options.now = args.now
+  const report = await validate(options)
+  const path = join(args.resultsDir, report.runId, 'validate.json')
+  writeJson(path, report)
+  return { report, path }
+}
+
+export async function runSweep(options: SweepOptions): Promise<SweepReport> {
+  const now = options.now ?? (() => new Date())
+  const runId = options.runId ?? `sweep-${now().toISOString().replace(/[:.]/g, '-')}`
+  const queue = loadQueue(options.queuePath)
+
+  const screenArgs: Parameters<typeof runScreen>[0] = {
+    queuePath: options.queuePath,
+    resultsDir: options.resultsDir,
+    runner: options.runner,
+    runId: `${runId}-screen`,
+    now,
+  }
+  if (options.screenerFloorTolerance !== undefined) {
+    screenArgs.floorTolerance = options.screenerFloorTolerance
+  }
+  const { report: screenReport } = await runScreen(screenArgs)
+
+  let validateReport: ValidatorReport | null = null
+  if (screenReport.passedFloor.length > 0) {
+    const validateArgs: Parameters<typeof runValidate>[0] = {
+      queuePath: options.queuePath,
+      resultsDir: options.resultsDir,
+      runner: options.runner,
+      candidateIds: screenReport.passedFloor,
+      runId: `${runId}-validate`,
+      now,
+    }
+    if (options.validatorReps !== undefined) validateArgs.reps = options.validatorReps
+    if (options.validatorAlpha !== undefined) validateArgs.alpha = options.validatorAlpha
+    if (options.validatorIterations !== undefined) {
+      validateArgs.iterations = options.validatorIterations
+    }
+    if (options.validatorEffectFloor !== undefined) {
+      validateArgs.effectFloor = options.validatorEffectFloor
+    }
+    if (options.validatorSeed !== undefined) validateArgs.seed = options.validatorSeed
+    const { report } = await runValidate(validateArgs)
+    validateReport = report
+  }
+
+  const scorecardPath = join(options.resultsDir, runId, 'scorecard.json')
+  const scorecard = {
+    runId,
+    queueSize: queue.hypotheses.length,
+    screen: screenReport,
+    validate: validateReport,
+  }
+  writeJson(scorecardPath, scorecard)
+
+  return {
+    runId,
+    queueSize: queue.hypotheses.length,
+    screen: screenReport,
+    validate: validateReport,
+    scorecardPath,
+  }
+}
+
+/**
+ * Steering-bundle pathway — direct pass-through to agent-eval's
+ * `PairwiseSteeringOptimizer`. Use when the consumer has enumerated steering
+ * bundles (vs hypothesis-driven treatments).
+ */
+export async function runSteeringLoop(
+  rows: SteeringOptimizationRow[],
+  config?: SteeringOptimizerConfig,
+): Promise<SteeringOptimizationResult> {
+  return new PairwiseSteeringOptimizer().optimize(rows, config)
+}
+
+/**
+ * Bridge captured run records to the canonical decision packet:
+ * score distributions, cost/quality frontier, lift when baseline/candidate
+ * ids are paired, and ranked recommendations.
+ *
+ * Why this lives here:
+ *
+ *   - It's the natural callsite right after an optimization or promotion
+ *     loop emits `RunRecord[]` — the result is in scope, the adapter context
+ *     is in scope, and the consumer has just finished a sweep.
+ *
+ *   - Lift and cost/quality analysis run at sweep-level (they need the cohort
+ *     of variants, not a single run). Wiring it in the family runner
+ *     guarantees consumers see the decision packet on every research cycle.
+ *
+ *   - The report is read-only and evidence-backed; it never fabricates run
+ *     records from aggregate scores.
+ *
+ * Idempotent and read-only with respect to the optimization result.
+ *
+ * Reference wiring:
+ *   - `agent-builder/src/lib/.server/eval/auto-research-runner.ts`
+ *     (cycles 763–803 — the RL-bridge invocation after `runEvolution`).
+ *   - `@tangle-network/agent-eval/contract` for the `analyzeRuns` contract.
+ */
+export async function analyzeSweep(
+  options: AnalyzeOptimizationResultOptions,
+): Promise<AnalyzeOptimizationResultReport> {
+  return analyzeRuns(options)
+}
+
+export type AnalyzeOptimizationResultOptions = AnalyzeRunsOptions
+export type AnalyzeOptimizationResultReport = InsightReport

package/registry/families/agent-research-harness-ts/files/src/research/screener.ts

@@ -0,0 +1,105 @@
+/**
+ * Screener — cheap 1-rep pass over every queued hypothesis.
+ *
+ * Goal: rank hypotheses by point-estimate score delta vs the baseline so
+ * the validator only spends real budget on plausible winners. Statistical
+ * sign-off lives in the validator (bootstrapCi, cohensD); the screener
+ * exists to cull obvious losers before paying for 5-rep runs.
+ *
+ * Floor logic: a hypothesis "passes the floor" if its mean score across
+ * the screener scenario set is no worse than `baselineMean - tolerance`.
+ * Anything below that is rejected outright. Default tolerance = 0.02.
+ */
+
+import type {
+  Hypothesis,
+  HypothesisResult,
+  ScenarioRunner,
+  ScenarioSample,
+  ScreenerReport,
+} from './types.js'
+
+export interface ScreenerOptions {
+  hypotheses: Hypothesis[]
+  runner: ScenarioRunner
+  /** Score floor relative to baseline mean. Default 0.02. */
+  floorTolerance?: number
+  runId?: string
+  now?: () => Date
+}
+
+function meanOf(samples: ScenarioSample[], pick: (s: ScenarioSample) => number): number {
+  if (samples.length === 0) return 0
+  let sum = 0
+  for (const s of samples) sum += pick(s)
+  return sum / samples.length
+}
+
+async function runOne(
+  runner: ScenarioRunner,
+  hypothesis: Hypothesis | null,
+): Promise<ScenarioSample[]> {
+  const samples: ScenarioSample[] = []
+  for (const scenarioId of runner.scenarioIds) {
+    const sample = await runner.runTrial({ hypothesis, scenarioId, rep: 0 })
+    samples.push(sample)
+  }
+  return samples
+}
+
+export async function screen(options: ScreenerOptions): Promise<ScreenerReport> {
+  const { hypotheses, runner } = options
+  const floorTolerance = options.floorTolerance ?? 0.02
+  const now = options.now ?? (() => new Date())
+  const runId = options.runId ?? `screen-${now().toISOString().replace(/[:.]/g, '-')}`
+
+  if (runner.scenarioIds.length < 3) {
+    throw new Error(
+      `screener requires >= 3 scenarios to rank meaningfully; got ${runner.scenarioIds.length}`,
+    )
+  }
+
+  const baseline = await runOne(runner, null)
+  const baselineMean = meanOf(baseline, (s) => s.score)
+
+  const ranked: HypothesisResult[] = []
+  for (const hypothesis of hypotheses) {
+    const samples = await runOne(runner, hypothesis)
+    const meanScore = meanOf(samples, (s) => s.score)
+    const meanCostUsd = meanOf(samples, (s) => s.costUsd)
+    const meanWallSeconds = meanOf(samples, (s) => s.wallSeconds)
+    const delta = meanScore - baselineMean
+    const passesFloor = meanScore >= baselineMean - floorTolerance
+    ranked.push({
+      hypothesisId: hypothesis.id,
+      reps: 1,
+      meanScore,
+      meanBaseline: baselineMean,
+      delta,
+      // 1-rep ⇒ CI / Cohen's d are statistically undefined. Surface null
+      // so downstream consumers (validator, scorecard) cannot mistake
+      // a degenerate {delta, delta} CI for a real interval estimate.
+      ci95: null,
+      cohensD: null,
+      meanCostUsd,
+      meanWallSeconds,
+      // 'estimate-only' is a hard signal: validator MUST run multi-rep
+      // before promoting. Floor-failure still gets 'reject' so we don't
+      // waste validator budget on point-estimate losers.
+      verdict: passesFloor ? 'estimate-only' : 'reject',
+      reason: passesFloor
+        ? `screener delta ${delta.toFixed(4)} cleared floor (baseline ${baselineMean.toFixed(4)} - tolerance ${floorTolerance}); estimate-only — needs multi-rep validation`
+        : `screener mean ${meanScore.toFixed(4)} below baseline ${baselineMean.toFixed(4)} - tolerance ${floorTolerance}`,
+    })
+  }
+
+  ranked.sort((a, b) => b.delta - a.delta)
+  const passedFloor = ranked.filter((r) => r.verdict === 'estimate-only').map((r) => r.hypothesisId)
+
+  return {
+    runId,
+    generatedAt: now().toISOString(),
+    ranked,
+    passedFloor,
+  }
+}