@xingular/harness 2.1.2 → 3.0.0

package/src/refine/host.ts

@@ -0,0 +1,102 @@
+// The application's part in refinement (the seventh extension point). The harness runs the
+// method; the application says where an execution may work, as whom, with which tools, and what
+// it produced. Nothing here names an application concept.
+import { cpSync, existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, relative, sep } from 'node:path'
+
+import type { CallerContext } from '../core/caller'
+import type { Launch } from '../prompt/compile'
+import type { Effort } from '../sessions/runner'
+import type { RefineTask } from './types'
+
+/** How one execution of a task runs: who it works as and with what. */
+export interface RolloutSession {
+  caller: CallerContext
+  tools: { allow: string[]; gated?: string[] }
+  launch?: Launch
+  model?: string | null
+  effort?: Effort | null
+  accountId?: string | null
+  poolId?: string | null
+  connections?: boolean | { readOnly?: boolean }
+  providers?: boolean
+}
+
+export interface RolloutWorkspace {
+  /** The folder the execution works in. It must not be a folder a person is working in. */
+  cwd: string
+  /** Called after what the execution produced has been read. */
+  cleanup?: () => Promise<void>
+}
+
+export interface RefineHost {
+  /** A fresh, isolated folder for one execution. */
+  prepareWorkspace(input: { refineId: string; label: string; task: RefineTask }): Promise<RolloutWorkspace>
+  /** The session settings an execution runs with. */
+  session(input: { refineId: string; label: string; task: RefineTask }): Promise<RolloutSession>
+  /** What one execution produced, filename → readable text. Defaults to its reply and the files it changed. */
+  deliverable?(input: { cwd: string; reply: string; task: RefineTask }): Promise<Record<string, string>>
+}
+
+const TEXT_LIMIT = 200_000
+const FILES_LIMIT = 40
+
+function files(dir: string, into: string[] = [], base = dir): string[] {
+  let names: string[]
+  try {
+    names = readdirSync(dir)
+  } catch {
+    return into
+  }
+  for (const name of names.sort()) {
+    if (name === '.git' || name === 'node_modules' || name.startsWith('.')) continue
+    const path = join(dir, name)
+    try {
+      if (statSync(path).isDirectory()) files(path, into, base)
+      else into.push(relative(base, path).split(sep).join('/'))
+    } catch {
+      // Skip what cannot be read.
+    }
+    if (into.length > 2_000) break
+  }
+  return into
+}
+
+function readable(path: string): string | null {
+  try {
+    const buffer = readFileSync(path)
+    if (buffer.length > TEXT_LIMIT || buffer.includes(0)) return null
+    return buffer.toString('utf8')
+  } catch {
+    return null
+  }
+}
+
+/** The reply, and every text file the execution added or changed relative to the task's source. */
+export function defaultDeliverable(cwd: string, reply: string, sourceDir: string | null | undefined): Record<string, string> {
+  const out: Record<string, string> = {}
+  if (reply.trim()) out['reply.md'] = reply
+  let count = 0
+  for (const rel of files(cwd)) {
+    const text = readable(join(cwd, rel))
+    if (text === null) continue
+    const before = sourceDir && existsSync(join(sourceDir, rel)) ? readable(join(sourceDir, rel)) : null
+    if (before === text) continue
+    out[`files/${rel}`] = text
+    if (++count >= FILES_LIMIT) break
+  }
+  return out
+}
+
+/** For a host that supplies nothing: a temporary copy of the task's source, the caller as the scope, and Pi's file and shell tools. */
+export const DEFAULT_REFINE_HOST: RefineHost = {
+  async prepareWorkspace({ task }) {
+    const cwd = mkdtempSync(join(tmpdir(), 'harness-rollout-'))
+    if (task.sourceDir && existsSync(task.sourceDir)) cpSync(task.sourceDir, cwd, { recursive: true })
+    return { cwd, cleanup: async () => rmSync(cwd, { recursive: true, force: true }) }
+  },
+  async session({ task }) {
+    return { caller: { agentId: task.scope, actor: 'Code' }, tools: { allow: ['read', 'grep', 'find', 'ls', 'edit', 'write', 'bash'] } }
+  }
+}

package/src/refine/ledger.ts

@@ -0,0 +1,104 @@
+// Budget accounting: R = 4k rollouts and P = 3k improvement stages per mode. Base executions
+// count toward each mode; retries are separate attempt counts; selection and reflection are
+// recorded outside this ledger.
+//
+// Ported from Harness² (Apache-2.0, Google LLC; `harness2/core/ledger.py`).
+
+/** R = 4k: k shared base + 2k candidates + k finals. */
+export const expectedRollouts = (k: number): number => 4 * k
+/** P = 3k: k INV + k DOM + k compositions. */
+export const expectedSessions = (k: number): number => 3 * k
+
+export class BudgetError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'BudgetError'
+  }
+}
+
+export interface LedgerEntry {
+  kind: 'rollout' | 'session'
+  label: string
+  usd: number
+  cached: boolean
+  attempts: number
+}
+
+export interface LedgerRecord {
+  mode: string
+  taskId: string
+  k: number
+  R: number
+  expectedR: number
+  P: number
+  expectedP: number
+  sessionAttempts: number
+  usd: number
+  rolloutUsd: number
+  sessionUsd: number
+  rollouts: string[]
+  sessions: string[]
+}
+
+const round = (value: number): number => Math.round(value * 1e6) / 1e6
+
+export class Ledger {
+  readonly entries: LedgerEntry[] = []
+
+  constructor(readonly mode: string, readonly taskId: string, readonly k: number) {}
+
+  rollout(label: string, usd = 0, cached = false): void {
+    this.entries.push({ kind: 'rollout', label, usd, cached, attempts: 1 })
+  }
+
+  session(label: string, usd = 0, options: { cached?: boolean; attempts?: number } = {}): void {
+    this.entries.push({ kind: 'session', label, usd, cached: options.cached ?? false, attempts: options.attempts ?? 1 })
+  }
+
+  labels(kind: LedgerEntry['kind']): string[] {
+    return this.entries.filter((entry) => entry.kind === kind).map((entry) => entry.label)
+  }
+
+  get R(): number {
+    return this.labels('rollout').length
+  }
+
+  get P(): number {
+    return this.labels('session').length
+  }
+
+  usd(kind?: LedgerEntry['kind']): number {
+    return round(this.entries.filter((entry) => !kind || entry.kind === kind).reduce((sum, entry) => sum + entry.usd, 0))
+  }
+
+  /** Throws unless the entries show exactly R=4k and P=3k with no repeated label. */
+  assertIdentity(): void {
+    const wantR = expectedRollouts(this.k)
+    const wantP = expectedSessions(this.k)
+    const rl = this.labels('rollout')
+    const sl = this.labels('session')
+    const dupes = [...new Set([...rl.filter((label, i) => rl.indexOf(label) !== i), ...sl.filter((label, i) => sl.indexOf(label) !== i)])].sort()
+    if (!dupes.length && this.R === wantR && this.P === wantP) return
+    const detail = [`${this.mode}-${this.k} ${this.taskId}: R=${this.R} (want ${wantR}), P=${this.P} (want ${wantP})`, `  rollouts: ${rl.join(', ') || '(none)'}`, `  sessions: ${sl.join(', ') || '(none)'}`]
+    if (dupes.length) detail.push(`  DUPLICATE labels: ${dupes.join(', ')} -- two stages wrote the same run id and one overwrote the other`)
+    throw new BudgetError(detail.join('\n'))
+  }
+
+  toRecord(): LedgerRecord {
+    return {
+      mode: this.mode,
+      taskId: this.taskId,
+      k: this.k,
+      R: this.R,
+      expectedR: expectedRollouts(this.k),
+      P: this.P,
+      expectedP: expectedSessions(this.k),
+      sessionAttempts: this.entries.filter((entry) => entry.kind === 'session').reduce((sum, entry) => sum + entry.attempts, 0),
+      usd: this.usd(),
+      rolloutUsd: this.usd('rollout'),
+      sessionUsd: this.usd('session'),
+      rollouts: this.labels('rollout'),
+      sessions: this.labels('session')
+    }
+  }
+}

package/src/refine/library.ts

@@ -0,0 +1,192 @@
+// The experience library: what earlier refinements in one scope and area adopted, and why.
+// Read by improvers before each task, appended after it. The whole tree is carried, never a
+// summary.
+//
+//   <data>/library/<scope>/<area>/INDEX.md
+//   <data>/library/<scope>/<area>/harnesses/<task>/harness/<rel>   the full adopted tree
+//   <data>/library/<scope>/<area>/harnesses/<task>/EXPERIENCE.md   why it was picked, and the lesson
+//
+// Ported from Harness² (Apache-2.0, Google LLC; `harness2/core/library.py`).
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { safeId } from '../core/fs'
+import { PI_SURFACE, type SurfaceSpec, type Tree } from '../tree/surface'
+import { alwaysOnText, pluginNames, readTree, skillNames, subagentNames, writeTree } from '../tree/tree'
+import { patterns } from '../tree/surface'
+import { assertTreeHasNoGraderVerdicts } from './evidence'
+
+export interface Experience {
+  cue: string
+  lesson: string
+  evidence: string
+  confidence?: string
+  /** The edit-by-edit record the lesson rests on. */
+  edits?: string
+}
+
+export function hasExperience(experience: Partial<Experience> | null | undefined): experience is Experience {
+  return Boolean(experience) && (['cue', 'lesson', 'evidence'] as const).every((key) => typeof experience![key] === 'string' && experience![key]!.trim().length > 0)
+}
+
+/** One folder name for a task or area: separators collapse, nothing escapes. */
+export function slug(text: string): string {
+  return text.replace(/\s*[/\\]\s*/g, '__').replace(/[^A-Za-z0-9_.\-]/g, '_').replace(/^\.+/, '_').slice(0, 120).trim() || 'unknown'
+}
+
+export interface LibraryEntry {
+  task: string
+  label: string
+  runId: string
+  cue: string | null
+  inventory: string
+}
+
+export function libraryRoot(root: string): string {
+  return join(root, 'library')
+}
+
+export class Library {
+  constructor(readonly root: string, readonly scope: string, readonly area: string = scope, readonly surface: SurfaceSpec = PI_SURFACE) {
+    safeId(scope)
+  }
+
+  get dir(): string {
+    return join(libraryRoot(this.root), this.scope, slug(this.area))
+  }
+
+  get index(): string {
+    return join(this.dir, 'INDEX.md')
+  }
+
+  entryDir(task: string): string {
+    return join(this.dir, 'harnesses', slug(task))
+  }
+
+  /** The folder handed to an improver, or null when nothing has been learned here yet. */
+  contextDir(): string | null {
+    return existsSync(this.index) && this.entries().length ? this.dir : null
+  }
+
+  entries(): string[] {
+    const dir = join(this.dir, 'harnesses')
+    try {
+      return readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isDirectory() && existsSync(join(dir, entry.name, 'EXPERIENCE.md'))).map((entry) => entry.name).sort()
+    } catch {
+      return []
+    }
+  }
+
+  treeOf(task: string): Tree {
+    return readTree(this.surface, join(this.entryDir(task), 'harness'))
+  }
+
+  experienceText(task: string): string | null {
+    try {
+      return readFileSync(join(this.entryDir(task), 'EXPERIENCE.md'), 'utf8')
+    } catch {
+      return null
+    }
+  }
+
+  list(): LibraryEntry[] {
+    return this.entries().map((name) => {
+      const text = readFileSync(join(this.dir, 'harnesses', name, 'EXPERIENCE.md'), 'utf8')
+      return {
+        task: name,
+        label: /^- adopted: `([^`]+)`/m.exec(text)?.[1] ?? '',
+        runId: /\(run `([^`]+)`\)/.exec(text)?.[1] ?? '',
+        cue: field(text, 'cue'),
+        inventory: inventory(this.surface, this.treeOf(name))
+      }
+    })
+  }
+
+  /** Drops these tasks' entries and rebuilds the index; unknown tasks are ignored. */
+  forget(tasks: readonly string[]): void {
+    let removed = false
+    for (const task of tasks) {
+      const dir = this.entryDir(task)
+      if (existsSync(dir)) {
+        rmSync(dir, { recursive: true, force: true })
+        removed = true
+      }
+    }
+    if (removed) this.rebuildIndex()
+  }
+
+  /**
+   * Stores one adopted harness and why it was adopted, then rebuilds the index. Nothing is
+   * stored without a supported lesson. Returns the entry folder, or null.
+   */
+  add(task: string, tree: Tree, input: { why?: string; label?: string; runId?: string; experience?: Partial<Experience> | null }): string | null {
+    const experience = input.experience
+    if (!hasExperience(experience)) return null
+    assertTreeHasNoGraderVerdicts(tree)
+    assertTreeHasNoGraderVerdicts({ 'EXPERIENCE.md': JSON.stringify(experience) + (input.why ?? '') })
+    const dir = this.entryDir(task)
+    rmSync(join(dir, 'harness'), { recursive: true, force: true })
+    writeTree(this.surface, tree, join(dir, 'harness'), { stubs: false })
+    const lines = [`# ${task}`, '', `- adopted: \`${input.label ?? ''}\`${input.runId ? `  (run \`${input.runId}\`)` : ''}`, `- components: ${inventory(this.surface, tree)}`, '']
+    if (input.why?.trim()) lines.push('## Why this was chosen', input.why.trim(), '')
+    if (experience.edits?.trim()) lines.push('## What each edit did', experience.edits.trim(), '')
+    lines.push('## Transferable lesson')
+    for (const key of ['cue', 'lesson', 'evidence', 'confidence'] as const) if (experience[key]) lines.push(`- **${key}**: ${String(experience[key]).trim()}`)
+    lines.push('')
+    mkdirSync(dir, { recursive: true })
+    writeFileSync(join(dir, 'EXPERIENCE.md'), lines.join('\n'))
+    this.rebuildIndex()
+    return dir
+  }
+
+  rebuildIndex(): string {
+    const rows = [`# Library — ${this.area}`, '', 'Each entry is an earlier task in THIS area whose harness was chosen, stored with its FULL component tree and why it was picked. Browse this index, then open the entries that look relevant to the matter in front of you.', '']
+    const names = this.entries()
+    if (!names.length) rows.push('_(empty — this is the first task in this area)_')
+    for (const name of names) {
+      const cue = field(readFileSync(join(this.dir, 'harnesses', name, 'EXPERIENCE.md'), 'utf8'), 'cue')
+      rows.push(`## \`harnesses/${name}/\``, `- harness: \`harnesses/${name}/harness/\` — ${inventory(this.surface, this.treeOf(name))}`, `- why:     \`harnesses/${name}/EXPERIENCE.md\``)
+      if (cue) rows.push(`- cue:     ${cue}`)
+      rows.push('')
+    }
+    mkdirSync(this.dir, { recursive: true })
+    writeFileSync(this.index, rows.join('\n'))
+    return this.index
+  }
+}
+
+/** A one-line census of the components in a tree. */
+export function inventory(surface: SurfaceSpec, tree: Tree): string {
+  const scripts = Object.keys(tree).filter((rel) => patterns(surface).script.test(rel)).length
+  const counts: [string, number][] = [
+    ['skills', skillNames(surface, tree).length],
+    ['sub_agents', subagentNames(surface, tree).length],
+    ['scripts', scripts],
+    ['plugins', pluginNames(surface, tree).length]
+  ]
+  const bits = counts.filter(([, n]) => n).map(([name, n]) => `${n} ${name}`)
+  return `${bits.length ? bits.join(', ') : 'always-on edits only'}; ${alwaysOnText(surface, tree).length.toLocaleString('en-US')} always-on chars`
+}
+
+function field(text: string, name: string): string | null {
+  for (const line of text.split('\n')) if (line.startsWith(`- **${name}**:`)) return line.split(':').slice(1).join(':').trim()
+  return null
+}
+
+/** Every scope that has a library. */
+export function libraryScopes(root: string): string[] {
+  try {
+    return readdirSync(libraryRoot(root), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort()
+  } catch {
+    return []
+  }
+}
+
+/** Every area a scope has entries in. */
+export function libraryAreas(root: string, scope: string): string[] {
+  try {
+    return readdirSync(join(libraryRoot(root), safeId(scope)), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort()
+  } catch {
+    return []
+  }
+}

package/src/refine/pi.test.ts

@@ -0,0 +1,90 @@
+// Refinement through real Pi sessions with a scripted model: executions are probe runs in their
+// own folders, improvers edit the harness with Pi's own file tools, and every session is recorded.
+import { mkdtempSync, readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, expect, it } from 'vitest'
+
+import { fauxModel } from '../../tests/faux'
+import { memorySecretStore } from '../accounts/secrets'
+import { createHarness, type Harness } from '../harness'
+import { listRuns } from '../sessions/runStore'
+import * as store from '../tree/store'
+import { DEFAULT_REFINE_HOST } from './host'
+import { Library } from './library'
+import { IMPROVER_TRIGGER, PiExecutor, PiImprover, PROBE_TRIGGER } from './pi'
+import { refineDir, runRefinement } from './runner'
+
+const open: Harness[] = []
+afterEach(async () => {
+  for (const harness of open.splice(0)) await harness.close()
+})
+
+type Message = { role: string; content: unknown }
+const text = (message: Message | undefined): string => (!message ? '' : Array.isArray(message.content) ? message.content.map((part: { text?: string }) => part.text ?? '').join('') : String(message.content))
+
+it('runs probes and improvers as Pi sessions, harvests the edited tree, and records a lesson', async () => {
+  const root = mkdtempSync(join(tmpdir(), 'refine-pi-'))
+  const harness = createHarness({ dataRoot: root, stateDir: mkdtempSync(join(tmpdir(), 'refine-pi-state-')), secrets: memorySecretStore() })
+  open.push(harness)
+  harness.accounts.create('Main', 'apiKey')
+  const faux = await fauxModel(harness)
+  const base = { 'systemprompt.md': 'Answer with the total.\n', 'guardrails.md': '', 'memory.md': '', 'tool_guidance.md': '' }
+  store.seed(root, 'ledger', base)
+
+  const seen: string[] = []
+  // One responder for every request: what it answers depends on what it is asked.
+  const respond = (context: { messages: Message[] }) => {
+    const last = context.messages.at(-1)
+    const first = context.messages.filter((message) => message.role === 'user').map(text).join('\n')
+    if (last?.role === 'toolResult') return faux.says('<notes>Recorded that credits count: the base total dropped them.</notes>')
+    if (first.includes('YOUR DIMENSION') || first.includes('composing THE final harness')) {
+      seen.push('improver')
+      return faux.calls(['write', { path: 'context/harness/memory.md', content: 'Credits are negative amounts and count toward the total.\n' }])
+    }
+    if (first.includes('complete record of one improvement round')) {
+      seen.push('reflector')
+      return faux.says('<edits>\ncomponent: memory.md\ncandidates: inv1, dom1, agg1\nchange: noted that credits count\nobserved: the probe total matched the signed sum\nverdict: helped\n</edits>\n<experience>\ncue: When a ledger mixes credits and debits\nlesson: Record in memory.md that credits are negative amounts that count.\nevidence: the memory.md row; the probe total included credits\nconfidence: medium, one probe\n</experience>')
+    }
+    seen.push('probe')
+    return faux.says('Total: 85.15')
+  }
+  faux.script(Array.from({ length: 30 }, () => respond))
+
+  const id = 'pi-refine'
+  const dir = refineDir(root, id)
+  const runtime = { root: () => root, runs: harness.runs, run: (input: Parameters<Harness['sessions']['run']>[0]) => harness.sessions.run({ ...input, model: faux.model }) }
+  const summary = await runRefinement({
+    root,
+    id,
+    task: { id: 'ledger-1', scope: 'ledger', prompt: 'Sum every signed amount: 100.25, -20.10, 5.00.' },
+    k: 1,
+    mode: 'parallel',
+    executor: new PiExecutor(runtime, DEFAULT_REFINE_HOST, id, dir),
+    improver: new PiImprover(runtime, dir, { model: faux.model }),
+    proposeWidth: 1,
+    rolloutWidth: 1
+  })
+
+  expect(summary.status).toBe('ok')
+  expect(summary.recursion?.ledger).toMatchObject({ R: 4, P: 3 })
+  expect(summary.recursion?.fellBack).toEqual([])
+  expect(seen.filter((kind) => kind === 'improver')).toHaveLength(3)
+  expect(seen.filter((kind) => kind === 'probe')).toHaveLength(4)
+  // The chosen harness carries the improver's edit, saved as a version without moving the head.
+  const chosen = store.readVersion(root, 'ledger', summary.chosenFingerprint!)
+  expect(chosen['memory.md']).toBe('Credits are negative amounts and count toward the total.\n')
+  expect(store.head(root, 'ledger')!.fingerprint).toBe(summary.baseFingerprint)
+  expect(readFileSync(join(dir, 'rollouts', 'agg1', 'deliverables', 'reply.md'), 'utf8')).toBe('Total: 85.15')
+  expect(new Library(root, 'ledger').list().map((entry) => entry.cue)).toEqual(['When a ledger mixes credits and debits'])
+
+  // Every execution and improver session is an ordinary recorded run, marked by its trigger.
+  const runs = listRuns(root, {})
+  expect(runs.filter((run) => run.trigger === PROBE_TRIGGER)).toHaveLength(4)
+  expect(runs.filter((run) => run.trigger === PROBE_TRIGGER).every((run) => run.harnessScope === 'ledger' && run.status === 'completed')).toBe(true)
+  expect(runs.filter((run) => run.trigger === IMPROVER_TRIGGER)).toHaveLength(4)
+
+  // Adopting makes the chosen harness the head.
+  const adopted = harness.refine.adopt(id)
+  expect(adopted.fingerprint).toBe(summary.chosenFingerprint)
+})