@tduyng/pi-harness-loop 0.0.0-stage → 0.0.1

package/src/effect.ts

@@ -0,0 +1,134 @@
+import type { SignatureUse } from './corpus.js'
+import type { Install } from './types.js'
+
+export interface Window {
+    sessions: number
+    occurrences: number
+    perSession: number
+    /** Median calls of every tool per session. Absent when the cache never counted tool calls. */
+    callsPerSession?: number
+}
+
+export type Verdict = 'held' | 'flat' | 'worse' | 'early' | 'out-of-window' | 'confounded' | 'avoided'
+
+export interface Effect {
+    install: string
+    finding: string
+    key: string
+    before: Window
+    after: Window
+    verdict: Verdict
+    commit?: string
+}
+
+export type Confound = (key: string, since: number) => string | null
+
+export const HELD_RATIO = 0.75
+export const WORSE_RATIO = 1.25
+export const MIN_SESSIONS_AFTER = 3
+/** The session has to do at least this much of its old work for the failure's fall to be the part's. */
+export const USED_RATIO = 0.5
+
+/** A session can do no work at all, so one session does not move the middle of the window. */
+function median(counts: number[]): number {
+    const sorted = counts.toSorted((a, b) => a - b)
+    const middle = Math.floor(sorted.length / 2)
+    if (sorted.length % 2 === 1) return sorted[middle] ?? 0
+    return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2
+}
+
+function windowOf(uses: SignatureUse[], at: number, side: 'before' | 'after', calls?: SignatureUse[]): Window {
+    const picked = uses.filter((use) => (side === 'before' ? use.at < at : use.at >= at))
+    const occurrences = picked.reduce((total, use) => total + use.count, 0)
+    const callPicked = calls?.filter((use) => (side === 'before' ? use.at < at : use.at >= at))
+    const window: Window = {
+        sessions: picked.length,
+        occurrences,
+        perSession: picked.length === 0 ? 0 : occurrences / picked.length
+    }
+    if (callPicked !== undefined && callPicked.length > 0) {
+        window.callsPerSession = median(callPicked.map((use) => use.count))
+    }
+    return window
+}
+
+/**
+ * A failure that stopped because the agent stopped working was not prevented. The rate of the
+ * failure fell, and the work the session did fell with it, so the part is not what carried.
+ */
+function avoided(before: Window, after: Window): boolean {
+    const was = before.callsPerSession
+    const now = after.callsPerSession
+    if (was === undefined || now === undefined || was === 0) return false
+    return now < was * USED_RATIO
+}
+
+/**
+ * A part is a claim about the future. The sessions after it are the only evidence
+ * the loop has, so the claim is read against them: fewer failures a session is a
+ * part that held, and the same rate or worse is a part that did not.
+ */
+export function verdictOf(before: Window, after: Window): Verdict {
+    if (before.sessions === 0) return 'out-of-window'
+    if (after.sessions < MIN_SESSIONS_AFTER) return 'early'
+    if (before.perSession === 0) return after.occurrences === 0 ? 'held' : 'worse'
+    const ratio = after.perSession / before.perSession
+    if (ratio > HELD_RATIO) return ratio >= WORSE_RATIO ? 'worse' : 'flat'
+    return avoided(before, after) ? 'avoided' : 'held'
+}
+
+export function effectFor(
+    installs: Install[],
+    history: Map<string, SignatureUse[]>,
+    finding: string,
+    key: string,
+    confound?: Confound,
+    calls?: SignatureUse[]
+): Effect | null {
+    const install = installs
+        .filter((entry) => entry.ok && !entry.reverted && entry.finding === finding)
+        .toSorted((a, b) => b.at - a.at)[0]
+    if (install === undefined) return null
+    const uses = history.get(install.key ?? key)
+    if (uses === undefined) return null
+    const before = windowOf(uses, install.at, 'before', calls)
+    const after = windowOf(uses, install.at, 'after', calls)
+    const verdict = verdictOf(before, after)
+    const commit =
+        verdict === 'early' || verdict === 'out-of-window' ? null : confound?.(install.key ?? key, install.at)
+    return {
+        install: install.id,
+        finding,
+        key: install.key ?? key,
+        before,
+        after,
+        verdict: commit === null || commit === undefined ? verdict : 'confounded',
+        commit: commit ?? undefined
+    }
+}
+
+/**
+ * One effect per finding, from the newest install. `keys` carries the signature the
+ * finding is about, for the installs written before the key was recorded.
+ */
+export function effectsOf(
+    installs: Install[],
+    history: Map<string, SignatureUse[]>,
+    keys: Map<string, string>,
+    confound?: Confound,
+    calls?: SignatureUse[]
+): Effect[] {
+    const findings = new Set(installs.filter((entry) => entry.ok && !entry.reverted).map((entry) => entry.finding))
+    const out: Effect[] = []
+    for (const finding of findings) {
+        const key = keys.get(finding)
+        if (key === undefined) continue
+        const effect = effectFor(installs, history, finding, key, confound, calls)
+        if (effect !== null) out.push(effect)
+    }
+    return out.toSorted((a, b) => a.verdict.localeCompare(b.verdict) || a.finding.localeCompare(b.finding))
+}
+
+export function rate(window: Window): string {
+    return `${window.perSession.toFixed(1)}/session`
+}

package/src/evaluate.ts

@@ -0,0 +1,81 @@
+import { corpusCachePath, signatureHistory, callHistory } from './corpus.js'
+import { effectsOf, rate, type Effect } from './effect.js'
+import { commitInWindow } from './producer.js'
+import { readScan } from './auto.js'
+import { revertApplied } from './apply.js'
+import { installRoots } from './install.js'
+import { appendInstall, failLesson, readInstalls, readLessons } from './store.js'
+import { appendSpend } from './ledger.js'
+import type { InstallContext } from './apply.js'
+import type { Install } from './types.js'
+
+export function failLessonsFor(cwd: string, effect: Effect, installs: Install[]): void {
+    const linked = installs
+        .filter((install) => install.finding === effect.finding)
+        .map((install) => install.lesson)
+        .filter((id): id is string => id !== undefined)
+    const prefixed = readLessons(cwd)
+        .filter((lesson) => lesson.id.startsWith(effect.finding))
+        .map((lesson) => lesson.id)
+    for (const id of new Set([...linked, ...prefixed])) failLesson(cwd, id)
+}
+
+export function evaluateInstalls(cwd: string): Effect[] {
+    const scan = readScan(cwd)
+    const keys = new Map(scan?.recurrences.map((recurrence) => [recurrence.id, recurrence.key]) ?? [])
+    const installs = readInstalls(cwd)
+    const effects = effectsOf(
+        installs,
+        signatureHistory(corpusCachePath(cwd)),
+        keys,
+        (key, since) => commitInWindow(cwd, key, since),
+        callHistory(corpusCachePath(cwd))
+    )
+    for (const effect of effects) {
+        if (effect.verdict === 'flat' || effect.verdict === 'worse' || effect.verdict === 'avoided') {
+            failLessonsFor(cwd, effect, installs)
+        }
+    }
+    return effects
+}
+
+/**
+ * A part that did not lower the rate it was written for is taken back out. The install is
+ * always reversible because it records the file it replaced, so a rule nobody can show a use
+ * for does not stay in the always-on context.
+ */
+export function autoRevert(cwd: string, context: InstallContext, effects: Effect[]): string[] {
+    const installs = readInstalls(cwd)
+    const lines: string[] = []
+    for (const effect of effects) {
+        if (effect.verdict !== 'flat' && effect.verdict !== 'worse' && effect.verdict !== 'avoided') continue
+        const install = installs.find((entry) => entry.id === effect.install)
+        if (install === undefined || install.reverted) continue
+        const result = revertApplied(install.files, installRoots(context))
+        const kept = result.refused.length > 0
+        appendInstall(cwd, {
+            ...install,
+            at: Date.now(),
+            ok: false,
+            reverted: true,
+            note: kept
+                ? `the file changed since the install: ${result.refused[0]}`
+                : `auto-reverted, the part was ${effect.verdict} at ${rate(effect.after)}`
+        })
+        appendSpend(cwd, {
+            at: Date.now(),
+            target: effect.finding,
+            turns: 0,
+            input: 0,
+            output: 0,
+            cost: 0,
+            outcome: 'auto-reverted'
+        })
+        lines.push(
+            kept
+                ? `harness: kept ${effect.finding}, the file changed since the install`
+                : `harness: reverted ${effect.finding}, the part was ${effect.verdict} at ${rate(effect.after)}`
+        )
+    }
+    return lines
+}

package/src/findings.ts

@@ -0,0 +1,141 @@
+import type { Transcript } from './transcript.js'
+import { idFor } from './corpus.js'
+
+export type FindingKindName = 'error' | 'command' | 'read' | 'no-change' | 'unanswered'
+
+export interface Finding {
+    id: string
+    kind: FindingKindName
+    title: string
+    count: number
+    evidence: string[]
+}
+
+const REPEAT_THRESHOLD = 3
+const MIN_SIGNATURE_TOKENS = 1
+const GENERIC_SIGNATURE =
+    /^(script failed|command failed|error|failed|ok|done|total n|command aborted|command timed out after n seconds|traceback \(most recent call last\))[:!.]?$/i
+const TRACEBACK = /^Traceback \(most recent call last\)/
+const NOT_AN_ERROR = /not an error/i
+const EXIT_LINE = /\n*command exited with code \d+\s*$/i
+
+export function usableSignature(value: string): boolean {
+    if (GENERIC_SIGNATURE.test(value.trim())) return false
+    const tokens = value
+        .split(/\s+/)
+        .filter(
+            (token) => token !== '' && token !== '<path>' && !/^['"]?<path>['"]?$/.test(token) && !/^N+$/.test(token)
+        )
+    if (tokens.length < MIN_SIGNATURE_TOKENS) return false
+    return tokens.some((token) => /[a-zA-Z]{4,}/.test(token))
+}
+
+export function signature(body: string): string {
+    const lines = body.split('\n').filter((line) => line.trim() !== '')
+    const head = lines[0] ?? ''
+    const pick = TRACEBACK.test(head) ? (lines[lines.length - 1] ?? head) : head
+    return pick
+        .replace(/[^\s'"]*\/[^\s'"]*/g, '<path>')
+        .replace(/\d+/g, 'N')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, 90)
+}
+
+export function failureSignature(body: string): string {
+    if (NOT_AN_ERROR.test(body)) return ''
+    const output = body.replace(EXIT_LINE, '').trim()
+    if (output === '' || output === '(no output)') return ''
+    const key = signature(output)
+    return usableSignature(key) ? key : ''
+}
+
+function topRepeat(values: string[]): { key: string; count: number } | null {
+    const counts = new Map<string, number>()
+    for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
+    let best: { key: string; count: number } | null = null
+    for (const [key, count] of counts) {
+        if (!best || count > best.count) best = { key, count }
+    }
+    return best
+}
+
+export function findFindings(transcript: Transcript): Finding[] {
+    const findings: Finding[] = []
+
+    const errors = transcript.results.filter((result) => result.isError)
+    const repeatedError = topRepeat(errors.map((result) => failureSignature(result.body)).filter((body) => body !== ''))
+    if (repeatedError && repeatedError.count >= REPEAT_THRESHOLD) {
+        findings.push({
+            id: idFor(`error:${repeatedError.key}`),
+            kind: 'error',
+            title: `the same failure came back ${repeatedError.count} times: ${repeatedError.key}`,
+            count: repeatedError.count,
+            evidence: errors
+                .filter((result) => failureSignature(result.body) === repeatedError.key)
+                .slice(0, 4)
+                .map((result) => `${result.toolName}: ${result.body.split('\n')[0]?.slice(0, 120) ?? ''}`)
+        })
+    }
+
+    const repeatedCommand = topRepeat(
+        transcript.calls
+            .filter((call) => call.name === 'bash')
+            .map((call) => call.command)
+            .filter((command) => command !== '')
+    )
+    if (repeatedCommand && repeatedCommand.count >= REPEAT_THRESHOLD) {
+        findings.push({
+            id: idFor(`command:${repeatedCommand.key}`),
+            kind: 'command',
+            title: `the same command ran ${repeatedCommand.count} times: ${repeatedCommand.key.slice(0, 120)}`,
+            count: repeatedCommand.count,
+            evidence: repeatedCommand.key
+                .split('\n')
+                .slice(0, 3)
+                .map((line) => `bash: ${line}`)
+        })
+    }
+
+    const repeatedRead = topRepeat(
+        transcript.calls
+            .filter((call) => call.name === 'read')
+            .map((call) => call.path)
+            .filter((path) => path !== '')
+    )
+    if (repeatedRead && repeatedRead.count >= REPEAT_THRESHOLD) {
+        findings.push({
+            id: idFor(`read:${repeatedRead.key}`),
+            kind: 'read',
+            title: `the same file was read ${repeatedRead.count} times: ${repeatedRead.key}`,
+            count: repeatedRead.count,
+            evidence: [`read: ${repeatedRead.key}`]
+        })
+    }
+
+    const changed = transcript.calls.some((call) => call.name === 'edit' || call.name === 'write')
+    const failed = transcript.results.some((result) => result.isError)
+    if (!changed && failed && transcript.calls.length >= 8) {
+        findings.push({
+            id: idFor('no-change'),
+            kind: 'no-change',
+            title: `the run made ${transcript.calls.length} tool calls, hit a failure and changed nothing`,
+            count: transcript.calls.length,
+            evidence: transcript.calls.slice(-4).map((call) => `${call.name} ${call.command || call.path}`)
+        })
+    }
+
+    const answered = new Set(transcript.results.map((result) => result.toolCallId))
+    const unanswered = transcript.calls.filter((call) => !answered.has(call.id))
+    if (unanswered.length > 0) {
+        findings.push({
+            id: idFor('unanswered'),
+            kind: 'unanswered',
+            title: `${unanswered.length} tool calls have no result in the transcript`,
+            count: unanswered.length,
+            evidence: unanswered.slice(0, 3).map((call) => `${call.name} ${call.command || call.path}`)
+        })
+    }
+
+    return findings
+}