@tduyng/pi-harness-loop 0.0.0-stage → 0.0.2

package/src/auto.ts

@@ -0,0 +1,483 @@
+import { createHash } from 'node:crypto'
+import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
+import { join, relative } from 'node:path'
+
+import { type ExtensionContext } from '@earendil-works/pi-coding-agent'
+import { type Skill } from '@earendil-works/pi-coding-agent'
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import {
+    DEAD_SKILL_SESSIONS,
+    corpusCachePath,
+    scanCorpus,
+    sessionFiles,
+    signatureHistory,
+    callHistory,
+    type CorpusScan,
+    type PromptWords,
+    type Recurrence,
+    type SkillUse
+} from './corpus.js'
+import { writeFileAtomic } from './atomic.js'
+import { effectFor } from './effect.js'
+import { DAY_MS, readSpend } from './ledger.js'
+import { rareTriggersOf } from './match.js'
+import { producerOf } from './producer.js'
+import { replayVerdict } from './replay.js'
+import { buildProposalPrompt } from './prompt.js'
+import { harnessDir, readInstalls, readSettings } from './store.js'
+import { targetFromRecurrence, targetFromSkillUse, type Target } from './target.js'
+import { loadHarness } from './tree.js'
+import type { FindingKind, Install } from './types.js'
+
+const RecurrenceSchema = t.object({
+    id: t.string(),
+    key: t.string(),
+    kind: t.union([t.literal('error'), t.literal('command'), t.literal('read')]),
+    title: t.string(),
+    sessions: t.number(),
+    occurrences: t.number(),
+    tool: t.string(),
+    sample: t.string(),
+    command: t.optional(t.string()),
+    blocked: t.optional(t.string()),
+    evidence: t.array(t.string())
+})
+
+const SkillUseSchema = t.object({
+    id: t.string(),
+    name: t.string(),
+    sessions: t.number(),
+    opens: t.number()
+})
+
+const ScanSchema = t.object({
+    at: t.number(),
+    fingerprint: t.string(),
+    sessions: t.number(),
+    available: t.optional(t.number()),
+    recurrences: t.array(RecurrenceSchema),
+    skills: t.array(SkillUseSchema),
+    promptCount: t.number(),
+    promptWords: t.record(t.number())
+})
+const isScanSchema = new Validator(ScanSchema).isValidObject
+
+export interface CachedScan {
+    at: number
+    fingerprint: string
+    sessions: number
+    available?: number
+    recurrences: Recurrence[]
+    skills: SkillUse[]
+    promptCount: number
+    promptWords: Record<string, number>
+}
+
+export interface LiveScan {
+    at: number
+    fingerprint: string
+    sessions: number
+    available: number
+    recurrences: Recurrence[]
+    skills: SkillUse[]
+    prompts: PromptWords
+}
+
+export interface SkillFacts {
+    name: string
+    description: string
+    filePath: string
+    advertised: boolean
+}
+
+export function notify(ctx: ExtensionContext, message: string, level: 'info' | 'warning' | 'error' = 'info'): void {
+    try {
+        ctx.ui.notify(message, level)
+    } catch {}
+}
+
+export function scanPath(cwd: string): string {
+    return join(harnessDir(cwd), 'scan.json')
+}
+
+export function readScan(cwd: string): CachedScan | null {
+    const path = scanPath(cwd)
+    if (!existsSync(path)) return null
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'))
+    } catch {
+        return null
+    }
+    return isScanSchema(parsed) ? parsed : null
+}
+
+export function writeScan(cwd: string, scan: LiveScan): void {
+    mkdirSync(harnessDir(cwd), { recursive: true })
+    const cached: CachedScan = {
+        at: scan.at,
+        fingerprint: scan.fingerprint,
+        sessions: scan.sessions,
+        available: scan.available,
+        recurrences: scan.recurrences,
+        skills: scan.skills,
+        promptCount: scan.prompts.count,
+        promptWords: Object.fromEntries(scan.prompts.words)
+    }
+    writeFileAtomic(scanPath(cwd), `${JSON.stringify(cached)}\n`)
+}
+
+export function toLive(scan: CachedScan): LiveScan {
+    return {
+        at: scan.at,
+        fingerprint: scan.fingerprint,
+        sessions: scan.sessions,
+        available: scan.available ?? scan.sessions,
+        recurrences: scan.recurrences,
+        skills: scan.skills,
+        prompts: { count: scan.promptCount, words: new Map(Object.entries(scan.promptWords)) }
+    }
+}
+
+export function liveScan(scan: CorpusScan): LiveScan {
+    return {
+        at: Date.now(),
+        fingerprint: '',
+        sessions: scan.sessions,
+        available: scan.available,
+        recurrences: scan.recurrences,
+        skills: scan.skills,
+        prompts: scan.prompts
+    }
+}
+
+/** The fingerprint of the sessions that are finished, so the one being written does not force a rescan. */
+export function fingerprintOf(sessionDir: string, current: string): string {
+    const parts: string[] = []
+    for (const file of sessionFiles(sessionDir)) {
+        if (current !== '' && file.endsWith(`${current}.jsonl`)) continue
+        try {
+            const stat = statSync(file)
+            parts.push(`${file}:${stat.mtimeMs}:${stat.size}`)
+        } catch {}
+    }
+    return createHash('sha1').update(parts.join('\n')).digest('hex').slice(0, 16)
+}
+
+export function freshScan(
+    cwd: string,
+    agentDir: string,
+    sessionDir: string,
+    current: string,
+    skills: SkillFacts[] = skillFacts(cwd, agentDir)
+): LiveScan {
+    const scan = scanCorpus(
+        sessionDir,
+        skills.map((skill) => skill.name),
+        join(cwd, '.pi', 'harness', 'corpus-cache.json'),
+        current === '' ? undefined : `${current}.jsonl`
+    )
+    return {
+        at: Date.now(),
+        fingerprint: fingerprintOf(sessionDir, current),
+        sessions: scan.sessions,
+        available: scan.available,
+        recurrences: scan.recurrences,
+        skills: scan.skills,
+        prompts: scan.prompts
+    }
+}
+
+export function skillFacts(cwd: string, agentDir: string): SkillFacts[] {
+    return loadHarness(cwd, agentDir).skills.map((skill: Skill) => ({
+        name: skill.name,
+        description: skill.description,
+        filePath: skill.filePath,
+        advertised: !skill.disableModelInvocation
+    }))
+}
+
+export function attempted(cwd: string, id: string): boolean {
+    return existsSync(join(harnessDir(cwd), id))
+}
+
+const EDIT_MATCH = /oldtext|old text|edits\[|no changes made|replacement produced identical/i
+
+function familyOf(recurrence: Recurrence): string | null {
+    if (recurrence.kind !== 'error' || recurrence.tool !== 'edit') return null
+    return EDIT_MATCH.test(recurrence.title) || EDIT_MATCH.test(recurrence.sample) ? 'edit-match' : null
+}
+
+function bigger(left: Recurrence, right: Recurrence): boolean {
+    if (left.sessions !== right.sessions) return left.sessions > right.sessions
+    return left.occurrences > right.occurrences
+}
+
+/**
+ * A new signature is not a new failure when it is a smaller variant of one the loop already
+ * tried. The edit tool has one matching mechanic, so the 2-session "oldText must be unique"
+ * is the 9-session "oldText must match" with a different error string, and a second proposal
+ * for it is the same rule said twice.
+ */
+function coveredByLarger(cwd: string, recurrence: Recurrence, all: Recurrence[]): boolean {
+    const family = familyOf(recurrence)
+    if (family === null) return false
+    return all.some(
+        (other) =>
+            other.id !== recurrence.id &&
+            familyOf(other) === family &&
+            bigger(other, recurrence) &&
+            (attempted(cwd, other.id) || needsCode(cwd, other))
+    )
+}
+
+/**
+ * A failure that recurs across the sessions of the project is a property of the harness, and
+ * the loop acts on it on its own. A command or a file that shows up in every session is a
+ * habit, and a habit is a judgement call the operator makes.
+ */
+export function pendingTarget(cwd: string, scan: LiveScan, skills: SkillFacts[]): Target | null {
+    for (const recurrence of scan.recurrences) {
+        if (recurrence.kind !== 'error') continue
+        if (recurrence.blocked !== undefined) continue
+        if (!wantsAttempt(cwd, recurrence)) continue
+        if (replayVerdict(cwd, recurrence) === 'stale') continue
+        if (theCodeIsGone(cwd, recurrence)) continue
+        if (coveredByLarger(cwd, recurrence, scan.recurrences)) continue
+        return targetFromRecurrence(recurrence)
+    }
+    for (const use of scan.skills) {
+        if (use.opens > 0 || use.sessions < DEAD_SKILL_SESSIONS) continue
+        if (attempted(cwd, use.id)) continue
+        const live = skills.find((skill) => skill.name === use.name)
+        if (live === undefined || !live.advertised) continue
+        return targetFromSkillUse(use, live.description, live.filePath)
+    }
+    return null
+}
+
+export function budgetLeft(cwd: string, now = Date.now()): boolean {
+    const today = readProposals(cwd).filter((meta) => meta.at > now - DAY_MS).length
+    return today < readSettings(cwd).maxProposalsPerDay
+}
+
+const OFF = new Set(['0', 'false', 'off', 'no'])
+
+/** `PI_HARNESS_LOOP=off` stops the automatic run and leaves the command working. */
+export function loopEnabled(): boolean {
+    const value = process.env['PI_HARNESS_LOOP']
+    return value === undefined || !OFF.has(value.trim().toLowerCase())
+}
+
+const KindSchema = t.union([
+    t.literal('repeat-error'),
+    t.literal('repeat-command'),
+    t.literal('repeat-read'),
+    t.literal('no-change'),
+    t.literal('abandoned-tool'),
+    t.literal('dead-skill')
+])
+
+const ProposalMetaSchema = t.object({
+    job: t.optional(t.string()),
+    target: t.string(),
+    kind: KindSchema,
+    triggers: t.array(t.string()),
+    at: t.integer(),
+    count: t.optional(t.integer()),
+    dir: t.optional(t.string())
+})
+const isProposalMetaSchema = new Validator(ProposalMetaSchema).isValidObject
+
+interface ProposalMeta {
+    job?: string
+    target: string
+    kind: FindingKind
+    triggers: string[]
+    at: number
+    count?: number
+    dir?: string
+}
+
+function proposalsFile(cwd: string): string {
+    return join(harnessDir(cwd), 'proposals.jsonl')
+}
+
+function readProposals(cwd: string): ProposalMeta[] {
+    const path = proposalsFile(cwd)
+    if (!existsSync(path)) return []
+    const out: ProposalMeta[] = []
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+        if (line.trim() === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(line)
+        } catch {
+            continue
+        }
+        if (isProposalMetaSchema(parsed)) out.push(parsed)
+    }
+    return out
+}
+
+function appendProposal(cwd: string, meta: ProposalMeta): void {
+    mkdirSync(harnessDir(cwd), { recursive: true })
+    appendFileSync(proposalsFile(cwd), `${JSON.stringify(meta)}\n`, 'utf8')
+}
+
+type AttemptState = 'new' | 'retry' | 'quiet' | 'needs-code'
+
+/** After this many completions on one finding, another one is a budget with no ceiling. */
+export const MAX_ATTEMPTS = 3
+
+function proseOnly(cwd: string, finding: string): boolean {
+    const install = readInstalls(cwd)
+        .filter((entry) => entry.finding === finding && entry.ok && !entry.reverted)
+        .toSorted((a, b) => b.at - a.at)[0]
+    if (install === undefined) return false
+    return install.files.length > 0 && install.files.every((file) => file.to.endsWith('.md'))
+}
+
+function proseReverted(cwd: string, finding: string): boolean {
+    const install = readInstalls(cwd)
+        .filter((entry) => entry.finding === finding && entry.reverted)
+        .toSorted((a, b) => b.at - a.at)[0]
+    if (install === undefined) return false
+    return install.files.length > 0 && install.files.every((file) => file.to.endsWith('.md'))
+}
+
+/**
+ * A finding the loop already tried comes back when the sessions after the install show the
+ * same rate of failure, or a worse one, and it stays quiet when the rate fell. A finding that
+ * never landed comes back as soon as it grows, up to the attempt ceiling. A prose part that
+ * did not carry is not tried as prose again, because the failure lives in tool behavior and a
+ * rule cannot reach it.
+ */
+function attemptState(cwd: string, recurrence: Recurrence): AttemptState {
+    const attempts = readProposals(cwd).filter((meta) => meta.target === recurrence.id)
+    const last = attempts[attempts.length - 1]
+    if (last === undefined) return attempted(cwd, recurrence.id) ? 'quiet' : 'new'
+    const spent = (): AttemptState => (proseOnly(cwd, recurrence.id) ? 'needs-code' : 'quiet')
+    const capped = attempts.length >= MAX_ATTEMPTS
+    const spends = readSpend(cwd).filter((spend) => spend.target === recurrence.id)
+    const outcome = spends[spends.length - 1]?.outcome
+    if (outcome === 'auto-reverted') {
+        if (proseReverted(cwd, recurrence.id)) return 'needs-code'
+        return capped ? spent() : 'retry'
+    }
+    if (outcome === 'installed') {
+        const effect = effectFor(
+            readInstalls(cwd),
+            signatureHistory(corpusCachePath(cwd)),
+            recurrence.id,
+            recurrence.key,
+            undefined,
+            callHistory(corpusCachePath(cwd))
+        )
+        if (effect === null) return 'quiet'
+        if (effect.verdict === 'flat' || effect.verdict === 'worse' || effect.verdict === 'avoided') {
+            return capped ? spent() : 'retry'
+        }
+        return 'quiet'
+    }
+    if (last.count === undefined) return 'quiet'
+    if (recurrence.occurrences <= last.count) return 'quiet'
+    return capped ? spent() : 'retry'
+}
+
+function wantsAttempt(cwd: string, recurrence: Recurrence): boolean {
+    const state = attemptState(cwd, recurrence)
+    return state === 'new' || state === 'retry'
+}
+
+export function needsCode(cwd: string, recurrence: Recurrence): boolean {
+    return attemptState(cwd, recurrence) === 'needs-code'
+}
+
+/**
+ * A failure whose producer was deleted upstream cannot be prevented by a part, and writing one
+ * for it costs a turn and leaves a rule nobody can act on. The verdict is cached, because it
+ * is a search of the tree.
+ */
+function theCodeIsGone(cwd: string, recurrence: Recurrence): boolean {
+    if (recurrence.sample === '') return false
+    const uses = signatureHistory(corpusCachePath(cwd)).get(recurrence.key) ?? []
+    const since = uses[uses.length - 1]?.at ?? 0
+    const verdict = producerOf(cwd, [join(cwd, 'node_modules')], recurrence.sample, undefined, true, since)
+    return verdict.state === 'removed'
+}
+
+/**
+ * The form the loop already tried, so the next brief can ask for the other one instead of a
+ * different wording of the same thing. A rule that did not carry is a routing problem, and a
+ * skill that did not carry is an always-on problem.
+ */
+export function formOf(install: Install): 'rule' | 'skill' | 'profile' | null {
+    for (const file of install.files) {
+        if (file.mode === 'append') return 'rule'
+        if (file.to.endsWith('SKILL.md')) return 'skill'
+        if (file.to.endsWith('.md')) return 'profile'
+    }
+    return null
+}
+
+function previousForm(cwd: string, finding: string): 'rule' | 'skill' | 'profile' | null {
+    const install = readInstalls(cwd)
+        .filter((entry) => entry.finding === finding && entry.ok && !entry.reverted)
+        .toSorted((a, b) => b.at - a.at)[0]
+    return install === undefined ? null : formOf(install)
+}
+
+function attemptDir(cwd: string, id: string): string {
+    const attempts = readProposals(cwd).filter((meta) => meta.target === id).length
+    return attempts === 0 ? join(harnessDir(cwd), id) : join(harnessDir(cwd), `${id}-${attempts + 1}`)
+}
+
+/**
+ * A proposal is a file, not a process. The brief is what the live session reads when it
+ * writes the part, and recording it marks the finding as tried, so the loop stays quiet
+ * until the failure doubles.
+ */
+export function proposeToDisk(cwd: string, target: Target, scan: LiveScan): string {
+    const dir = attemptDir(cwd, target.id)
+    mkdirSync(dir, { recursive: true })
+    const key = scan.recurrences.find((recurrence) => recurrence.id === target.id)?.key
+    const standing = readInstalls(cwd)
+        .filter((install) => install.ok && !install.reverted)
+        .reduce((total, install) => total + (install.cost ?? 0), 0)
+    const effect =
+        key === undefined
+            ? null
+            : effectFor(
+                  readInstalls(cwd),
+                  signatureHistory(corpusCachePath(cwd)),
+                  target.id,
+                  key,
+                  undefined,
+                  callHistory(corpusCachePath(cwd))
+              )
+    writeFileSync(
+        join(dir, 'BRIEF.md'),
+        buildProposalPrompt(
+            target,
+            relative(cwd, dir),
+            scan.recurrences,
+            effect,
+            standing,
+            previousForm(cwd, target.id)
+        ),
+        'utf8'
+    )
+    appendProposal(cwd, {
+        target: target.id,
+        kind: target.kind,
+        triggers: rareTriggersOf(`${target.title} ${target.cue}`, scan.prompts),
+        at: Date.now(),
+        count: target.count,
+        dir: relative(cwd, dir)
+    })
+    return dir
+}

package/src/check.ts

@@ -0,0 +1,123 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { createLocalBashOperations, truncateTail } from '@earendil-works/pi-coding-agent'
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+export interface CheckResult {
+    command: string
+    ok: boolean
+    exitCode: number | null
+    failures: number | null
+    output: string
+    durationMs: number
+}
+
+const PackageJson = t.object({
+    scripts: t.optional(t.record(t.string()))
+})
+const isPackageJson = new Validator(PackageJson).isValidObject
+
+const TEST_BY_LOCKFILE = [
+    ['bun.lock', 'bun run test'],
+    ['bun.lockb', 'bun run test'],
+    ['pnpm-lock.yaml', 'pnpm test'],
+    ['yarn.lock', 'yarn test']
+] as const satisfies readonly (readonly [string, string])[]
+
+function packageTest(cwd: string): string {
+    const path = join(cwd, 'package.json')
+    if (!existsSync(path)) return ''
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'))
+    } catch {
+        return ''
+    }
+    if (!isPackageJson(parsed)) return ''
+    if (parsed.scripts?.['test'] === undefined) return ''
+    const lockfile = TEST_BY_LOCKFILE.find(([file]) => existsSync(join(cwd, file)))
+    return lockfile?.[1] ?? 'npm test'
+}
+
+function justTest(cwd: string): string {
+    const path = join(cwd, 'justfile')
+    if (!existsSync(path)) return ''
+    return /^test:/m.test(readFileSync(path, 'utf8')) ? 'just test' : ''
+}
+
+export function discoverCheck(cwd: string, configured: string): string {
+    if (configured.trim() !== '') return configured.trim()
+    return packageTest(cwd) || justTest(cwd)
+}
+
+export function failedCount(output: string): number | null {
+    const summary = /Tests\s+(\d+)\s+failed/.exec(output)
+    if (summary?.[1] !== undefined) return Number(summary[1])
+    const generic = /(\d+)\s+(?:tests?\s+)?failed/.exec(output)
+    return generic?.[1] === undefined ? null : Number(generic[1])
+}
+
+export function failureNames(output: string): string[] | null {
+    const names = [...output.matchAll(/^\s*(?:\(fail\)|✕|×|✗)\s+(.+)$/gm)]
+        .map((match) => (match[1] ?? '').trim())
+        .filter((name) => name !== '')
+    if (names.length > 0) return names.toSorted()
+    return failedCount(output) === 0 ? [] : null
+}
+
+const MAX_CHECK_OUTPUT_BYTES = 1500
+const DEFAULT_CHECK_TIMEOUT_MS = 300_000
+
+async function runSharedCheck(
+    cwd: string,
+    command: string,
+    options: { timeoutMs?: number } = {}
+): Promise<{ ok: boolean; exitCode: number | null; durationMs: number; output: string }> {
+    const timeoutMs = options.timeoutMs && options.timeoutMs > 0 ? options.timeoutMs : DEFAULT_CHECK_TIMEOUT_MS
+    let captured = Buffer.alloc(0)
+    const started = Date.now()
+    const finish = (exitCode: number | null, error?: string) => {
+        const durationMs = Date.now() - started
+        const output = captured.toString('utf8')
+        const body = error ? `${output}${output ? '\n' : ''}${error}` : output
+        const tail = truncateTail(body, { maxBytes: MAX_CHECK_OUTPUT_BYTES })
+        return {
+            ok: exitCode === 0,
+            exitCode,
+            durationMs,
+            output: tail.content.trim()
+        }
+    }
+    let exitCode: number | null
+    try {
+        ;({ exitCode } = await createLocalBashOperations().exec(command, cwd, {
+            onData: (data) => {
+                const combined = Buffer.concat([captured, data])
+                captured = combined.subarray(Math.max(0, combined.length - MAX_CHECK_OUTPUT_BYTES))
+            },
+            timeout: timeoutMs / 1000
+        }))
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error)
+        return finish(null, message)
+    }
+    return finish(exitCode)
+}
+
+export async function runCheck(
+    cwd: string,
+    command: string,
+    options: { timeoutMs?: number } = {}
+): Promise<CheckResult> {
+    const result = await runSharedCheck(cwd, command, options)
+    return {
+        command,
+        ok: result.ok,
+        exitCode: result.exitCode,
+        failures: failedCount(result.output),
+        output: result.output,
+        durationMs: result.durationMs
+    }
+}

package/src/coordination.ts

@@ -0,0 +1,30 @@
+import type { SessionEntry } from '@earendil-works/pi-coding-agent'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+const GOAL_TYPE = 'goal'
+
+const GoalData = t.object({
+    goal: t.optional(t.union([t.object({ status: t.optional(t.string()) }), t.null()]))
+})
+const isGoalData = new Validator(GoalData).isValidObject
+
+/** The status of the goal in this session, or null when there is no goal. */
+export function goalStatus(entries: SessionEntry[]): string | null {
+    let status: string | null = null
+    for (const entry of entries) {
+        if (entry.type !== 'custom' || entry.customType !== GOAL_TYPE) continue
+        if (!isGoalData(entry.data)) continue
+        const goal = entry.data.goal
+        status = goal === undefined || goal === null ? null : (goal.status ?? null)
+    }
+    return status
+}
+
+/** The goal drives the session with its own turns, so the loop stays out of its way. */
+export function goalInTheWay(entries: SessionEntry[]): boolean {
+    const status = goalStatus(entries)
+    return status === 'active'
+}