@tduyng/pi-harness-loop 0.0.0-stage → 0.0.1

package/src/producer.ts

@@ -0,0 +1,236 @@
+import { execFileSync } from 'node:child_process'
+import { existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import { writeFileAtomic } from './atomic.js'
+import { harnessDir } from './store.js'
+
+export type ProducerState = 'live' | 'upstream' | 'removed' | 'unknown'
+
+export interface ProducerVerdict {
+    state: ProducerState
+    phrase: string
+    detail: string
+}
+
+/** Returns the command output, an empty string when it found nothing, and null when the tool is not there. */
+export type Runner = (file: string, args: string[], cwd: string) => string | null
+const SEARCH_SKIP = ['!node_modules', '!.git', '!.pi', '!dist', '!build', '!target']
+const PHRASE_SIZE = 40
+
+function isHole(token: string): boolean {
+    return /[\\/:]/.test(token) || /\d/.test(token) || token.startsWith('$')
+}
+
+function runsOf(line: string): string[] {
+    const runs: string[] = []
+    let run: string[] = []
+    for (const token of line.split(/\s+/)) {
+        if (token === '') continue
+        if (isHole(token)) {
+            if (run.length > 0) runs.push(run.join(' '))
+            run = []
+            continue
+        }
+        run.push(token)
+    }
+    if (run.length > 0) runs.push(run.join(' '))
+    return runs
+}
+
+/**
+ * A message is the part of it that the code writes as text. The ids, the paths and the
+ * numbers are the run's, so they are cut out, and the longest run left is what a search
+ * can find in a producer.
+ */
+export function phrasesOf(body: string, limit = 3): string[] {
+    const out: string[] = []
+    for (const line of body.split('\n')) {
+        for (const run of runsOf(line.trim())) {
+            const phrase = run
+                .replace(/^[.,;:]+|[.,;:]+$/g, '')
+                .slice(0, PHRASE_SIZE)
+                .trim()
+            if (phrase.split(' ').length < 4) continue
+            if (!out.includes(phrase)) out.push(phrase)
+        }
+    }
+    return out.toSorted((a, b) => b.length - a.length).slice(0, limit)
+}
+
+const runRg: Runner = (file, args, cwd) => {
+    try {
+        return execFileSync(file, args, {
+            cwd,
+            encoding: 'utf8',
+            stdio: ['ignore', 'pipe', 'ignore'],
+            timeout: 10_000
+        })
+    } catch (error) {
+        const code = (error as { code?: string }).code
+        return code === 'ENOENT' ? null : ''
+    }
+}
+
+function search(run: Runner, cwd: string, phrase: string, root: string, skip: boolean): boolean {
+    const globs = skip ? SEARCH_SKIP.flatMap((glob) => ['--glob', glob]) : []
+    const out = run('rg', ['-F', '-l', '-m1', '--no-messages', ...globs, '--', phrase, root], cwd)
+    return out !== null && out.trim() !== ''
+}
+
+function removedAt(run: Runner, cwd: string, phrase: string): { commit: string; at: number } | null {
+    const out = run('git', ['log', '-S', phrase, '--format=%h %ct', '-n', '1'], cwd)
+    if (out === null) return null
+    const [commit, at] = out.trim().split(' ')
+    if (commit === undefined || at === undefined) return null
+    const seconds = Number(at)
+    return Number.isNaN(seconds) ? null : { commit, at: seconds * 1000 }
+}
+
+/**
+ * `since` is the newest session that hit the failure. A commit that touched the
+ * message before that session did not remove it, so the message still comes from
+ * somewhere, and a commit after it is the producer going away.
+ */
+export function producerOf(
+    cwd: string,
+    roots: string[],
+    body: string,
+    run: Runner = runRg,
+    cache = true,
+    since = 0
+): ProducerVerdict {
+    const phrases = phrasesOf(body)
+    if (phrases.length === 0) return { state: 'unknown', phrase: '', detail: '' }
+    const key = `${phrases[0] ?? ''}@${since}`
+    const head = cache ? cleanHead(run, cwd) : undefined
+    if (head !== undefined) {
+        const hit = readCache(cwd, head).get(key)
+        if (hit !== undefined) return hit
+    }
+    const verdict = decide(cwd, roots, phrases, run, since)
+    if (head !== undefined && verdict.state !== 'unknown') writeCache(cwd, head, key, verdict)
+    return verdict
+}
+
+function decide(cwd: string, roots: string[], phrases: string[], run: Runner, since: number): ProducerVerdict {
+    for (const phrase of phrases) {
+        if (search(run, cwd, phrase, cwd, true)) {
+            return { state: 'live', phrase, detail: 'the string is in the tree' }
+        }
+    }
+    for (const phrase of phrases) {
+        if (roots.some((root) => search(run, cwd, phrase, root, false))) {
+            return { state: 'upstream', phrase, detail: 'the string is in a dependency' }
+        }
+    }
+    if (run('git', ['rev-parse', '--git-dir'], cwd) === null) {
+        return { state: 'unknown', phrase: phrases[0] ?? '', detail: 'no git and no match' }
+    }
+    for (const phrase of phrases) {
+        const gone = removedAt(run, cwd, phrase)
+        if (gone !== null && gone.at >= since) {
+            return { state: 'removed', phrase, detail: `removed by ${gone.commit}` }
+        }
+    }
+    return { state: 'unknown', phrase: phrases[0] ?? '', detail: 'no producer found' }
+}
+
+const CacheSchema = t.object({
+    head: t.optional(t.string()),
+    entries: t.record(t.object({
+            state: t.union([
+                t.literal('live'),
+                t.literal('upstream'),
+                t.literal('removed'),
+                t.literal('unknown')
+            ]),
+            phrase: t.string(),
+            detail: t.string()
+        })
+    )
+})
+const isCacheSchema = new Validator(CacheSchema).isValidObject
+
+interface ProducerCache {
+    head?: string
+    entries: Record<string, ProducerVerdict>
+}
+
+function cacheFile(cwd: string): string {
+    return join(harnessDir(cwd), 'producers.json')
+}
+
+function headOf(run: Runner, cwd: string): string | undefined {
+    const head = run('git', ['rev-parse', 'HEAD'], cwd)?.trim()
+    return head ? head : undefined
+}
+
+/**
+ * A verdict is only reusable when the tree is the commit it was made on. Search results depend
+ * on uncommitted edits, which a HEAD key cannot see, so a dirty tree is never read or written.
+ */
+function cleanHead(run: Runner, cwd: string): string | undefined {
+    const head = headOf(run, cwd)
+    if (head === undefined) return undefined
+    return run('git', ['status', '--porcelain'], cwd) === '' ? head : undefined
+}
+
+function readCache(cwd: string, head: string | undefined): Map<string, ProducerVerdict> {
+    const out = new Map<string, ProducerVerdict>()
+    const path = cacheFile(cwd)
+    if (!existsSync(path)) return out
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'))
+    } catch {
+        return out
+    }
+    if (!isCacheSchema(parsed)) return out
+    if (head === undefined || parsed.head !== head) return out
+    for (const [phrase, entry] of Object.entries(parsed.entries)) out.set(phrase, entry)
+    return out
+}
+
+export function readProducers(cwd: string): ProducerVerdict[] {
+    return [...readCache(cwd, headOf(runRg, cwd)).values()]
+}
+
+const confoundCache = new Map<string, string | null>()
+
+export function commitInWindow(cwd: string, key: string, since: number): string | null {
+    const phrase = phrasesOf(key)[0]
+    if (phrase === undefined) return null
+    const cacheKey = `${cwd}:${headOf(runRg, cwd) ?? ''}:${phrase}:${since}`
+    if (confoundCache.has(cacheKey)) return confoundCache.get(cacheKey) ?? null
+    const out = runRg('git', ['log', '-S', phrase, '--format=%h %ct', '-n', '20'], cwd)
+    let found: string | null = null
+    if (out !== null) {
+        for (const line of out.trim().split('\n')) {
+            const [commit, at] = line.trim().split(' ')
+            if (commit === undefined || at === undefined) continue
+            if (Number(at) * 1000 >= since) {
+                found = commit
+                break
+            }
+        }
+    }
+    confoundCache.set(cacheKey, found)
+    return found
+}
+
+function writeCache(cwd: string, head: string, key: string, verdict: ProducerVerdict): void {
+    const path = cacheFile(cwd)
+    const current = readCache(cwd, head)
+    current.set(key, verdict)
+    const cache: ProducerCache = {
+        head,
+        entries: Object.fromEntries(current)
+    }
+    mkdirSync(harnessDir(cwd), { recursive: true })
+    writeFileAtomic(path, `${JSON.stringify(cache)}\n`)
+}

package/src/prompt.ts

@@ -0,0 +1,162 @@
+import type { Recurrence } from './corpus.js'
+import { rate, type Effect } from './effect.js'
+import type { Target } from './target.js'
+
+export function buildProposalPrompt(
+    target: Target,
+    dir: string,
+    recurrences: Recurrence[] = [],
+    effect: Effect | null = null,
+    standing = 0,
+    form: 'rule' | 'skill' | 'profile' | null = null
+): string {
+    const evidence = target.evidence.map((line) => `- ${line}`).join('\n')
+    const across = recurrences
+        .filter((recurrence) => recurrence.id !== target.id)
+        .map((recurrence) => `- ${recurrence.title} (${recurrence.occurrences} times)`)
+    return [
+        '# WHAT YOU ARE EDITING',
+        '',
+        "pi's harness is the set of files that shape every run: the skills it advertises, the agent",
+        'profiles it can delegate to, the context files it reads at startup, and the extensions that',
+        `sit in the tool loop. You are writing one piece of it, and it goes in ${dir}.`,
+        '',
+        '# THE FINDING',
+        '',
+        target.title,
+        '',
+        `${target.count} occurrences in ${target.scope}. The evidence:`,
+        '',
+        evidence,
+        '',
+        ...(target.sample === '' ? [] : ['The failure itself:', '', '```', target.sample.slice(0, 600), '```', '']),
+        ...(target.tool === '' ? [] : [`The tool that produced it: ${target.tool}.`, '']),
+        ...(across.length === 0
+            ? []
+            : [
+                  '# THE SAME THING IN EARLIER SESSIONS',
+                  '',
+                  'These recur across the sessions of this project, so they are a property of the',
+                  'harness rather than of one run:',
+                  '',
+                  ...across,
+                  ''
+              ]),
+        ...(effect === null || (effect.verdict !== 'flat' && effect.verdict !== 'worse')
+            ? []
+            : [
+                  '# THE EARLIER ATTEMPT',
+                  '',
+                  `A part was installed for this finding already, and the sessions after it show`,
+                  `${effect.verdict === 'worse' ? 'more' : 'the same'} failure: ${rate(effect.before)} before it and`,
+                  `${rate(effect.after)} after it. The form of that part did not carry, so write a`,
+                  'different form, or say KEEP_BASE when the evidence does not argue for one.',
+                  ...(form === null
+                      ? []
+                      : [
+                            '',
+                            `That part was a ${form}.`,
+                            form === 'rule'
+                                ? 'A rule that did not carry is a situation the model cannot place, so a skill that routes on it is the stronger form.'
+                                : 'A skill that did not carry is a situation the model does not route to, so a rule that always applies is what is left.'
+                        ]),
+                  ''
+              ]),
+        '# WHAT TO WRITE',
+        '',
+        ...(target.kind === 'dead-skill'
+            ? [
+                  'pi advertises every skill by name and description in the system prompt, so a skill',
+                  'nobody opens is a tax on every request of every session. The forms:',
+                  '',
+                  `- rewrite ${dir}/<name>/SKILL.md with \`disable-model-invocation: true\` in the`,
+                  '  frontmatter. The skill stays reachable through /skill:name and stops being',
+                  '  advertised.',
+                  '- rewrite the description so it names the situation, when the description is what',
+                  '  keeps the model from choosing it. The description is the only part of a skill the',
+                  '  model sees before it decides.',
+                  '- move the skill to a project skills directory, so it is advertised only in the',
+                  '  projects that need it.',
+                  '- say KEEP_BASE, when the skill earns its place on the situations that rarely come',
+                  '  up and a router would miss it.',
+                  '',
+                  'A skill that was never opened in the sessions on disk might still be the right',
+                  'tool for a task that has not come up yet. Say so if you think so.'
+              ]
+            : [
+                  'One of:',
+                  '',
+                  `- a skill: ${dir}/<name>/SKILL.md, with frontmatter \`name:\` and \`description:\`, and any`,
+                  `  helper it runs under ${dir}/<name>/scripts/. The description is what the model routes on,`,
+                  '  so it says what the skill does and when it applies.',
+                  `- an agent profile: ${dir}/<name>.md, with frontmatter \`name:\` and \`description:\`.`,
+                  `- a rule: ${dir}/RULE.md, appended to the project context file.`,
+                  '',
+                  `Put a skill or a rule under ${dir}/global/ when it has to hold in every project.`,
+                  'A skill under global/ goes to the skills every project loads, and a global rule',
+                  'goes to the context file every project reads.',
+                  '',
+                  'Both forms cost standing text on every request. A skill costs its name and',
+                  'description, about 300 characters, and a rule costs its length. The difference is',
+                  'that a rule always applies while a skill only applies when the model routes to',
+                  'it, and a skill body costs nothing until it is opened. Say how many characters',
+                  'the part adds when it goes into an always-on file, because that number is a tax',
+                  'on every request from then on.',
+                  '',
+                  ...(standing === 0
+                      ? []
+                      : [
+                            `The parts installed for this project already cost ${standing} characters on`,
+                            'every request. The cheaper form that still holds is the right one.',
+                            ''
+                        ])
+              ]),
+        '',
+        '# THE CHECK',
+        '',
+        `Write ${dir}/CONFIRM.md with one shell command that exits 0 only when the failure above`,
+        'can no longer happen. It must not write files, install anything, or use the network. The loop',
+        'runs it after the project check passes, and a confirmation that fails puts the part back.',
+        'When you cannot name a command that proves the change, leave the file out and say why.',
+        '',
+        ...(target.command === undefined
+            ? []
+            : [
+                  'The command that produced the failure is replayed after the install, so a fix that',
+                  'does not change its result is visible in the record:',
+                  '',
+                  '```',
+                  target.command.slice(0, 300),
+                  '```',
+                  ''
+              ]),
+        '# THE OUTPUT',
+        '',
+        'The loop writes these files itself when the finding comes from the automatic run. A person or',
+        'a session may write them by hand instead, and nothing else is installed.',
+        '',
+        '```',
+        '<file path="RULE.md">',
+        '## Heading',
+        'the rule',
+        '</file>',
+        '```',
+        '',
+        `A path is relative to ${dir}: RULE.md, CONFIRM.md, <name>/SKILL.md, <name>.md for a`,
+        'profile, <name>/scripts/<file> for a helper, and the same under global/ for every project.',
+        '',
+        '# THE RULES',
+        '',
+        '1. Every change traces to a moment in the evidence above. Name the moment. A change you',
+        '   cannot trace is speculation, and speculation in a harness is paid for on every later run.',
+        '2. KEEP_BASE is a real answer. When the evidence does not argue for a change, write nothing',
+        '   and reply KEEP_BASE.',
+        '3. The harness changes how pi works, not what the work produces. Do not change the shape of',
+        '   the deliverable.',
+        '4. Every helper your component names ships with it, or the component is dead on arrival.',
+        '5. The part is read by every later run, so it never names the session files, the finding id, or',
+        '   the harness directory. It is a rule about the work, not a report of the evidence.',
+        '6. Reply with the file blocks and nothing outside them. A reply that names the moment in a',
+        '   comment instead of a file installs nothing.'
+    ].join('\n')
+}

package/src/replay.ts

@@ -0,0 +1,173 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import { runCheck } from './check.js'
+import type { Recurrence } from './corpus.js'
+import { harnessDir } from './store.js'
+
+export const REPLAY_TTL_MS = 24 * 60 * 60 * 1000
+export const REPLAY_TIMEOUT_MS = 60_000
+
+export interface Replay {
+    at: number
+    id: string
+    command: string
+    ok: boolean
+    exitCode: number | null
+    output: string
+}
+
+export type ReplayVerdict = 'stale' | 'live' | 'unknown'
+
+const ReplaySchema = t.object({
+    at: t.number(),
+    id: t.string(),
+    command: t.string(),
+    ok: t.boolean(),
+    exitCode: t.union([t.number(), t.null()]),
+    output: t.string()
+})
+const isReplaySchema = new Validator(ReplaySchema).isValidObject
+
+const READ_ONLY_BINS = new Set([
+    '[',
+    'awk',
+    'basename',
+    'cat',
+    'comm',
+    'cut',
+    'date',
+    'diff',
+    'dirname',
+    'echo',
+    'fd',
+    'fdfind',
+    'grep',
+    'head',
+    'id',
+    'jq',
+    'ls',
+    'printenv',
+    'printf',
+    'pwd',
+    'realpath',
+    'rg',
+    'sort',
+    'stat',
+    'tail',
+    'test',
+    'tr',
+    'type',
+    'uniq',
+    'wc',
+    'which'
+])
+
+const GIT_READ = new Set([
+    'blame',
+    'branch',
+    'describe',
+    'diff',
+    'grep',
+    'log',
+    'ls-files',
+    'remote',
+    'rev-parse',
+    'show',
+    'status'
+])
+
+const SHELL_SYNTAX = /[<>&;`\n]|\$\(/
+
+function segmentAllowed(segment: string): boolean {
+    const [bin, ...args] = segment.split(/\s+/)
+    if (bin === undefined) return false
+    if (args.some((arg) => arg.startsWith('--output'))) return false
+    if (bin === 'git') {
+        const sub = args[0] ?? ''
+        if (!GIT_READ.has(sub)) return false
+        if (sub === 'branch') {
+            return args.length === 1 || args.slice(1).every((arg) => /^(-a|-r|-v|--list|--all|--remotes)$/.test(arg))
+        }
+        if (sub === 'remote') return args.length === 1 || args[1] === '-v' || args[1] === 'show'
+        return true
+    }
+    if (bin === 'sed') return !args.some((arg) => arg === '-i' || arg.startsWith('--in-place'))
+    if (bin === 'npx') return args[0] === 'tsc' && args.includes('--noEmit')
+    if (bin === 'node') return args.length === 1 && args[0] === '--version'
+    if (bin === 'find') return !args.some((arg) => /^-(delete|exec|execdir|ok)$/.test(arg))
+    if (bin === 'awk') return !args.some((arg) => arg.includes('system('))
+    return READ_ONLY_BINS.has(bin)
+}
+
+export function canReplay(command: string): boolean {
+    const trimmed = command
+        .trim()
+        .replace(/\s*2>&1\s*/g, ' ')
+        .trim()
+    if (trimmed === '' || trimmed.length > 600 || SHELL_SYNTAX.test(trimmed)) return false
+    return trimmed.split('|').every((segment) => segmentAllowed(segment.trim()))
+}
+
+function replayFile(cwd: string): string {
+    return join(harnessDir(cwd), 'replay.jsonl')
+}
+
+export function readReplays(cwd: string): Replay[] {
+    const path = replayFile(cwd)
+    if (!existsSync(path)) return []
+    const out: Replay[] = []
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+        if (line.trim() === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(line)
+        } catch {
+            continue
+        }
+        if (isReplaySchema(parsed)) out.push(parsed)
+    }
+    return out
+}
+
+export function appendReplay(cwd: string, replay: Replay): void {
+    mkdirSync(harnessDir(cwd), { recursive: true })
+    appendFileSync(replayFile(cwd), `${JSON.stringify(replay)}\n`, 'utf8')
+}
+
+export function replayVerdict(cwd: string, recurrence: Recurrence): ReplayVerdict {
+    if (recurrence.command === undefined) return 'unknown'
+    const latest = readReplays(cwd)
+        .filter((replay) => replay.id === recurrence.id && replay.command === recurrence.command)
+        .toSorted((a, b) => b.at - a.at)[0]
+    if (latest === undefined || Date.now() - latest.at > REPLAY_TTL_MS) return 'unknown'
+    return latest.ok ? 'stale' : 'live'
+}
+
+export async function runReplay(cwd: string, id: string, command: string): Promise<Replay | null> {
+    if (!canReplay(command)) return null
+    const result = await runCheck(cwd, command, { timeoutMs: REPLAY_TIMEOUT_MS })
+    const replay: Replay = {
+        at: Date.now(),
+        id,
+        command,
+        ok: result.ok,
+        exitCode: result.exitCode,
+        output: result.output.slice(0, 2000)
+    }
+    appendReplay(cwd, replay)
+    return replay
+}
+
+export async function probeReplay(cwd: string, recurrence: Recurrence): Promise<ReplayVerdict> {
+    const cached = replayVerdict(cwd, recurrence)
+    if (cached !== 'unknown') return cached
+    if (recurrence.command === undefined) return 'unknown'
+    const replay = await runReplay(cwd, recurrence.id, recurrence.command)
+    if (replay === null) return 'unknown'
+    return replay.ok ? 'stale' : 'live'
+}