@tduyng/pi-harness-loop 0.0.0-stage → 0.0.2

package/src/match.ts

@@ -0,0 +1,192 @@
+import type { Lesson } from './types.js'
+import type { PromptWords } from './corpus.js'
+
+/** A trigger that shows up in more than this share of the prompts of the project says nothing. */
+const COMMON_RARITY = 0.02
+
+function commonIn(word: string, rarity?: PromptWords): boolean {
+    if (rarity === undefined || rarity.count === 0) return false
+    return (rarity.words.get(word) ?? 0) / rarity.count > COMMON_RARITY
+}
+
+const STOP = new Set([
+    'about',
+    'above',
+    'after',
+    'again',
+    'against',
+    'all',
+    'also',
+    'and',
+    'another',
+    'any',
+    'anyway',
+    'are',
+    'around',
+    'ask',
+    'asks',
+    'away',
+    'back',
+    'because',
+    'been',
+    'before',
+    'being',
+    'below',
+    'both',
+    'but',
+    'can',
+    'calls',
+    'could',
+    'did',
+    'does',
+    'doing',
+    'down',
+    'during',
+    'each',
+    'every',
+    'from',
+    'further',
+    'get',
+    'gets',
+    'had',
+    'has',
+    'have',
+    'here',
+    'how',
+    'into',
+    'its',
+    'just',
+    'least',
+    'less',
+    'like',
+    'made',
+    'make',
+    'makes',
+    'many',
+    'more',
+    'most',
+    'much',
+    'must',
+    'not',
+    'now',
+    'only',
+    'onto',
+    'other',
+    'our',
+    'out',
+    'over',
+    'own',
+    'per',
+    'put',
+    'same',
+    'see',
+    'set',
+    'she',
+    'should',
+    'some',
+    'such',
+    'than',
+    'that',
+    'the',
+    'their',
+    'them',
+    'then',
+    'there',
+    'these',
+    'they',
+    'this',
+    'those',
+    'through',
+    'too',
+    'two',
+    'under',
+    'until',
+    'used',
+    'uses',
+    'using',
+    'very',
+    'want',
+    'was',
+    'way',
+    'were',
+    'what',
+    'when',
+    'where',
+    'which',
+    'while',
+    'who',
+    'whom',
+    'whose',
+    'will',
+    'with',
+    'would',
+    'yes',
+    'yet',
+    'you',
+    'your'
+])
+
+export function triggersOf(text: string, limit = 8): string[] {
+    const words = text.toLowerCase().match(/[a-z][a-z0-9_-]{3,}/g) ?? []
+    const counts = new Map<string, number>()
+    for (const word of words) {
+        if (STOP.has(word)) continue
+        counts.set(word, (counts.get(word) ?? 0) + 1)
+    }
+    return [...counts.entries()]
+        .toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .slice(0, limit)
+        .map(([word]) => word)
+}
+
+const MIN_TRIGGER_PROMPTS = 2
+
+function plausible(word: string, count: number): boolean {
+    if (count >= MIN_TRIGGER_PROMPTS) return true
+    if (word.length < 8) return false
+    return !/[bcdfghjklmnpqrstvwxz]{4,}/.test(word)
+}
+
+/**
+ * The trigger words are the ones that say the situation, not the ones every
+ * request has. The corpus says which is which: a word the prompts never used
+ * cannot route to anything, so it is not a trigger, and a word most of them
+ * carry would put the tool in every request. A token seen once has to look like
+ * a word, because a typo is the rarest token in the corpus and routes nowhere.
+ */
+export function rareTriggersOf(cue: string, rarity: PromptWords, limit = 6): string[] {
+    const words = [
+        ...new Set((cue.toLowerCase().match(/[a-z][a-z0-9_-]{3,}/g) ?? []).filter((word) => !STOP.has(word)))
+    ]
+    const seen = words
+        .map((word) => ({ word, count: rarity.words.get(word) ?? 0 }))
+        .filter((entry) => entry.count > 0 && plausible(entry.word, entry.count))
+        .toSorted((a, b) => a.count - b.count || a.word.localeCompare(b.word))
+    const rare = seen.filter((entry) => entry.count <= Math.max(1, Math.floor(rarity.count * COMMON_RARITY)))
+    const kept = rare.length > 0 ? rare : seen.slice(0, 3)
+    return kept.slice(0, limit).map((entry) => entry.word)
+}
+
+function rareEnough(word: string, rarity?: PromptWords): boolean {
+    if (rarity === undefined || rarity.count === 0) return word.length >= 6
+    return (rarity.words.get(word) ?? 0) / rarity.count <= 0.005
+}
+
+/**
+ * One word is a coincidence: a request that says "key" is not a request about
+ * credentials. Two triggers, or one long rare one, is a situation.
+ */
+export function matchLessons(lessons: Lesson[], prompt: string, rarity?: PromptWords): Lesson[] {
+    const words = new Set(prompt.toLowerCase().match(/[a-z][a-z0-9_-]{2,}/g) ?? [])
+    const scored = lessons
+        .filter((lesson) => lesson.state !== 'superseded' && lesson.state !== 'rejected' && lesson.state !== 'failed')
+        .map((lesson) => {
+            const triggers = lesson.triggers.length > 0 ? lesson.triggers : triggersOf(lesson.cue)
+            const matched = triggers.filter((word) => words.has(word) && !commonIn(word, rarity))
+            const strong = matched.some((word) => word.length >= 6 && rareEnough(word, rarity))
+            return { lesson, hits: matched.length, strong }
+        })
+        .filter((entry) => entry.hits >= 2 || (entry.hits === 1 && entry.strong))
+        .toSorted((a, b) => b.hits - a.hits || b.lesson.at - a.lesson.at)
+    return scored.map((entry) => entry.lesson)
+}

package/src/producer.ts

@@ -0,0 +1,236 @@
+import { execFileSync } from 'node:child_process'
+import { existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import { writeFileAtomic } from './atomic.js'
+import { harnessDir } from './store.js'
+
+export type ProducerState = 'live' | 'upstream' | 'removed' | 'unknown'
+
+export interface ProducerVerdict {
+    state: ProducerState
+    phrase: string
+    detail: string
+}
+
+/** Returns the command output, an empty string when it found nothing, and null when the tool is not there. */
+export type Runner = (file: string, args: string[], cwd: string) => string | null
+const SEARCH_SKIP = ['!node_modules', '!.git', '!.pi', '!dist', '!build', '!target']
+const PHRASE_SIZE = 40
+
+function isHole(token: string): boolean {
+    return /[\\/:]/.test(token) || /\d/.test(token) || token.startsWith('$')
+}
+
+function runsOf(line: string): string[] {
+    const runs: string[] = []
+    let run: string[] = []
+    for (const token of line.split(/\s+/)) {
+        if (token === '') continue
+        if (isHole(token)) {
+            if (run.length > 0) runs.push(run.join(' '))
+            run = []
+            continue
+        }
+        run.push(token)
+    }
+    if (run.length > 0) runs.push(run.join(' '))
+    return runs
+}
+
+/**
+ * A message is the part of it that the code writes as text. The ids, the paths and the
+ * numbers are the run's, so they are cut out, and the longest run left is what a search
+ * can find in a producer.
+ */
+export function phrasesOf(body: string, limit = 3): string[] {
+    const out: string[] = []
+    for (const line of body.split('\n')) {
+        for (const run of runsOf(line.trim())) {
+            const phrase = run
+                .replace(/^[.,;:]+|[.,;:]+$/g, '')
+                .slice(0, PHRASE_SIZE)
+                .trim()
+            if (phrase.split(' ').length < 4) continue
+            if (!out.includes(phrase)) out.push(phrase)
+        }
+    }
+    return out.toSorted((a, b) => b.length - a.length).slice(0, limit)
+}
+
+const runRg: Runner = (file, args, cwd) => {
+    try {
+        return execFileSync(file, args, {
+            cwd,
+            encoding: 'utf8',
+            stdio: ['ignore', 'pipe', 'ignore'],
+            timeout: 10_000
+        })
+    } catch (error) {
+        const code = (error as { code?: string }).code
+        return code === 'ENOENT' ? null : ''
+    }
+}
+
+function search(run: Runner, cwd: string, phrase: string, root: string, skip: boolean): boolean {
+    const globs = skip ? SEARCH_SKIP.flatMap((glob) => ['--glob', glob]) : []
+    const out = run('rg', ['-F', '-l', '-m1', '--no-messages', ...globs, '--', phrase, root], cwd)
+    return out !== null && out.trim() !== ''
+}
+
+function removedAt(run: Runner, cwd: string, phrase: string): { commit: string; at: number } | null {
+    const out = run('git', ['log', '-S', phrase, '--format=%h %ct', '-n', '1'], cwd)
+    if (out === null) return null
+    const [commit, at] = out.trim().split(' ')
+    if (commit === undefined || at === undefined) return null
+    const seconds = Number(at)
+    return Number.isNaN(seconds) ? null : { commit, at: seconds * 1000 }
+}
+
+/**
+ * `since` is the newest session that hit the failure. A commit that touched the
+ * message before that session did not remove it, so the message still comes from
+ * somewhere, and a commit after it is the producer going away.
+ */
+export function producerOf(
+    cwd: string,
+    roots: string[],
+    body: string,
+    run: Runner = runRg,
+    cache = true,
+    since = 0
+): ProducerVerdict {
+    const phrases = phrasesOf(body)
+    if (phrases.length === 0) return { state: 'unknown', phrase: '', detail: '' }
+    const key = `${phrases[0] ?? ''}@${since}`
+    const head = cache ? cleanHead(run, cwd) : undefined
+    if (head !== undefined) {
+        const hit = readCache(cwd, head).get(key)
+        if (hit !== undefined) return hit
+    }
+    const verdict = decide(cwd, roots, phrases, run, since)
+    if (head !== undefined && verdict.state !== 'unknown') writeCache(cwd, head, key, verdict)
+    return verdict
+}
+
+function decide(cwd: string, roots: string[], phrases: string[], run: Runner, since: number): ProducerVerdict {
+    for (const phrase of phrases) {
+        if (search(run, cwd, phrase, cwd, true)) {
+            return { state: 'live', phrase, detail: 'the string is in the tree' }
+        }
+    }
+    for (const phrase of phrases) {
+        if (roots.some((root) => search(run, cwd, phrase, root, false))) {
+            return { state: 'upstream', phrase, detail: 'the string is in a dependency' }
+        }
+    }
+    if (run('git', ['rev-parse', '--git-dir'], cwd) === null) {
+        return { state: 'unknown', phrase: phrases[0] ?? '', detail: 'no git and no match' }
+    }
+    for (const phrase of phrases) {
+        const gone = removedAt(run, cwd, phrase)
+        if (gone !== null && gone.at >= since) {
+            return { state: 'removed', phrase, detail: `removed by ${gone.commit}` }
+        }
+    }
+    return { state: 'unknown', phrase: phrases[0] ?? '', detail: 'no producer found' }
+}
+
+const CacheSchema = t.object({
+    head: t.optional(t.string()),
+    entries: t.record(t.object({
+            state: t.union([
+                t.literal('live'),
+                t.literal('upstream'),
+                t.literal('removed'),
+                t.literal('unknown')
+            ]),
+            phrase: t.string(),
+            detail: t.string()
+        })
+    )
+})
+const isCacheSchema = new Validator(CacheSchema).isValidObject
+
+interface ProducerCache {
+    head?: string
+    entries: Record<string, ProducerVerdict>
+}
+
+function cacheFile(cwd: string): string {
+    return join(harnessDir(cwd), 'producers.json')
+}
+
+function headOf(run: Runner, cwd: string): string | undefined {
+    const head = run('git', ['rev-parse', 'HEAD'], cwd)?.trim()
+    return head ? head : undefined
+}
+
+/**
+ * A verdict is only reusable when the tree is the commit it was made on. Search results depend
+ * on uncommitted edits, which a HEAD key cannot see, so a dirty tree is never read or written.
+ */
+function cleanHead(run: Runner, cwd: string): string | undefined {
+    const head = headOf(run, cwd)
+    if (head === undefined) return undefined
+    return run('git', ['status', '--porcelain'], cwd) === '' ? head : undefined
+}
+
+function readCache(cwd: string, head: string | undefined): Map<string, ProducerVerdict> {
+    const out = new Map<string, ProducerVerdict>()
+    const path = cacheFile(cwd)
+    if (!existsSync(path)) return out
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'))
+    } catch {
+        return out
+    }
+    if (!isCacheSchema(parsed)) return out
+    if (head === undefined || parsed.head !== head) return out
+    for (const [phrase, entry] of Object.entries(parsed.entries)) out.set(phrase, entry)
+    return out
+}
+
+export function readProducers(cwd: string): ProducerVerdict[] {
+    return [...readCache(cwd, headOf(runRg, cwd)).values()]
+}
+
+const confoundCache = new Map<string, string | null>()
+
+export function commitInWindow(cwd: string, key: string, since: number): string | null {
+    const phrase = phrasesOf(key)[0]
+    if (phrase === undefined) return null
+    const cacheKey = `${cwd}:${headOf(runRg, cwd) ?? ''}:${phrase}:${since}`
+    if (confoundCache.has(cacheKey)) return confoundCache.get(cacheKey) ?? null
+    const out = runRg('git', ['log', '-S', phrase, '--format=%h %ct', '-n', '20'], cwd)
+    let found: string | null = null
+    if (out !== null) {
+        for (const line of out.trim().split('\n')) {
+            const [commit, at] = line.trim().split(' ')
+            if (commit === undefined || at === undefined) continue
+            if (Number(at) * 1000 >= since) {
+                found = commit
+                break
+            }
+        }
+    }
+    confoundCache.set(cacheKey, found)
+    return found
+}
+
+function writeCache(cwd: string, head: string, key: string, verdict: ProducerVerdict): void {
+    const path = cacheFile(cwd)
+    const current = readCache(cwd, head)
+    current.set(key, verdict)
+    const cache: ProducerCache = {
+        head,
+        entries: Object.fromEntries(current)
+    }
+    mkdirSync(harnessDir(cwd), { recursive: true })
+    writeFileAtomic(path, `${JSON.stringify(cache)}\n`)
+}

package/src/prompt.ts

@@ -0,0 +1,162 @@
+import type { Recurrence } from './corpus.js'
+import { rate, type Effect } from './effect.js'
+import type { Target } from './target.js'
+
+export function buildProposalPrompt(
+    target: Target,
+    dir: string,
+    recurrences: Recurrence[] = [],
+    effect: Effect | null = null,
+    standing = 0,
+    form: 'rule' | 'skill' | 'profile' | null = null
+): string {
+    const evidence = target.evidence.map((line) => `- ${line}`).join('\n')
+    const across = recurrences
+        .filter((recurrence) => recurrence.id !== target.id)
+        .map((recurrence) => `- ${recurrence.title} (${recurrence.occurrences} times)`)
+    return [
+        '# WHAT YOU ARE EDITING',
+        '',
+        "pi's harness is the set of files that shape every run: the skills it advertises, the agent",
+        'profiles it can delegate to, the context files it reads at startup, and the extensions that',
+        `sit in the tool loop. You are writing one piece of it, and it goes in ${dir}.`,
+        '',
+        '# THE FINDING',
+        '',
+        target.title,
+        '',
+        `${target.count} occurrences in ${target.scope}. The evidence:`,
+        '',
+        evidence,
+        '',
+        ...(target.sample === '' ? [] : ['The failure itself:', '', '```', target.sample.slice(0, 600), '```', '']),
+        ...(target.tool === '' ? [] : [`The tool that produced it: ${target.tool}.`, '']),
+        ...(across.length === 0
+            ? []
+            : [
+                  '# THE SAME THING IN EARLIER SESSIONS',
+                  '',
+                  'These recur across the sessions of this project, so they are a property of the',
+                  'harness rather than of one run:',
+                  '',
+                  ...across,
+                  ''
+              ]),
+        ...(effect === null || (effect.verdict !== 'flat' && effect.verdict !== 'worse')
+            ? []
+            : [
+                  '# THE EARLIER ATTEMPT',
+                  '',
+                  `A part was installed for this finding already, and the sessions after it show`,
+                  `${effect.verdict === 'worse' ? 'more' : 'the same'} failure: ${rate(effect.before)} before it and`,
+                  `${rate(effect.after)} after it. The form of that part did not carry, so write a`,
+                  'different form, or say KEEP_BASE when the evidence does not argue for one.',
+                  ...(form === null
+                      ? []
+                      : [
+                            '',
+                            `That part was a ${form}.`,
+                            form === 'rule'
+                                ? 'A rule that did not carry is a situation the model cannot place, so a skill that routes on it is the stronger form.'
+                                : 'A skill that did not carry is a situation the model does not route to, so a rule that always applies is what is left.'
+                        ]),
+                  ''
+              ]),
+        '# WHAT TO WRITE',
+        '',
+        ...(target.kind === 'dead-skill'
+            ? [
+                  'pi advertises every skill by name and description in the system prompt, so a skill',
+                  'nobody opens is a tax on every request of every session. The forms:',
+                  '',
+                  `- rewrite ${dir}/<name>/SKILL.md with \`disable-model-invocation: true\` in the`,
+                  '  frontmatter. The skill stays reachable through /skill:name and stops being',
+                  '  advertised.',
+                  '- rewrite the description so it names the situation, when the description is what',
+                  '  keeps the model from choosing it. The description is the only part of a skill the',
+                  '  model sees before it decides.',
+                  '- move the skill to a project skills directory, so it is advertised only in the',
+                  '  projects that need it.',
+                  '- say KEEP_BASE, when the skill earns its place on the situations that rarely come',
+                  '  up and a router would miss it.',
+                  '',
+                  'A skill that was never opened in the sessions on disk might still be the right',
+                  'tool for a task that has not come up yet. Say so if you think so.'
+              ]
+            : [
+                  'One of:',
+                  '',
+                  `- a skill: ${dir}/<name>/SKILL.md, with frontmatter \`name:\` and \`description:\`, and any`,
+                  `  helper it runs under ${dir}/<name>/scripts/. The description is what the model routes on,`,
+                  '  so it says what the skill does and when it applies.',
+                  `- an agent profile: ${dir}/<name>.md, with frontmatter \`name:\` and \`description:\`.`,
+                  `- a rule: ${dir}/RULE.md, appended to the project context file.`,
+                  '',
+                  `Put a skill or a rule under ${dir}/global/ when it has to hold in every project.`,
+                  'A skill under global/ goes to the skills every project loads, and a global rule',
+                  'goes to the context file every project reads.',
+                  '',
+                  'Both forms cost standing text on every request. A skill costs its name and',
+                  'description, about 300 characters, and a rule costs its length. The difference is',
+                  'that a rule always applies while a skill only applies when the model routes to',
+                  'it, and a skill body costs nothing until it is opened. Say how many characters',
+                  'the part adds when it goes into an always-on file, because that number is a tax',
+                  'on every request from then on.',
+                  '',
+                  ...(standing === 0
+                      ? []
+                      : [
+                            `The parts installed for this project already cost ${standing} characters on`,
+                            'every request. The cheaper form that still holds is the right one.',
+                            ''
+                        ])
+              ]),
+        '',
+        '# THE CHECK',
+        '',
+        `Write ${dir}/CONFIRM.md with one shell command that exits 0 only when the failure above`,
+        'can no longer happen. It must not write files, install anything, or use the network. The loop',
+        'runs it after the project check passes, and a confirmation that fails puts the part back.',
+        'When you cannot name a command that proves the change, leave the file out and say why.',
+        '',
+        ...(target.command === undefined
+            ? []
+            : [
+                  'The command that produced the failure is replayed after the install, so a fix that',
+                  'does not change its result is visible in the record:',
+                  '',
+                  '```',
+                  target.command.slice(0, 300),
+                  '```',
+                  ''
+              ]),
+        '# THE OUTPUT',
+        '',
+        'The loop writes these files itself when the finding comes from the automatic run. A person or',
+        'a session may write them by hand instead, and nothing else is installed.',
+        '',
+        '```',
+        '<file path="RULE.md">',
+        '## Heading',
+        'the rule',
+        '</file>',
+        '```',
+        '',
+        `A path is relative to ${dir}: RULE.md, CONFIRM.md, <name>/SKILL.md, <name>.md for a`,
+        'profile, <name>/scripts/<file> for a helper, and the same under global/ for every project.',
+        '',
+        '# THE RULES',
+        '',
+        '1. Every change traces to a moment in the evidence above. Name the moment. A change you',
+        '   cannot trace is speculation, and speculation in a harness is paid for on every later run.',
+        '2. KEEP_BASE is a real answer. When the evidence does not argue for a change, write nothing',
+        '   and reply KEEP_BASE.',
+        '3. The harness changes how pi works, not what the work produces. Do not change the shape of',
+        '   the deliverable.',
+        '4. Every helper your component names ships with it, or the component is dead on arrival.',
+        '5. The part is read by every later run, so it never names the session files, the finding id, or',
+        '   the harness directory. It is a rule about the work, not a report of the evidence.',
+        '6. Reply with the file blocks and nothing outside them. A reply that names the moment in a',
+        '   comment instead of a file installs nothing.'
+    ].join('\n')
+}