@tduyng/pi-harness-loop 0.0.0-stage → 0.0.2

package/src/corpus.ts

@@ -0,0 +1,641 @@
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { basename, dirname, join } from 'node:path'
+
+import { Validator, type Infer } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import { writeFileAtomic } from './atomic.js'
+import { failureSignature, usableSignature } from './findings.js'
+const isString = new Validator(t.string()).isValidObject
+
+export const MAX_CACHED_SESSIONS = 400
+export const MAX_SESSION_BYTES = 24_000_000
+export const MAX_NEW_SESSION_BYTES = 8_000_000
+export const MIN_SESSIONS = 2
+export const MIN_FAILURE_OCCURRENCES = 2
+export const MIN_HABIT_OCCURRENCES = 3
+export const DEAD_SKILL_SESSIONS = 20
+
+const CorpusSessionSchema = t.object({
+    file: t.string(),
+    prompts: t.array(t.string()),
+    errors: t.array(
+        t.object({
+            signature: t.string(),
+            tool: t.string(),
+            body: t.string(),
+            command: t.optional(t.string())
+        })
+    ),
+    commands: t.array(t.string()),
+    reads: t.array(t.string()),
+    skills: t.array(t.string()),
+    /** Missing in caches written before edits were counted, so an old cache reads as unknown. */
+    edits: t.optional(t.array(t.string())),
+    /** Missing in caches written before tool use was counted, so an old cache reads as unknown. */
+    tools: t.optional(t.record(t.number()))
+})
+
+const CorpusCacheSchema = t.object({
+    files: t.array(
+        t.object({
+            path: t.string(),
+            mtime: t.number(),
+            size: t.number(),
+            session: CorpusSessionSchema
+        })
+    )
+})
+const isCorpusCacheSchema = new Validator(CorpusCacheSchema).isValidObject
+
+export interface Recurrence {
+    id: string
+    /** The raw signature, so an install can be measured against the sessions after it. */
+    key: string
+    kind: 'error' | 'command' | 'read'
+    title: string
+    sessions: number
+    occurrences: number
+    tool: string
+    sample: string
+    command?: string
+    /** Why the loop will not act on it, when the evidence comes from one piece of work. */
+    blocked?: string
+    evidence: string[]
+}
+
+export interface SkillUse {
+    id: string
+    name: string
+    sessions: number
+    opens: number
+}
+
+export interface CorpusSession {
+    file: string
+    prompts: string[]
+    errors: { signature: string; tool: string; body: string; command?: string }[]
+    commands: string[]
+    reads: string[]
+    skills: string[]
+    edits: string[]
+    tools?: Record<string, number>
+}
+
+const Block = t.object({
+    type: t.string(),
+    id: t.optional(t.string()),
+    name: t.optional(t.string()),
+    arguments: t.optional(t.unknown())
+})
+
+const AssistantMessage = t.object({
+    type: t.literal('message'),
+    message: t.object({ role: t.literal('assistant'), content: t.array(Block) })
+})
+const isAssistantMessage = new Validator(AssistantMessage).isValidObject
+
+const ResultBlock = t.object({ type: t.string(), text: t.optional(t.string()) })
+type ResultBlock = Infer<typeof ResultBlock>
+
+const ToolMessage = t.object({
+    type: t.literal('message'),
+    message: t.object({
+        role: t.literal('toolResult'),
+        toolCallId: t.optional(t.string()),
+        toolName: t.string(),
+        isError: t.boolean(),
+        content: t.union([t.string(), t.array(ResultBlock)])
+    })
+})
+const isToolMessage = new Validator(ToolMessage).isValidObject
+
+const UserMessage = t.object({
+    type: t.literal('message'),
+    message: t.object({
+        role: t.literal('user'),
+        content: t.union([t.string(), t.array(ResultBlock)])
+    })
+})
+const isUserMessage = new Validator(UserMessage).isValidObject
+
+const CommandArguments = t.object({ command: t.string() })
+const isCommandArguments = new Validator(CommandArguments).isValidObject
+const PathArguments = t.object({ path: t.string() })
+const isPathArguments = new Validator(PathArguments).isValidObject
+
+export function sessionFiles(dir: string): string[] {
+    if (!existsSync(dir)) return []
+    try {
+        return readdirSync(dir, { recursive: true, withFileTypes: true })
+            .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
+            .map((entry) => {
+                const file = join(entry.parentPath, entry.name)
+                return { file, at: statOf(file).mtime }
+            })
+            .toSorted((a, b) => b.at - a.at)
+            .map((entry) => entry.file)
+    } catch {
+        return []
+    }
+}
+
+function resultBody(content: string | ResultBlock[]): string {
+    if (isString(content)) return content
+    return content.map((block) => block.text ?? '').join('\n')
+}
+
+function skillNamesIn(command: string): string[] {
+    const out: string[] = []
+    for (const match of command.matchAll(/([\w.-]+)\/SKILL\.md/g)) {
+        const name = match[1]
+        if (name !== undefined) out.push(name)
+    }
+    return out
+}
+
+export function readCorpusSession(file: string): CorpusSession {
+    const session: CorpusSession = {
+        file,
+        prompts: [],
+        errors: [],
+        commands: [],
+        reads: [],
+        skills: [],
+        edits: [],
+        tools: {}
+    }
+    const commands = new Map<string, string>()
+    let text: string
+    try {
+        if (statSync(file).size > MAX_SESSION_BYTES) return session
+        text = readFileSync(file, 'utf8')
+    } catch {
+        return session
+    }
+    for (const line of text.split('\n')) {
+        if (line.trim() === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(line)
+        } catch {
+            continue
+        }
+        if (isUserMessage(parsed)) {
+            const body = resultBody(parsed.message.content)
+            for (const match of body.matchAll(/\/skill:([\w.-]+)/g)) {
+                const name = match[1]
+                if (name !== undefined) session.skills.push(name)
+            }
+            if (body.trim() !== '' && !body.trimStart().startsWith('<')) session.prompts.push(body)
+            continue
+        }
+        if (isAssistantMessage(parsed)) {
+            for (const block of parsed.message.content) {
+                if (block.type !== 'toolCall') continue
+                if (block.name !== undefined) {
+                    session.tools ??= {}
+                    session.tools[block.name] = (session.tools[block.name] ?? 0) + 1
+                }
+                if (block.name === 'bash' && isCommandArguments(block.arguments)) {
+                    const command = block.arguments.command
+                    session.commands.push(command)
+                    if (block.id !== undefined) commands.set(block.id, command)
+                    for (const name of skillNamesIn(command)) session.skills.push(name)
+                }
+                if (block.name === 'read' && isPathArguments(block.arguments)) {
+                    const path = block.arguments.path
+                    session.reads.push(path)
+                    if (basename(path) === 'SKILL.md') session.skills.push(basename(dirname(path)))
+                }
+                if ((block.name === 'edit' || block.name === 'write') && isPathArguments(block.arguments)) {
+                    session.edits.push(basename(block.arguments.path))
+                }
+            }
+            continue
+        }
+        if (!isToolMessage(parsed)) continue
+        if (!parsed.message.isError) continue
+        const body = resultBody(parsed.message.content)
+        const signature = failureSignature(body)
+        if (signature === '') continue
+        const command = parsed.message.toolCallId === undefined ? undefined : commands.get(parsed.message.toolCallId)
+        const error = { signature, tool: parsed.message.toolName, body: body.slice(0, 400), command }
+        session.errors.push(error)
+    }
+    session.edits = [...new Set(session.edits)]
+    return session
+}
+
+export function idFor(key: string): string {
+    return createHash('sha1').update(key).digest('hex').slice(0, 8)
+}
+
+function tally(
+    sessions: CorpusSession[],
+    pick: (session: CorpusSession) => string[]
+): Map<string, { files: Set<string>; count: number }> {
+    const out = new Map<string, { files: Set<string>; count: number }>()
+    for (const session of sessions) {
+        for (const value of pick(session)) {
+            if (value === '') continue
+            const found = out.get(value) ?? { files: new Set<string>(), count: 0 }
+            found.files.add(session.file)
+            found.count += 1
+            out.set(value, found)
+        }
+    }
+    return out
+}
+
+function collect(
+    kind: Recurrence['kind'],
+    sessions: CorpusSession[],
+    pick: (session: CorpusSession) => string[],
+    label: (key: string, files: number) => string,
+    minOccurrences: number
+): Recurrence[] {
+    const out: Recurrence[] = []
+    for (const [key, found] of tally(sessions, pick)) {
+        if (found.files.size < MIN_SESSIONS || found.count < minOccurrences) continue
+        out.push({
+            id: idFor(`${kind}:${key}`),
+            key,
+            kind,
+            title: label(key, found.files.size),
+            sessions: found.files.size,
+            occurrences: found.count,
+            tool: '',
+            sample: '',
+            evidence: [...found.files].map((file) => basename(file)).slice(0, 4)
+        })
+    }
+    return out
+}
+
+function collectErrors(sessions: CorpusSession[]): Recurrence[] {
+    const byKey = new Map<
+        string,
+        {
+            files: Set<string>
+            edits: Map<string, Set<string>>
+            count: number
+            tool: string
+            sample: string
+            command?: string
+        }
+    >()
+    for (const session of sessions) {
+        for (const error of session.errors) {
+            if (!usableSignature(error.signature)) continue
+            const found = byKey.get(error.signature) ?? {
+                files: new Set<string>(),
+                edits: new Map<string, Set<string>>(),
+                count: 0,
+                tool: error.tool,
+                sample: error.body,
+                command: error.command
+            }
+            if (found.command === undefined && error.command !== undefined) found.command = error.command
+            found.files.add(session.file)
+            found.edits.set(session.file, new Set(session.edits))
+            found.count += 1
+            byKey.set(error.signature, found)
+        }
+    }
+    const out: Recurrence[] = []
+    for (const [key, found] of byKey) {
+        if (found.files.size < MIN_SESSIONS || found.count < MIN_FAILURE_OCCURRENCES) continue
+        out.push({
+            id: idFor(`error:${key}`),
+            key,
+            kind: 'error',
+            title: `the same failure shows up in ${found.files.size} sessions: ${key}`,
+            sessions: found.files.size,
+            occurrences: found.count,
+            tool: found.tool,
+            sample: found.sample,
+            command: found.command,
+            blocked: onePieceOfWork(found.edits) ? 'the sessions that hit it edited the same files' : undefined,
+            evidence: [...found.files].map((file) => basename(file)).slice(0, 4)
+        })
+    }
+    return out
+}
+
+/**
+ * Are the sessions that hit this failure one piece of work? They are when every one of them
+ * edited a file and every pair shares one. A session that edited nothing met the failure
+ * outside the edit loop, so its evidence is independent.
+ */
+function onePieceOfWork(work: Map<string, Set<string>>): boolean {
+    const sets = [...work.values()]
+    if (sets.some((edits) => edits.size === 0)) return false
+    for (let i = 0; i < sets.length; i += 1) {
+        for (let j = i + 1; j < sets.length; j += 1) {
+            const left = sets[i]
+            const right = sets[j]
+            if (left === undefined || right === undefined) continue
+            let shared = false
+            for (const file of left) {
+                if (right.has(file)) {
+                    shared = true
+                    break
+                }
+            }
+            if (!shared) return false
+        }
+    }
+    return true
+}
+
+export function findRecurrences(sessionDir: string): Recurrence[] {
+    return scanCorpus(sessionDir, []).recurrences
+}
+
+export interface CorpusScan {
+    recurrences: Recurrence[]
+    skills: SkillUse[]
+    sessions: number
+    available: number
+    prompts: PromptWords
+}
+
+export interface PromptWords {
+    count: number
+    words: Map<string, number>
+}
+
+/**
+ * One pass over the session files, because the corpus is the biggest thing the
+ * loop reads and reading it twice for two answers is the same work twice.
+ */
+export function scanCorpus(sessionDir: string, names: string[], cachePath?: string, skipSuffix?: string): CorpusScan {
+    const cache = cachePath === undefined ? new Map<string, CorpusCacheEntry>() : readCorpusCache(cachePath)
+    const files = sessionFiles(sessionDir)
+    const kept = new Map<string, CorpusCacheEntry>()
+    let parsedBytes = 0
+    let parsed = 0
+    let changed = false
+    for (const file of files) {
+        if (kept.size >= MAX_CACHED_SESSIONS) break
+        if (skipSuffix !== undefined && file.endsWith(skipSuffix)) continue
+        const stat = statOf(file)
+        const hit = cache.get(file)
+        if (
+            hit !== undefined &&
+            hit.mtime === stat.mtime &&
+            hit.size === stat.size &&
+            hit.session.edits !== undefined
+        ) {
+            kept.set(file, hit)
+            continue
+        }
+        if (parsed > 0 && parsedBytes + stat.size > MAX_NEW_SESSION_BYTES) {
+            if (hit !== undefined && hit.session.edits !== undefined) kept.set(file, hit)
+            continue
+        }
+        if (stat.size > MAX_SESSION_BYTES) {
+            if (hit !== undefined && hit.session.edits !== undefined) kept.set(file, hit)
+            continue
+        }
+        const session = readCorpusSession(file)
+        kept.set(file, { path: file, mtime: stat.mtime, size: stat.size, session })
+        parsedBytes += stat.size
+        parsed += 1
+        changed = true
+    }
+    if (cache.size !== kept.size) changed = true
+    if (cachePath !== undefined && changed) writeCorpusCache(cachePath, [...kept.values()])
+    const sessions: CorpusSession[] = [...kept.values()].map(({ session }) => ({
+        ...session,
+        edits: session.edits ?? []
+    }))
+    const available = files.length
+    if (sessions.length < MIN_SESSIONS) {
+        return {
+            recurrences: [],
+            skills: [],
+            sessions: sessions.length,
+            available,
+            prompts: { count: 0, words: new Map() }
+        }
+    }
+    return {
+        recurrences: recurrencesFrom(sessions),
+        skills: skillUseFrom(sessions, names),
+        sessions: sessions.length,
+        available,
+        prompts: promptWords(sessions)
+    }
+}
+
+interface CorpusCacheEntry {
+    path: string
+    mtime: number
+    size: number
+    /** Older caches have no `edits`, which reads as unknown and forces a reparse of that file. */
+    session: Infer<typeof CorpusSessionSchema>
+}
+
+function readCorpusCache(path: string): Map<string, CorpusCacheEntry> {
+    const cache = new Map<string, CorpusCacheEntry>()
+    if (!existsSync(path)) return cache
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'))
+    } catch {
+        return cache
+    }
+    if (!isCorpusCacheSchema(parsed)) return cache
+    for (const entry of parsed.files) cache.set(entry.path, entry)
+    return cache
+}
+
+function writeCorpusCache(path: string, files: CorpusCacheEntry[]): void {
+    try {
+        mkdirSync(dirname(path), { recursive: true })
+        writeFileAtomic(path, `${JSON.stringify({ files })}\n`)
+    } catch {
+        return
+    }
+}
+
+export function corpusCachePath(cwd: string): string {
+    return join(cwd, '.pi', 'harness', 'corpus-cache.json')
+}
+
+export interface SignatureUse {
+    /** When the session started, from the name of the session file. */
+    at: number
+    count: number
+}
+
+let historyCache: {
+    path: string
+    mtime: number
+    size: number
+    value: History
+} | null = null
+
+export interface History {
+    /** Failures per session, for each signature. */
+    signatures: Map<string, SignatureUse[]>
+    /** Calls per session, for each tool name. */
+    tools: Map<string, SignatureUse[]>
+    /** Calls of every tool per session, which is how much work the session did. */
+    calls: SignatureUse[]
+}
+
+interface SessionHistory {
+    at: number
+    counts: Map<string, number>
+    tools: Record<string, number>
+}
+
+/**
+ * The session file is named after the run's start, and its mtime is when the file
+ * was last copied, so the name is the date the failures happened on.
+ */
+export function sessionStartedAt(file: string, fallback: number): number {
+    const match = /(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})/.exec(basename(file))
+    if (match === null) return fallback
+    const [year, month, day, hour, minute, second] = match.slice(1).map(Number)
+    const at = Date.UTC(year ?? 0, (month ?? 1) - 1, day ?? 1, hour ?? 0, minute ?? 0, second ?? 0)
+    return Number.isNaN(at) || at === 0 ? fallback : at
+}
+
+/**
+ * What each failure signature did per session, read from the parsed corpus
+ * instead of the transcripts. It is what an install can be measured against:
+ * the sessions before it and the sessions after it, including the ones that
+ * never hit the failure, because a session that did not fail is the evidence.
+ */
+export function signatureHistory(cachePath: string): Map<string, SignatureUse[]> {
+    return historyOf(cachePath).signatures
+}
+
+/**
+ * A part can lower the rate of a failure without preventing it, by making the agent do less
+ * work. The calls of every tool per session is what tells the two apart, so it is read from
+ * the same parse as the signatures.
+ */
+export function callHistory(cachePath: string): SignatureUse[] {
+    return historyOf(cachePath).calls
+}
+
+export function historyOf(cachePath: string): History {
+    const stat = statOf(cachePath)
+    if (
+        historyCache !== null &&
+        historyCache.path === cachePath &&
+        historyCache.mtime === stat.mtime &&
+        historyCache.size === stat.size
+    ) {
+        return historyCache.value
+    }
+    const sessions: SessionHistory[] = []
+    for (const entry of readCorpusCache(cachePath).values()) {
+        const counts = new Map<string, number>()
+        for (const error of entry.session.errors) {
+            if (!usableSignature(error.signature)) continue
+            counts.set(error.signature, (counts.get(error.signature) ?? 0) + 1)
+        }
+        sessions.push({
+            at: sessionStartedAt(entry.path, entry.mtime),
+            counts,
+            tools: entry.session.tools ?? {}
+        })
+    }
+    sessions.sort((a, b) => a.at - b.at)
+    const signatures = new Set<string>()
+    const tools = new Set<string>()
+    for (const session of sessions) {
+        for (const signature of session.counts.keys()) signatures.add(signature)
+        for (const tool of Object.keys(session.tools)) tools.add(tool)
+    }
+    const series = (keys: Set<string>, count: (session: SessionHistory, key: string) => number) => {
+        const out = new Map<string, SignatureUse[]>()
+        for (const key of keys) {
+            out.set(
+                key,
+                sessions.map((session) => ({ at: session.at, count: count(session, key) }))
+            )
+        }
+        return out
+    }
+    const value: History = {
+        signatures: series(signatures, (session, key) => session.counts.get(key) ?? 0),
+        tools: series(tools, (session, key) => session.tools[key] ?? 0),
+        calls: sessions.map((session) => ({
+            at: session.at,
+            count: Object.values(session.tools).reduce((total, count) => total + count, 0)
+        }))
+    }
+    historyCache = { path: cachePath, mtime: stat.mtime, size: stat.size, value }
+    return value
+}
+
+function statOf(file: string) {
+    try {
+        const stat = statSync(file)
+        return { mtime: stat.mtimeMs, size: stat.size }
+    } catch {
+        return { mtime: 0, size: 0 }
+    }
+}
+
+function recurrencesFrom(sessions: CorpusSession[]): Recurrence[] {
+    const errors = collectErrors(sessions)
+    const commands = collect(
+        'command',
+        sessions,
+        (session) => session.commands,
+        (key, files) => `the same command runs in ${files} sessions: ${key.slice(0, 100)}`,
+        MIN_HABIT_OCCURRENCES
+    )
+    const reads = collect(
+        'read',
+        sessions,
+        (session) => session.reads,
+        (key, files) => `the same file is read in ${files} sessions: ${key}`,
+        MIN_HABIT_OCCURRENCES
+    )
+    return [...errors, ...commands, ...reads].toSorted(
+        (a, b) => b.sessions - a.sessions || b.occurrences - a.occurrences
+    )
+}
+
+export function findSkillUse(sessionDir: string, names: string[]): SkillUse[] {
+    return scanCorpus(sessionDir, names).skills
+}
+
+function promptWords(sessions: CorpusSession[]): PromptWords {
+    const words = new Map<string, number>()
+    let count = 0
+    for (const session of sessions) {
+        for (const prompt of session.prompts) {
+            count += 1
+            for (const word of new Set(prompt.toLowerCase().match(/[a-z][a-z0-9_-]{2,}/g) ?? [])) {
+                words.set(word, (words.get(word) ?? 0) + 1)
+            }
+        }
+    }
+    return { count, words }
+}
+
+function skillUseFrom(sessions: CorpusSession[], names: string[]): SkillUse[] {
+    const wanted = new Set(names)
+    const out = new Map<string, SkillUse>()
+    for (const name of wanted) out.set(name, { id: `skill-${name}`, name, sessions: sessions.length, opens: 0 })
+    for (const session of sessions) {
+        for (const name of new Set(session.skills)) {
+            const found = out.get(name)
+            if (found === undefined) continue
+            found.opens += 1
+        }
+    }
+    return [...out.values()].toSorted((a, b) => a.opens - b.opens)
+}

package/src/diversity.ts

@@ -0,0 +1,73 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export interface ProposalText {
+    id: string
+    text: string
+}
+
+export function proposalTexts(harnessDir: string): ProposalText[] {
+    if (!existsSync(harnessDir)) return []
+    const out: ProposalText[] = []
+    for (const name of readdirSync(harnessDir).toSorted()) {
+        const dir = join(harnessDir, name)
+        let entries: string[]
+        try {
+            entries = readdirSync(dir)
+        } catch {
+            continue
+        }
+        const text: string[] = []
+        for (const entry of entries.toSorted()) {
+            const path = join(dir, entry)
+            if (entry === 'BRIEF.md') continue
+            try {
+                text.push(readFileSync(path, 'utf8'))
+            } catch {
+                continue
+            }
+        }
+        if (text.length > 0) out.push({ id: name, text: text.join('\n') })
+    }
+    return out
+}
+
+function lines(text: string): Set<string> {
+    return new Set(
+        text
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line !== '')
+    )
+}
+
+export function jaccard(left: string, right: string): number {
+    const a = lines(left)
+    const b = lines(right)
+    if (a.size === 0 || b.size === 0) return 0
+    let shared = 0
+    for (const line of a) if (b.has(line)) shared += 1
+    return shared / (a.size + b.size - shared)
+}
+
+export interface Collapse {
+    pair: [string, string]
+    similarity: number
+}
+
+/**
+ * The loop writing the same part twice is the failure of a self-improving loop
+ * that nobody watches, so the pair that says it is named.
+ */
+export function findCollapse(proposals: ProposalText[], threshold = 0.8): Collapse | null {
+    let worst: Collapse | null = null
+    for (let left = 0; left < proposals.length; left += 1) {
+        for (let right = left + 1; right < proposals.length; right += 1) {
+            const similarity = jaccard(proposals[left]?.text ?? '', proposals[right]?.text ?? '')
+            if (similarity < threshold) continue
+            if (worst !== null && worst.similarity >= similarity) continue
+            worst = { pair: [proposals[left]?.id ?? '', proposals[right]?.id ?? ''], similarity }
+        }
+    }
+    return worst
+}