@tduyng/pi-harness-loop 0.0.0-stage → 0.0.2

package/src/replay.ts

@@ -0,0 +1,173 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import { runCheck } from './check.js'
+import type { Recurrence } from './corpus.js'
+import { harnessDir } from './store.js'
+
+export const REPLAY_TTL_MS = 24 * 60 * 60 * 1000
+export const REPLAY_TIMEOUT_MS = 60_000
+
+export interface Replay {
+    at: number
+    id: string
+    command: string
+    ok: boolean
+    exitCode: number | null
+    output: string
+}
+
+export type ReplayVerdict = 'stale' | 'live' | 'unknown'
+
+const ReplaySchema = t.object({
+    at: t.number(),
+    id: t.string(),
+    command: t.string(),
+    ok: t.boolean(),
+    exitCode: t.union([t.number(), t.null()]),
+    output: t.string()
+})
+const isReplaySchema = new Validator(ReplaySchema).isValidObject
+
+const READ_ONLY_BINS = new Set([
+    '[',
+    'awk',
+    'basename',
+    'cat',
+    'comm',
+    'cut',
+    'date',
+    'diff',
+    'dirname',
+    'echo',
+    'fd',
+    'fdfind',
+    'grep',
+    'head',
+    'id',
+    'jq',
+    'ls',
+    'printenv',
+    'printf',
+    'pwd',
+    'realpath',
+    'rg',
+    'sort',
+    'stat',
+    'tail',
+    'test',
+    'tr',
+    'type',
+    'uniq',
+    'wc',
+    'which'
+])
+
+const GIT_READ = new Set([
+    'blame',
+    'branch',
+    'describe',
+    'diff',
+    'grep',
+    'log',
+    'ls-files',
+    'remote',
+    'rev-parse',
+    'show',
+    'status'
+])
+
+const SHELL_SYNTAX = /[<>&;`\n]|\$\(/
+
+function segmentAllowed(segment: string): boolean {
+    const [bin, ...args] = segment.split(/\s+/)
+    if (bin === undefined) return false
+    if (args.some((arg) => arg.startsWith('--output'))) return false
+    if (bin === 'git') {
+        const sub = args[0] ?? ''
+        if (!GIT_READ.has(sub)) return false
+        if (sub === 'branch') {
+            return args.length === 1 || args.slice(1).every((arg) => /^(-a|-r|-v|--list|--all|--remotes)$/.test(arg))
+        }
+        if (sub === 'remote') return args.length === 1 || args[1] === '-v' || args[1] === 'show'
+        return true
+    }
+    if (bin === 'sed') return !args.some((arg) => arg === '-i' || arg.startsWith('--in-place'))
+    if (bin === 'npx') return args[0] === 'tsc' && args.includes('--noEmit')
+    if (bin === 'node') return args.length === 1 && args[0] === '--version'
+    if (bin === 'find') return !args.some((arg) => /^-(delete|exec|execdir|ok)$/.test(arg))
+    if (bin === 'awk') return !args.some((arg) => arg.includes('system('))
+    return READ_ONLY_BINS.has(bin)
+}
+
+export function canReplay(command: string): boolean {
+    const trimmed = command
+        .trim()
+        .replace(/\s*2>&1\s*/g, ' ')
+        .trim()
+    if (trimmed === '' || trimmed.length > 600 || SHELL_SYNTAX.test(trimmed)) return false
+    return trimmed.split('|').every((segment) => segmentAllowed(segment.trim()))
+}
+
+function replayFile(cwd: string): string {
+    return join(harnessDir(cwd), 'replay.jsonl')
+}
+
+export function readReplays(cwd: string): Replay[] {
+    const path = replayFile(cwd)
+    if (!existsSync(path)) return []
+    const out: Replay[] = []
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+        if (line.trim() === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(line)
+        } catch {
+            continue
+        }
+        if (isReplaySchema(parsed)) out.push(parsed)
+    }
+    return out
+}
+
+export function appendReplay(cwd: string, replay: Replay): void {
+    mkdirSync(harnessDir(cwd), { recursive: true })
+    appendFileSync(replayFile(cwd), `${JSON.stringify(replay)}\n`, 'utf8')
+}
+
+export function replayVerdict(cwd: string, recurrence: Recurrence): ReplayVerdict {
+    if (recurrence.command === undefined) return 'unknown'
+    const latest = readReplays(cwd)
+        .filter((replay) => replay.id === recurrence.id && replay.command === recurrence.command)
+        .toSorted((a, b) => b.at - a.at)[0]
+    if (latest === undefined || Date.now() - latest.at > REPLAY_TTL_MS) return 'unknown'
+    return latest.ok ? 'stale' : 'live'
+}
+
+export async function runReplay(cwd: string, id: string, command: string): Promise<Replay | null> {
+    if (!canReplay(command)) return null
+    const result = await runCheck(cwd, command, { timeoutMs: REPLAY_TIMEOUT_MS })
+    const replay: Replay = {
+        at: Date.now(),
+        id,
+        command,
+        ok: result.ok,
+        exitCode: result.exitCode,
+        output: result.output.slice(0, 2000)
+    }
+    appendReplay(cwd, replay)
+    return replay
+}
+
+export async function probeReplay(cwd: string, recurrence: Recurrence): Promise<ReplayVerdict> {
+    const cached = replayVerdict(cwd, recurrence)
+    if (cached !== 'unknown') return cached
+    if (recurrence.command === undefined) return 'unknown'
+    const replay = await runReplay(cwd, recurrence.id, recurrence.command)
+    if (replay === null) return 'unknown'
+    return replay.ok ? 'stale' : 'live'
+}

package/src/store.ts

@@ -0,0 +1,274 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { randomUUID } from 'node:crypto'
+import { dirname, join } from 'node:path'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+import { writeFileAtomic } from './atomic.js'
+import type { HarnessSettings, Install, Lesson } from './types.js'
+
+export function harnessDir(cwd: string): string {
+    return join(cwd, '.pi', 'harness')
+}
+
+export function lessonsFile(cwd: string): string {
+    return join(harnessDir(cwd), 'lessons.jsonl')
+}
+
+export function installsFile(cwd: string): string {
+    return join(harnessDir(cwd), 'installs.jsonl')
+}
+
+export function settingsFile(cwd: string): string {
+    return join(cwd, '.pi', 'harness.json')
+}
+
+const LessonSchema = t.object({
+    id: t.string(),
+    at: t.number(),
+    cue: t.string(),
+    lesson: t.string(),
+    evidence: t.string(),
+    confidence: t.union([t.literal('high'), t.literal('medium'), t.literal('low')]),
+    triggers: t.array(t.string()),
+    kind: t.union([
+        t.literal('repeat-error'),
+        t.literal('repeat-command'),
+        t.literal('repeat-read'),
+        t.literal('no-change'),
+        t.literal('abandoned-tool')
+    ]),
+    state: t.union([
+        t.literal('recorded'),
+        t.literal('installed'),
+        t.literal('failed'),
+        t.literal('rejected'),
+        t.literal('superseded')
+    ]),
+    confirm: t.optional(t.string()),
+    supersedes: t.optional(t.string()),
+    uses: t.optional(t.number()),
+    lastUsed: t.optional(t.number())
+})
+const isLessonSchema = new Validator(LessonSchema).isValidObject
+
+const SettingsSchema = t.object({
+    check: t.optional(t.string()),
+    auto: t.optional(t.boolean()),
+    autoInstall: t.optional(t.boolean()),
+    global: t.optional(t.boolean()),
+    maxInstallsPerDay: t.optional(t.number()),
+    maxProposalsPerDay: t.optional(t.number()),
+    maxTurnsPerProposal: t.optional(t.number()),
+    maxStandingTokens: t.optional(t.number()),
+    writerModel: t.optional(t.string())
+})
+const isSettingsSchema = new Validator(SettingsSchema).isValidObject
+
+const InstalledFile = t.object({
+    to: t.string(),
+    existed: t.boolean(),
+    before: t.string(),
+    after: t.optional(t.string()),
+    afterHash: t.optional(t.string()),
+    block: t.optional(t.string()),
+    mode: t.union([t.literal('replace'), t.literal('append')])
+})
+
+const InstallSchema = t.object({
+    id: t.string(),
+    at: t.number(),
+    finding: t.string(),
+    dir: t.string(),
+    files: t.array(InstalledFile),
+    check: t.string(),
+    ok: t.boolean(),
+    reverted: t.boolean(),
+    lesson: t.optional(t.string()),
+    key: t.optional(t.string()),
+    cost: t.optional(t.number()),
+    confirm: t.optional(t.string()),
+    confirmOk: t.optional(t.boolean()),
+    confirmOutput: t.optional(t.string()),
+    replay: t.optional(t.string()),
+    replayOk: t.optional(t.boolean()),
+    replayOutput: t.optional(t.string()),
+    note: t.optional(t.string())
+})
+const isInstallSchema = new Validator(InstallSchema).isValidObject
+
+function normalizedCue(cue: string): string {
+    return cue.toLowerCase().replace(/\s+/g, ' ').trim()
+}
+
+function active(lesson: Lesson): boolean {
+    return lesson.state === 'recorded' || lesson.state === 'installed'
+}
+
+/**
+ * The file is append-only, so two processes can record the same cue as two active
+ * lines. The view keeps the newest active line of a cue, and every settled line,
+ * because a superseded record is how the report finds the rule still installed.
+ */
+function withoutDuplicateActives(lessons: Lesson[]): Lesson[] {
+    const newest = new Map<string, Lesson>()
+    for (const lesson of lessons) {
+        if (!active(lesson)) continue
+        const key = normalizedCue(lesson.cue)
+        const seen = newest.get(key)
+        if (seen === undefined || seen.at <= lesson.at) newest.set(key, lesson)
+    }
+    return lessons.filter((lesson) => !active(lesson) || newest.get(normalizedCue(lesson.cue))?.id === lesson.id)
+}
+
+export function readLessons(cwd: string): Lesson[] {
+    const path = lessonsFile(cwd)
+    if (!existsSync(path)) return []
+    const lessons = new Map<string, Lesson>()
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+        const trimmed = line.trim()
+        if (trimmed === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(trimmed)
+        } catch {
+            continue
+        }
+        if (isLessonSchema(parsed)) lessons.set(parsed.id, parsed)
+    }
+    return withoutDuplicateActives([...lessons.values()])
+}
+
+export function appendLesson(cwd: string, lesson: Lesson): void {
+    const path = lessonsFile(cwd)
+    mkdirSync(dirname(path), { recursive: true })
+    appendFileSync(path, `${JSON.stringify(lesson)}\n`, 'utf8')
+}
+
+function sameCue(left: string, right: string): boolean {
+    return normalizedCue(left) === normalizedCue(right)
+}
+
+/** A lesson whose part did not hold stops matching, and the record keeps why. */
+export function failLesson(cwd: string, id: string): boolean {
+    const found = readLessons(cwd).find((lesson) => lesson.id === id)
+    if (found === undefined || found.state === 'failed') return false
+    appendLesson(cwd, { ...found, state: 'failed' })
+    return true
+}
+
+/**
+ * A lesson about a situation that another lesson already covers replaces it. The old one is
+ * kept, marked superseded, because it is the record of what the loop believed before, and it
+ * stops matching, because a later session has to read one rule for a situation, not two.
+ */
+export function recordLesson(cwd: string, lesson: Lesson) {
+    const previous =
+        readLessons(cwd).find(
+            (entry) => entry.state !== 'rejected' && entry.state !== 'superseded' && sameCue(entry.cue, lesson.cue)
+        ) ?? null
+    if (previous !== null) appendLesson(cwd, { ...previous, state: 'superseded' })
+    const next = previous === null ? lesson : { ...lesson, supersedes: previous.id }
+    appendLesson(cwd, next)
+    return { lesson: next, superseded: previous }
+}
+
+export function writeLessons(cwd: string, lessons: Lesson[]): void {
+    const path = lessonsFile(cwd)
+    mkdirSync(dirname(path), { recursive: true })
+    writeFileAtomic(path, lessons.map((lesson) => `${JSON.stringify(lesson)}\n`).join(''))
+}
+
+const DEFAULT_SETTINGS: HarnessSettings = {
+    check: '',
+    auto: true,
+    autoInstall: true,
+    global: false,
+    maxInstallsPerDay: 3,
+    maxProposalsPerDay: 5,
+    maxTurnsPerProposal: 6,
+    maxStandingTokens: 1_500,
+    writerModel: ''
+}
+
+export function readSettings(cwd: string): HarnessSettings {
+    const path = settingsFile(cwd)
+    if (!existsSync(path)) return { ...DEFAULT_SETTINGS }
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'))
+    } catch {
+        return { ...DEFAULT_SETTINGS }
+    }
+    if (!isSettingsSchema(parsed)) return { ...DEFAULT_SETTINGS }
+    return {
+        check: parsed.check ?? DEFAULT_SETTINGS.check,
+        auto: parsed.auto ?? DEFAULT_SETTINGS.auto,
+        autoInstall: parsed.autoInstall ?? DEFAULT_SETTINGS.autoInstall,
+        global: parsed.global ?? DEFAULT_SETTINGS.global,
+        maxInstallsPerDay: Math.max(0, parsed.maxInstallsPerDay ?? DEFAULT_SETTINGS.maxInstallsPerDay),
+        maxProposalsPerDay: Math.max(0, parsed.maxProposalsPerDay ?? DEFAULT_SETTINGS.maxProposalsPerDay),
+        maxTurnsPerProposal: Math.max(1, parsed.maxTurnsPerProposal ?? DEFAULT_SETTINGS.maxTurnsPerProposal),
+        maxStandingTokens: Math.max(0, parsed.maxStandingTokens ?? DEFAULT_SETTINGS.maxStandingTokens),
+        writerModel: parsed.writerModel ?? DEFAULT_SETTINGS.writerModel
+    }
+}
+
+export function readInstalls(cwd: string): Install[] {
+    const path = installsFile(cwd)
+    if (!existsSync(path)) return []
+    const installs = new Map<string, Install>()
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+        if (line.trim() === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(line)
+        } catch {
+            continue
+        }
+        if (isInstallSchema(parsed)) installs.set(parsed.id, parsed)
+    }
+    return [...installs.values()].toSorted((a, b) => a.at - b.at)
+}
+
+export function appendInstall(cwd: string, install: Install): void {
+    const path = installsFile(cwd)
+    mkdirSync(dirname(path), { recursive: true })
+    appendFileSync(path, `${JSON.stringify(install)}\n`, 'utf8')
+}
+
+const LOCK_STALE_MS = 5 * 60_000
+
+export function acquireLock(cwd: string): (() => void) | null {
+    const path = join(harnessDir(cwd), 'lock')
+    mkdirSync(dirname(path), { recursive: true })
+    const token = `${process.pid} ${Date.now()} ${randomUUID()}\n`
+    const take = (): boolean => {
+        try {
+            writeFileSync(path, token, { encoding: 'utf8', flag: 'wx' })
+            return true
+        } catch {
+            return false
+        }
+    }
+    if (!take()) {
+        try {
+            const existing = readFileSync(path, 'utf8')
+            const at = Number(existing.split(' ')[1])
+            if (Number.isFinite(at) && Date.now() - at > LOCK_STALE_MS) rmSync(path, { force: true })
+        } catch {}
+        if (!take()) return null
+    }
+    return () => {
+        try {
+            if (readFileSync(path, 'utf8') === token) rmSync(path, { force: true })
+        } catch {}
+    }
+}
+
+export function recordLessonUse(cwd: string, id: string): void {
+    const found = readLessons(cwd).find((lesson) => lesson.id === id)
+    if (found === undefined) return
+    appendLesson(cwd, { ...found, uses: (found.uses ?? 0) + 1, lastUsed: Date.now() })
+}

package/src/target.ts

@@ -0,0 +1,76 @@
+import type { Recurrence, SkillUse } from './corpus.js'
+import type { Finding } from './findings.js'
+import type { FindingKind } from './types.js'
+
+export interface Target {
+    id: string
+    title: string
+    count: number
+    evidence: string[]
+    sample: string
+    command?: string
+    tool: string
+    scope: string
+    kind: FindingKind
+    cue: string
+}
+
+export function targetFromFinding(finding: Finding): Target {
+    const kind: FindingKind =
+        finding.kind === 'command'
+            ? 'repeat-command'
+            : finding.kind === 'read'
+              ? 'repeat-read'
+              : finding.kind === 'no-change'
+                ? 'no-change'
+                : finding.kind === 'unanswered'
+                  ? 'abandoned-tool'
+                  : 'repeat-error'
+    return {
+        id: finding.id,
+        title: finding.title,
+        count: finding.count,
+        evidence: finding.evidence,
+        sample: finding.evidence[0] ?? '',
+        tool: '',
+        scope: 'this session',
+        kind,
+        cue: finding.title
+    }
+}
+
+export function targetFromSkillUse(
+    use: SkillUse,
+    description: string,
+    path: string
+): Target {
+    const evidence = [`${path} :: ${description}`]
+    return {
+        id: use.id,
+        title: `the skill ${use.name} is advertised in every request and was opened in none of ${use.sessions} sessions`,
+        count: use.sessions,
+        evidence,
+        sample: '',
+        tool: '',
+        scope: `${use.sessions} sessions of this project`,
+        kind: 'dead-skill',
+        cue: `the skill ${use.name} is never opened`
+    }
+}
+
+export function targetFromRecurrence(recurrence: Recurrence): Target {
+    const kind: FindingKind =
+        recurrence.kind === 'command' ? 'repeat-command' : recurrence.kind === 'read' ? 'repeat-read' : 'repeat-error'
+    return {
+        id: recurrence.id,
+        title: recurrence.title,
+        count: recurrence.occurrences,
+        evidence: recurrence.evidence,
+        sample: recurrence.sample,
+        command: recurrence.command,
+        tool: recurrence.tool,
+        scope: `${recurrence.sessions} sessions of this project`,
+        kind,
+        cue: recurrence.title
+    }
+}

package/src/transcript.ts

@@ -0,0 +1,139 @@
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+import type { SessionEntry } from '@earendil-works/pi-coding-agent'
+const isString = new Validator(t.string()).isValidObject
+
+export interface ToolCall {
+    id: string
+    name: string
+    command: string
+    path: string
+}
+
+export interface ToolResult {
+    toolCallId: string
+    toolName: string
+    isError: boolean
+    body: string
+}
+
+export interface Transcript {
+    calls: ToolCall[]
+    results: ToolResult[]
+}
+
+const CommandArguments = t.object({ command: t.string() })
+const isCommandArguments = new Validator(CommandArguments).isValidObject
+const PathArguments = t.object({ path: t.string() })
+const isPathArguments = new Validator(PathArguments).isValidObject
+
+export function readTranscript(entries: SessionEntry[]): Transcript {
+    const calls: ToolCall[] = []
+    const results: ToolResult[] = []
+    for (const entry of entries) {
+        if (entry.type !== 'message') continue
+        const message = entry.message
+        if (message.role === 'assistant') {
+            for (const block of message.content) {
+                if (block.type !== 'toolCall') continue
+                calls.push({
+                    id: block.id,
+                    name: block.name,
+                    command: isCommandArguments(block.arguments) ? block.arguments.command : '',
+                    path: isPathArguments(block.arguments) ? block.arguments.path : ''
+                })
+            }
+            continue
+        }
+        if (message.role === 'toolResult') {
+            results.push({
+                toolCallId: message.toolCallId,
+                toolName: message.toolName,
+                isError: message.isError,
+                body: message.content
+                    .filter((block) => block.type === 'text')
+                    .map((block) => block.text)
+                    .join('\n')
+            })
+        }
+    }
+    return { calls, results }
+}
+
+export interface Usage {
+    input: number
+    output: number
+    cost: number
+}
+
+export function sumUsage(entries: SessionEntry[]): Usage {
+    let input = 0
+    let output = 0
+    let cost = 0
+    for (const entry of entries) {
+        if (entry.type !== 'message' || entry.message.role !== 'assistant') continue
+        const usage = entry.message.usage
+        input += usage.input
+        output += usage.output
+        cost += usage.cost.total
+    }
+    return { input, output, cost }
+}
+
+export function countAssistantTurns(entries: SessionEntry[]): number {
+    return entries.filter((entry) => entry.type === 'message' && entry.message.role === 'assistant').length
+}
+
+export function lastAssistantText(entries: SessionEntry[]): string {
+    for (const entry of entries.toReversed()) {
+        if (entry.type !== 'message' || entry.message.role !== 'assistant') continue
+        return entry.message.content
+            .filter((block) => block.type === 'text')
+            .map((block) => block.text)
+            .join('\n')
+    }
+    return ''
+}
+
+export function lastUserText(entries: SessionEntry[]): string {
+    for (const entry of entries.toReversed()) {
+        if (entry.type !== 'message' || entry.message.role !== 'user') continue
+        const content = entry.message.content
+        if (isString(content)) return content
+        return content
+            .filter((block) => block.type === 'text')
+            .map((block) => block.text)
+            .join('\n')
+    }
+    return ''
+}
+
+export function recentText(entries: SessionEntry[], limit = 6): string {
+    const texts: string[] = []
+    for (const entry of entries.toReversed().slice(0, limit)) {
+        if (entry.type !== 'message') continue
+        const message = entry.message
+        if (message.role === 'assistant') {
+            texts.push(
+                message.content
+                    .filter((block) => block.type === 'text')
+                    .map((block) => block.text)
+                    .join('\n')
+            )
+            continue
+        }
+        if (message.role !== 'user') continue
+        const content = message.content
+        texts.push(
+            isString(content)
+                ? content
+                : content
+                      .filter((block) => block.type === 'text')
+                      .map((block) => block.text)
+                      .join('\n')
+        )
+    }
+    return texts.join('\n')
+}