@tduyng/pi-harness-loop 0.0.0-stage → 0.0.2

package/src/tree.ts

@@ -0,0 +1,307 @@
+import { existsSync, readFileSync, readdirSync, type Dirent } from 'node:fs'
+import { homedir } from 'node:os'
+import { basename, dirname, join, resolve } from 'node:path'
+
+import { formatSkillsForPrompt, loadSkills, type Skill } from '@earendil-works/pi-coding-agent'
+
+export interface AgentProfile {
+    name: string
+    description: string
+    file: string
+    body: string
+}
+
+export interface Problem {
+    entry: string
+    file: string
+    rule: string
+    detail: string
+}
+
+export interface Harness {
+    skills: Skill[]
+    agents: AgentProfile[]
+    problems: Problem[]
+    skillBlockChars: number
+}
+
+interface Frontmatter {
+    fields: Record<string, string>
+    body: string
+}
+
+const RUN_COMMAND = /(?:python3?|bash|sh|node)\s+([^\s`'"]+)/g
+
+function skillDirs(cwd: string, agentDir: string): string[] {
+    const dirs = [join(agentDir, 'skills'), join(cwd, '.pi', 'skills')]
+    let dir = resolve(cwd)
+    while (true) {
+        dirs.push(join(dir, '.agents', 'skills'))
+        const parent = join(dir, '..')
+        if (parent === dir || existsSync(join(dir, '.git'))) break
+        dir = parent
+    }
+    dirs.push(join(homedir(), '.agents', 'skills'))
+    return dirs.filter((candidate) => existsSync(candidate))
+}
+
+function agentDirs(cwd: string, agentDir: string): string[] {
+    return [join(agentDir, 'agents'), join(cwd, '.pi', 'agents')].filter((dir) => existsSync(dir))
+}
+
+function frontmatter(raw: string): Frontmatter {
+    const match = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/.exec(raw)
+    if (!match?.[1]) return { fields: {}, body: raw }
+    const fields: Record<string, string> = {}
+    for (const line of match[1].split('\n')) {
+        const entry = /^([A-Za-z_][\w-]*)\s*:\s*(.*)$/.exec(line)
+        if (entry?.[1] !== undefined) fields[entry[1]] = (entry[2] ?? '').trim().replace(/^["']|["']$/g, '')
+    }
+    return { fields, body: raw.slice(match[0].length) }
+}
+
+export function loadAgents(cwd: string, agentDir: string): AgentProfile[] {
+    const agents: AgentProfile[] = []
+    for (const dir of agentDirs(cwd, agentDir)) {
+        let entries: string[]
+        try {
+            entries = readdirSync(dir)
+        } catch {
+            continue
+        }
+        for (const entry of entries.toSorted()) {
+            if (!entry.endsWith('.md')) continue
+            const file = join(dir, entry)
+            let raw: string
+            try {
+                raw = readFileSync(file, 'utf8')
+            } catch {
+                continue
+            }
+            const { fields, body } = frontmatter(raw)
+            agents.push({
+                name: fields['name'] || basename(entry, '.md'),
+                description: fields['description'] ?? '',
+                file,
+                body: body.trim()
+            })
+        }
+    }
+    return agents
+}
+
+function resolveHelper(target: string, skillDir: string, searchDirs: string[]): string | null {
+    const candidates: string[] = []
+    if (target.startsWith('~/')) candidates.push(join(homedir(), target.slice(2)))
+    else if (target.startsWith('/')) candidates.push(target)
+    else {
+        candidates.push(resolve(skillDir, target))
+        for (const dir of searchDirs) candidates.push(resolve(dir, target))
+    }
+    return candidates.find((candidate) => existsSync(candidate)) ?? null
+}
+
+export interface SkillFiles {
+    filePath: string
+    baseDir: string
+}
+
+export function missingHelpers(skill: SkillFiles, searchDirs: string[]): string[] {
+    let body: string
+    try {
+        body = readFileSync(skill.filePath, 'utf8')
+    } catch {
+        return []
+    }
+    const missing = new Set<string>()
+    for (const match of body.matchAll(RUN_COMMAND)) {
+        const target = match[1]
+        if (!target?.includes('scripts/')) continue
+        if (resolveHelper(target, skill.baseDir, searchDirs) === null) missing.add(target)
+    }
+    return [...missing]
+}
+
+function findSkillFiles(dir: string): string[] {
+    const files: string[] = []
+    let entries: Dirent[]
+    try {
+        entries = readdirSync(dir, { withFileTypes: true })
+    } catch {
+        return files
+    }
+    for (const entry of entries.toSorted((a, b) => a.name.localeCompare(b.name))) {
+        const full = join(dir, entry.name)
+        if (entry.isFile() && entry.name === 'SKILL.md') {
+            files.push(full)
+            continue
+        }
+        if (entry.isDirectory()) files.push(...findSkillFiles(full))
+    }
+    return files
+}
+
+export function validateProposal(dir: string, cwd: string): Problem[] {
+    const problems: Problem[] = []
+    for (const file of findSkillFiles(dir)) {
+        const raw = readFileSync(file, 'utf8')
+        const { fields } = frontmatter(raw)
+        const name = fields['name'] ?? ''
+        const description = fields['description'] ?? ''
+        if (description.trim() === '') {
+            problems.push({
+                entry: name,
+                file,
+                rule: 'no-description',
+                detail: 'pi drops a skill with no description at load, so it never reaches the model'
+            })
+        }
+        if (name !== '' && !/^[a-z0-9-]+$/.test(name)) {
+            problems.push({
+                entry: name,
+                file,
+                rule: 'bad-name',
+                detail: 'a skill name is lowercase letters, digits and hyphens'
+            })
+        }
+        for (const helper of missingHelpers({ filePath: file, baseDir: dirname(file) }, [cwd])) {
+            problems.push({
+                entry: name,
+                file,
+                rule: 'missing-helper',
+                detail: `the skill tells the agent to run ${helper}, and no such file exists`
+            })
+        }
+    }
+    for (const file of findRuleFiles(dir)) problems.push(...ruleProblems(file))
+    return problems
+}
+
+const RULE_MAX_LINES = 30
+const CREDENTIAL = /\b(api[_-]?key|access[_-]?token|secret|password|passwd|bearer)\b\s*[:=]\s*\S+/i
+
+/**
+ * A rule lands in a file every request of every project reads, so the project check cannot
+ * fail for it and the lint is the only gate it passes.
+ */
+function ruleProblems(file: string): Problem[] {
+    const problems: Problem[] = []
+    let raw: string
+    try {
+        raw = readFileSync(file, 'utf8')
+    } catch {
+        return problems
+    }
+    const body = raw.trim()
+    if (!/^##\s+\S/m.test(body)) {
+        problems.push({
+            entry: 'RULE',
+            file,
+            rule: 'no-heading',
+            detail: 'a rule is addressed by its heading, and without one the loop cannot replace it later'
+        })
+    }
+    const lines = body.split('\n').length
+    if (lines > RULE_MAX_LINES) {
+        problems.push({
+            entry: 'RULE',
+            file,
+            rule: 'too-long',
+            detail: `the rule is ${lines} lines and every request of every project pays for it`
+        })
+    }
+    if (CREDENTIAL.test(body)) {
+        problems.push({
+            entry: 'RULE',
+            file,
+            rule: 'credential',
+            detail: 'the rule carries a credential and lands in a file every session reads'
+        })
+    }
+    return problems
+}
+
+function findRuleFiles(dir: string): string[] {
+    const files: string[] = []
+    for (const path of [join(dir, 'RULE.md'), join(dir, 'global', 'RULE.md')]) {
+        if (existsSync(path)) files.push(path)
+    }
+    return files
+}
+
+export function loadHarness(cwd: string, agentDir: string): Harness {
+    const { skills, diagnostics } = loadSkills({
+        agentDir,
+        skillPaths: skillDirs(cwd, agentDir),
+        includeDefaults: true,
+        cwd
+    })
+    const problems: Problem[] = []
+    for (const diagnostic of diagnostics) {
+        problems.push({
+            entry: diagnostic.path ?? '',
+            file: diagnostic.path ?? '',
+            rule: diagnostic.type,
+            detail: diagnostic.message
+        })
+    }
+
+    const searchDirs = [cwd, agentDir]
+    const names = new Map<string, string>()
+    for (const skill of skills) {
+        const dir = basename(skill.baseDir)
+        if (skill.name !== dir) {
+            problems.push({
+                entry: skill.name,
+                file: skill.filePath,
+                rule: 'name-differs-from-directory',
+                detail: `pi advertises the frontmatter name "${skill.name}" and the folder is "${dir}", so a prompt that names the folder does not name the skill`
+            })
+        }
+        const previous = names.get(skill.name)
+        if (previous !== undefined) {
+            problems.push({
+                entry: skill.name,
+                file: skill.filePath,
+                rule: 'duplicate-name',
+                detail: `pi keeps the first skill named "${skill.name}" (${previous}) and drops this one`
+            })
+        } else {
+            names.set(skill.name, skill.filePath)
+        }
+        for (const helper of missingHelpers(skill, searchDirs)) {
+            problems.push({
+                entry: skill.name,
+                file: skill.filePath,
+                rule: 'missing-helper',
+                detail: `the skill tells the agent to run ${helper}, and no such file exists`
+            })
+        }
+    }
+
+    const agents = loadAgents(cwd, agentDir)
+    const agentNames = new Map<string, string>()
+    for (const agent of agents) {
+        if (agent.description.trim() === '') {
+            problems.push({
+                entry: agent.name,
+                file: agent.file,
+                rule: 'no-description',
+                detail: 'a profile with no description is listed as "no description", so the model cannot choose it'
+            })
+        }
+        const previous = agentNames.get(agent.name)
+        if (previous !== undefined) {
+            problems.push({
+                entry: agent.name,
+                file: agent.file,
+                rule: 'duplicate-name',
+                detail: `this profile replaces the earlier one named "${agent.name}" (${previous})`
+            })
+        } else {
+            agentNames.set(agent.name, agent.file)
+        }
+    }
+
+    return { skills, agents, problems, skillBlockChars: formatSkillsForPrompt(skills).length }
+}

package/src/types.ts

@@ -0,0 +1,77 @@
+export type Confidence = 'high' | 'medium' | 'low'
+
+export type FindingKind =
+    | 'repeat-error'
+    | 'repeat-command'
+    | 'repeat-read'
+    | 'no-change'
+    | 'abandoned-tool'
+    | 'dead-skill'
+
+export type LessonState = 'recorded' | 'installed' | 'failed' | 'rejected' | 'superseded'
+
+export type InstallState = 'recorded' | 'installed' | 'rejected'
+
+export interface Lesson {
+    id: string
+    at: number
+    cue: string
+    lesson: string
+    evidence: string
+    confidence: Confidence
+    triggers: string[]
+    kind: FindingKind
+    state: LessonState
+    confirm?: string
+    supersedes?: string
+    uses?: number
+    lastUsed?: number
+}
+
+export interface HarnessSettings {
+    check: string
+    /** Whether the loop starts a proposal on its own. The manual command works either way. */
+    auto: boolean
+    autoInstall: boolean
+    /** Whether a part may reach outside the project, into the skills or the context every project reads. */
+    global: boolean
+    maxInstallsPerDay: number
+    maxProposalsPerDay: number
+    maxTurnsPerProposal: number
+    /** Tokens of always-on prompt the active installs may add, all of them together. */
+    maxStandingTokens: number
+    /** `provider/model` for the writer, so a hard session does not pay its model for a short part. */
+    writerModel: string
+}
+
+export interface InstalledFile {
+    to: string
+    existed: boolean
+    before: string
+    after?: string
+    afterHash?: string
+    block?: string
+    mode: 'replace' | 'append'
+}
+
+export interface Install {
+    id: string
+    at: number
+    finding: string
+    dir: string
+    files: InstalledFile[]
+    check: string
+    ok: boolean
+    reverted: boolean
+    lesson?: string
+    key?: string
+    /** Characters the part added to the always-on prompt, which is what it costs on every later request. */
+    cost?: number
+    confirm?: string
+    confirmOk?: boolean
+    confirmOutput?: string
+    replay?: string
+    replayOk?: boolean
+    replayOutput?: string
+    note?: string
+}

package/src/writer.ts

@@ -0,0 +1,134 @@
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+export interface Completion {
+    text: string
+    stop: string
+    input: number
+    output: number
+    cost: number
+}
+
+export type Complete = (systemPrompt: string, prompt: string) => Promise<Completion>
+
+export interface WriteOutcome {
+    ok: boolean
+    message: string
+    files: string[]
+    turns: number
+    input: number
+    output: number
+    cost: number
+}
+
+export interface PartFile {
+    path: string
+    content: string
+}
+
+const FILE_BLOCK = /<file path="([^"]+)">\r?\n([\s\S]*?)<\/file>/g
+const MAX_PART_CHARS = 40_000
+const MAX_FILE_CHARS = 20_000
+const ATTEMPTS = 2
+
+const REPLY = [
+    'Reply with one block per file, and nothing outside them:',
+    '',
+    '<file path="RULE.md">',
+    '## Heading',
+    'the rule',
+    '</file>',
+    '',
+    'A path is relative to the proposal directory: RULE.md, CONFIRM.md, <name>/SKILL.md,',
+    '<name>.md for a profile, <name>/scripts/<file> for a helper, and the same under global/ for',
+    'every project. Reply KEEP_BASE alone when the evidence does not argue for a change.'
+].join('\n')
+
+function nameFile(name: string): boolean {
+    return /^[a-z0-9][a-z0-9._-]*$/.test(name)
+}
+
+/** The paths a part may write, so a reply cannot reach outside its proposal directory. */
+export function proposalPath(raw: string): string | null {
+    const path = raw.trim().replaceAll('\\', '/')
+    if (path === '' || path.startsWith('/') || path.includes('..') || path.includes(' ')) return null
+    const parts = path.split('/').filter((part) => part !== '')
+    const tail = parts[0] === 'global' ? parts.slice(1) : parts
+    if (tail.length === 1) {
+        const single = tail[0] ?? ''
+        if (single === 'RULE.md' || single === 'CONFIRM.md') return path
+        return single.endsWith('.md') && nameFile(single.slice(0, -3)) ? path : null
+    }
+    if (tail.length === 2 && (tail[1] === 'SKILL.md' || tail[1] === 'RULE.md')) {
+        return nameFile(tail[0] ?? '') ? path : null
+    }
+    if (tail.length === 3 && tail[1] === 'scripts') {
+        return nameFile(tail[0] ?? '') && nameFile(tail[2] ?? '') ? path : null
+    }
+    return null
+}
+
+export function parsePart(reply: string): PartFile[] {
+    const files: PartFile[] = []
+    let total = 0
+    for (const match of reply.matchAll(FILE_BLOCK)) {
+        const path = proposalPath(match[1] ?? '')
+        const body = (match[2] ?? '').trimEnd()
+        if (path === null || body.trim() === '' || body.length > MAX_FILE_CHARS) continue
+        total += body.length
+        if (total > MAX_PART_CHARS) return []
+        if (files.some((file) => file.path === path)) continue
+        files.push({ path, content: `${body}\n` })
+    }
+    return files
+}
+
+export function keptBase(reply: string): boolean {
+    return /^\s*KEEP_BASE\s*[.!]?\s*$/m.test(reply)
+}
+
+/**
+ * One completion writes the part, because a proposal that waits for a person to read a brief
+ * is a proposal that never lands. A reply is parsed into files, never executed.
+ */
+export async function writePart(dir: string, brief: string, complete: Complete): Promise<WriteOutcome> {
+    const system = `${brief}\n\n# HOW TO REPLY\n\n${REPLY}`
+    let problem = ''
+    let input = 0
+    let output = 0
+    let cost = 0
+    for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {
+        const prompt =
+            problem === ''
+                ? 'Write the part now.'
+                : `The last reply was rejected: ${problem}. Reply again with the file blocks.`
+        const reply = await complete(system, prompt)
+        input += reply.input
+        output += reply.output
+        cost += reply.cost
+        if (keptBase(reply.text)) {
+            return { ok: false, message: 'the writer kept the base', files: [], turns: attempt, input, output, cost }
+        }
+        const files = parsePart(reply.text)
+        if (files.length === 0) {
+            writeFileSync(join(dir, 'REPLY.txt'), `[${reply.stop}]\n${reply.text}`, 'utf8')
+            problem = 'the reply held no <file> block with an allowed path'
+            continue
+        }
+        for (const file of files) {
+            const to = join(dir, file.path)
+            mkdirSync(dirname(to), { recursive: true })
+            writeFileSync(to, file.content, 'utf8')
+        }
+        return {
+            ok: true,
+            message: `wrote ${files.map((file) => file.path).join(', ')}`,
+            files: files.map((file) => file.path),
+            turns: attempt,
+            input,
+            output,
+            cost
+        }
+    }
+    return { ok: false, message: problem, files: [], turns: ATTEMPTS, input, output, cost }
+}