@tduyng/pi-harness-loop 0.0.0-stage → 0.0.1

package/src/install.ts

@@ -0,0 +1,297 @@
+import { homedir } from 'node:os'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+import { applyProposal, readProposal, revertApplied } from './apply.js'
+import { readScan } from './auto.js'
+import { discoverCheck, runCheck } from './check.js'
+import { runReplay } from './replay.js'
+import { acquireLock, appendInstall, readInstalls, readSettings } from './store.js'
+import { loadHarness } from './tree.js'
+import { validateProposal } from './tree.js'
+import type { Install, InstallState, InstalledFile } from './types.js'
+import type { InstallContext, OwnedFile } from './apply.js'
+
+export const DAY_MS = 86_400_000
+const CONFIRM_TIMEOUT_MS = 60_000
+
+export interface InstallOutcome {
+    state: InstallState
+    message: string
+    retryable?: boolean
+}
+
+export interface InstallOptions {
+    forced?: boolean
+    context?: InstallContext
+    lesson?: string
+    key?: string
+    command?: string
+}
+
+export function liveSkills(cwd: string, agentDir: string): Map<string, string> {
+    const out = new Map<string, string>()
+    for (const skill of loadHarness(cwd, agentDir).skills) {
+        out.set(skill.name, skill.filePath)
+    }
+    return out
+}
+
+export function installContext(cwd: string, agentDir: string): InstallContext {
+    return {
+        cwd,
+        agentDir,
+        homeSkills: join(homedir(), '.agents', 'skills'),
+        live: liveSkills(cwd, agentDir),
+        owned: ownedFiles(cwd)
+    }
+}
+
+/**
+ * What the part costs on every later request, which is the front of the prompt: the rule as it
+ * is appended, or the name and the description the model routes a skill or a profile on. The
+ * body of a skill is not in it, because a body is paid only when the model opens it.
+ */
+export function advertisedChars(files: InstalledFile[]): number {
+    let total = 0
+    for (const file of files) {
+        if (file.mode === 'append') {
+            total += (file.block ?? '').length
+            continue
+        }
+        if (!file.to.endsWith('.md')) continue
+        const match = /^---\r?\n([\s\S]*?)\r?\n---/.exec(file.after ?? '')
+        if (match?.[1] === undefined) continue
+        for (const field of ['name', 'description']) {
+            total += new RegExp(`^${field}\\s*:\\s*(.*)$`, 'm').exec(match[1])?.[1]?.length ?? 0
+        }
+    }
+    return total
+}
+
+/** Characters the active installs add to the front of every later request. */
+export function standingChars(installs: Install[]): number {
+    return installs
+        .filter((install) => install.ok && !install.reverted)
+        .reduce((total, install) => total + (install.cost ?? advertisedChars(install.files)), 0)
+}
+
+function ruleBlock(dir: string): string {
+    for (const path of [join(dir, 'RULE.md'), join(dir, 'global', 'RULE.md')]) {
+        try {
+            const text = readFileSync(path, 'utf8').trim()
+            if (text !== '') return text
+        } catch {
+            continue
+        }
+    }
+    return ''
+}
+
+export function ownedFiles(cwd: string): Map<string, OwnedFile> {
+    const owned = new Map<string, OwnedFile>()
+    for (const install of readInstalls(cwd)) {
+        if (!install.ok || install.reverted) continue
+        for (const file of install.files) {
+            const entry = owned.get(file.to) ?? { blocks: [] }
+            if (file.after !== undefined) entry.after = file.after
+            if (file.mode === 'append') {
+                const block = file.block ?? ruleBlock(install.dir)
+                if (block !== '' && !entry.blocks.includes(block)) entry.blocks.push(block)
+            }
+            owned.set(file.to, entry)
+        }
+    }
+    return owned
+}
+
+export function installRoots(context: InstallContext): string[] {
+    return [
+        context.homeSkills,
+        join(context.cwd, '.agents', 'skills'),
+        join(context.cwd, '.pi', 'agents'),
+        context.agentDir
+    ]
+}
+
+export function confirmOf(dir: string): string {
+    const path = join(dir, 'CONFIRM.md')
+    if (!existsSync(path)) return ''
+    const text = readFileSync(path, 'utf8')
+    const fenced = /```(?:sh|bash|shell)?\s*\n([\s\S]*?)```/.exec(text)
+    const body = (fenced?.[1] ?? text).trim()
+    return (
+        body
+            .split('\n')
+            .map((line) => line.trim())
+            .find((line) => line !== '' && !line.startsWith('#')) ?? ''
+    )
+}
+
+async function replayOf(cwd: string, finding: string, command?: string) {
+    const found = command ?? readScan(cwd)?.recurrences.find((recurrence) => recurrence.id === finding)?.command
+    if (found === undefined) return null
+    return runReplay(cwd, finding, found)
+}
+
+async function finalize(
+    cwd: string,
+    dir: string,
+    install: Install,
+    context: InstallContext,
+    refusal: string,
+    command?: string
+): Promise<InstallOutcome> {
+    const confirm = confirmOf(dir)
+    const confirmResult = confirm === '' ? null : await runCheck(cwd, confirm, { timeoutMs: CONFIRM_TIMEOUT_MS })
+    const base: Install = {
+        ...install,
+        confirm: confirm === '' ? undefined : confirm,
+        confirmOk: confirmResult?.ok,
+        confirmOutput: confirmResult?.output.slice(0, 2000)
+    }
+    if (confirmResult !== null && !confirmResult.ok) {
+        revertApplied(install.files, installRoots(context), true)
+        appendInstall(cwd, { ...base, ok: false, reverted: true })
+        return {
+            state: 'rejected',
+            message: `reverted, the confirm command failed: ${confirmResult.output.slice(0, 300)}`
+        }
+    }
+    const replay = await replayOf(cwd, install.finding, command)
+    const recorded: Install = {
+        ...base,
+        replay: replay?.command,
+        replayOk: replay?.ok,
+        replayOutput: replay?.output
+    }
+    appendInstall(cwd, recorded)
+    return {
+        state: 'installed',
+        message: `installed ${install.files.length} file(s), ${
+            install.check === '' ? 'no check command' : `${install.check} found no new failure`
+        }${confirmResult === null ? ', no confirm command' : ', confirm passed'}${refusal}`
+    }
+}
+
+export async function installProposal(
+    dir: string,
+    finding: string,
+    cwd: string,
+    options: InstallOptions = {}
+): Promise<InstallOutcome> {
+    const release = acquireLock(cwd)
+    if (release === null) {
+        return { state: 'recorded', message: 'another install is running, so this one waits', retryable: true }
+    }
+    try {
+        return await installLocked(dir, finding, cwd, options)
+    } finally {
+        release()
+    }
+}
+
+async function installLocked(
+    dir: string,
+    finding: string,
+    cwd: string,
+    options: InstallOptions
+): Promise<InstallOutcome> {
+    const settings = readSettings(cwd)
+    if (!settings.autoInstall && options.forced !== true) {
+        return { state: 'recorded', message: `autoInstall is off, the proposal stays in ${dir}` }
+    }
+    const check = discoverCheck(cwd, settings.check)
+    const proposal = readProposal(dir)
+    const markdownOnly = proposal.rule !== null && proposal.skills.length === 0 && proposal.profiles.length === 0
+    if (check === '' && !markdownOnly) {
+        return {
+            state: 'recorded',
+            message: `no check command, and a skill or profile needs one, so the proposal stays in ${dir}`
+        }
+    }
+    const spent = readInstalls(cwd).filter(
+        (install) => install.ok && !install.reverted && install.at > Date.now() - DAY_MS
+    )
+    if (options.forced !== true && spent.length >= settings.maxInstallsPerDay) {
+        return {
+            state: 'recorded',
+            message: `the install budget for today is spent (${spent.length})`,
+            retryable: true
+        }
+    }
+    const context = options.context ?? {
+        cwd,
+        agentDir: '',
+        homeSkills: join(homedir(), '.agents', 'skills'),
+        live: new Map(),
+        owned: new Map()
+    }
+    const problems = validateProposal(dir, cwd)
+    if (problems.length > 0) {
+        return {
+            state: 'recorded',
+            message: `the proposal breaks ${problems.length} rules, so it stays on disk: ${problems[0]?.rule ?? ''} ${problems[0]?.detail ?? ''}`
+        }
+    }
+    const { applied, refused } = applyProposal(dir, context)
+    if (applied.length === 0) {
+        return {
+            state: 'recorded',
+            message:
+                refused.length === 0
+                    ? 'the proposal holds no skill or profile to install'
+                    : `the part stays on disk: ${refused[0]}`
+        }
+    }
+    const added = advertisedChars(applied)
+    const standing = standingChars(readInstalls(cwd))
+    const capChars = settings.maxStandingTokens * 4
+    if (options.forced !== true && capChars > 0 && standing + added > capChars) {
+        revertApplied(applied, installRoots(context), true)
+        return {
+            state: 'recorded',
+            message: `the always-on block would reach ${Math.round((standing + added) / 4)} of ${settings.maxStandingTokens} tokens, so the part stays on disk; revert an install first`,
+            retryable: true
+        }
+    }
+    const refusal = refused.length === 0 ? '' : `, ${refused.length} file(s) left alone: ${refused[0]}`
+
+    const install: Install = {
+        id: `${finding}-${Date.now()}`,
+        at: Date.now(),
+        finding,
+        dir,
+        files: applied,
+        check,
+        ok: true,
+        reverted: false,
+        lesson: options.lesson,
+        key: options.key,
+        cost: advertisedChars(applied)
+    }
+    if (check === '') return finalize(cwd, dir, install, context, `${refusal}, installed without a check`, options.command)
+
+    const result = await runCheck(cwd, check)
+    if (result.ok) return finalize(cwd, dir, install, context, refusal, options.command)
+
+    revertApplied(applied, installRoots(context), true)
+    const baseline = await runCheck(cwd, check)
+    const innocent =
+        !baseline.ok && baseline.failures !== null && result.failures !== null && result.failures <= baseline.failures
+    if (innocent) {
+        for (const file of applied) {
+            mkdirSync(dirname(file.to), { recursive: true })
+            writeFileSync(file.to, file.after, 'utf8')
+        }
+        return finalize(cwd, dir, install, context, refusal, options.command)
+    }
+    appendInstall(cwd, { ...install, ok: false, reverted: true })
+    return {
+        state: 'rejected',
+        message:
+            baseline.failures === null || result.failures === null
+                ? `reverted, the check could not say whether the change broke anything: ${result.output.slice(0, 300)}`
+                : `reverted, ${check} went from ${baseline.failures} failures to ${result.failures}: ${result.output.slice(0, 300)}`
+    }
+}

package/src/ledger.ts

@@ -0,0 +1,89 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+import { Validator } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+export type SpendOutcome = 'installed' | 'reverted' | 'recorded' | 'rejected' | 'auto-reverted'
+
+export interface LoopSpend {
+    at: number
+    target: string
+    turns: number
+    input: number
+    output: number
+    cost: number
+    outcome: SpendOutcome
+}
+
+export interface SpendTotal {
+    proposals: number
+    turns: number
+    input: number
+    output: number
+    cost: number
+}
+
+const SpendSchema = t.object({
+    at: t.number(),
+    target: t.string(),
+    turns: t.number(),
+    input: t.number(),
+    output: t.number(),
+    cost: t.number(),
+    outcome: t.union([
+        t.literal('installed'),
+        t.literal('reverted'),
+        t.literal('recorded'),
+        t.literal('rejected'),
+        t.literal('auto-reverted')
+    ])
+})
+const isSpendSchema = new Validator(SpendSchema).isValidObject
+
+export function spendFile(cwd: string): string {
+    return join(cwd, '.pi', 'harness', 'spend.jsonl')
+}
+
+export function readSpend(cwd: string): LoopSpend[] {
+    const path = spendFile(cwd)
+    if (!existsSync(path)) return []
+    const out: LoopSpend[] = []
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+        if (line.trim() === '') continue
+        let parsed: unknown
+        try {
+            parsed = JSON.parse(line)
+        } catch {
+            continue
+        }
+        if (isSpendSchema(parsed)) out.push(parsed)
+    }
+    return out
+}
+
+export function appendSpend(cwd: string, spend: LoopSpend): void {
+    const path = spendFile(cwd)
+    mkdirSync(dirname(path), { recursive: true })
+    appendFileSync(path, `${JSON.stringify(spend)}\n`, 'utf8')
+}
+
+export function sumSpend(spends: LoopSpend[]): SpendTotal {
+    return spends.reduce<SpendTotal>(
+        (total, spend) => ({
+            proposals: total.proposals + 1,
+            turns: total.turns + spend.turns,
+            input: total.input + spend.input,
+            output: total.output + spend.output,
+            cost: total.cost + spend.cost
+        }),
+        { proposals: 0, turns: 0, input: 0, output: 0, cost: 0 }
+    )
+}
+
+export const DAY_MS = 86_400_000
+
+export function spentToday(spends: LoopSpend[], now = Date.now()): SpendTotal {
+    return sumSpend(spends.filter((spend) => spend.at > now - DAY_MS))
+}

package/src/lesson-tool.ts

@@ -0,0 +1,39 @@
+import { t } from 'ata-validator/t'
+
+import { matchLessons } from './match.js'
+import { readLessons } from './store.js'
+import type { PromptWords } from './corpus.js'
+import type { Lesson } from './types.js'
+
+export const LESSON_TOOL = 'lesson'
+export const MAX_LESSONS_SHOWN = 3
+
+export const LessonParameters = t.object({
+    action: t.union([t.literal('match'), t.literal('list'), t.literal('record'), t.literal('drop')]),
+    cue: t.optional(t.string()),
+    lesson: t.optional(t.string()),
+    confirm: t.optional(t.string()),
+    evidence: t.optional(t.string()),
+    confidence: t.optional(t.union([t.literal('high'), t.literal('medium'), t.literal('low')])),
+    id: t.optional(t.string())
+})
+
+export function matchedLessons(cwd: string, prompt: string, rarity?: PromptWords): Lesson[] {
+    return matchLessons(readLessons(cwd), prompt, rarity)
+}
+
+export function formatLesson(lesson: Lesson): string {
+    return [
+        `cue: ${lesson.cue}`,
+        `lesson: ${lesson.lesson}`,
+        ...(lesson.confirm === undefined ? [] : [`confirm: ${lesson.confirm}`]),
+        `evidence: ${lesson.evidence}`,
+        `confidence: ${lesson.confidence}`,
+        `state: ${lesson.state}`
+    ].join('\n')
+}
+
+export function formatLessons(lessons: Lesson[]): string {
+    if (lessons.length === 0) return 'no lesson matches this request'
+    return lessons.slice(0, MAX_LESSONS_SHOWN).map(formatLesson).join('\n\n')
+}

package/src/lint.ts

@@ -0,0 +1,87 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { Validator, type Infer } from 'ata-validator'
+import { t } from 'ata-validator/t'
+
+
+export interface LintProblem {
+    rule: string
+    entry: string
+    detail: string
+}
+
+const Manifest = t.object({
+    pi: t.object({ extensions: t.array(t.string()) })
+})
+const isManifest = new Validator(Manifest).isValidObject
+
+const PackageJson = t.object({
+    name: t.optional(t.string()),
+    pi: t.optional(t.object({ extensions: t.array(t.string()) }))
+})
+const isPackageJson = new Validator(PackageJson).isValidObject
+
+function readManifest(path: string): Infer<typeof Manifest> | null {
+    try {
+        const parsed: unknown = JSON.parse(readFileSync(path, 'utf8'))
+        return isManifest(parsed) ? parsed : null
+    } catch {
+        return null
+    }
+}
+
+function readPackage(path: string): Infer<typeof PackageJson> | null {
+    try {
+        const parsed: unknown = JSON.parse(readFileSync(path, 'utf8'))
+        return isPackageJson(parsed) ? parsed : null
+    } catch {
+        return null
+    }
+}
+
+export function lintManifest(root: string): LintProblem[] {
+    const problems: LintProblem[] = []
+    const manifest = readManifest(join(root, 'package.json'))
+    if (manifest === null) {
+        return [
+            { rule: 'no-manifest', entry: 'package.json', detail: 'the root package.json has no pi.extensions list' }
+        ]
+    }
+    const registered = new Set(manifest.pi.extensions)
+
+    for (const entry of manifest.pi.extensions) {
+        if (existsSync(join(root, entry))) continue
+        problems.push({
+            rule: 'missing-entry-file',
+            entry,
+            detail: 'the manifest loads this file and it is not on disk'
+        })
+    }
+
+    const extensionDir = join(root, 'extensions')
+    if (!existsSync(extensionDir)) return problems
+    for (const name of readdirSync(extensionDir).toSorted()) {
+        const packagePath = join(extensionDir, name, 'package.json')
+        if (!existsSync(packagePath)) continue
+        const pkg = readPackage(packagePath)
+        if (pkg === null) continue
+        for (const entry of pkg.pi?.extensions ?? []) {
+            const relative = `./extensions/${name}/${entry.replace(/^\.\//, '')}`
+            if (!existsSync(join(extensionDir, name, entry))) {
+                problems.push({
+                    rule: 'missing-package-entry',
+                    entry: relative,
+                    detail: 'the package declares this entry and it is not on disk'
+                })
+            }
+            if (registered.has(relative)) continue
+            problems.push({
+                rule: 'unregistered-extension',
+                entry: relative,
+                detail: 'the package declares this entry and the root manifest never loads it'
+            })
+        }
+    }
+    return problems
+}

package/src/match.ts

@@ -0,0 +1,192 @@
+import type { Lesson } from './types.js'
+import type { PromptWords } from './corpus.js'
+
+/** A trigger that shows up in more than this share of the prompts of the project says nothing. */
+const COMMON_RARITY = 0.02
+
+function commonIn(word: string, rarity?: PromptWords): boolean {
+    if (rarity === undefined || rarity.count === 0) return false
+    return (rarity.words.get(word) ?? 0) / rarity.count > COMMON_RARITY
+}
+
+const STOP = new Set([
+    'about',
+    'above',
+    'after',
+    'again',
+    'against',
+    'all',
+    'also',
+    'and',
+    'another',
+    'any',
+    'anyway',
+    'are',
+    'around',
+    'ask',
+    'asks',
+    'away',
+    'back',
+    'because',
+    'been',
+    'before',
+    'being',
+    'below',
+    'both',
+    'but',
+    'can',
+    'calls',
+    'could',
+    'did',
+    'does',
+    'doing',
+    'down',
+    'during',
+    'each',
+    'every',
+    'from',
+    'further',
+    'get',
+    'gets',
+    'had',
+    'has',
+    'have',
+    'here',
+    'how',
+    'into',
+    'its',
+    'just',
+    'least',
+    'less',
+    'like',
+    'made',
+    'make',
+    'makes',
+    'many',
+    'more',
+    'most',
+    'much',
+    'must',
+    'not',
+    'now',
+    'only',
+    'onto',
+    'other',
+    'our',
+    'out',
+    'over',
+    'own',
+    'per',
+    'put',
+    'same',
+    'see',
+    'set',
+    'she',
+    'should',
+    'some',
+    'such',
+    'than',
+    'that',
+    'the',
+    'their',
+    'them',
+    'then',
+    'there',
+    'these',
+    'they',
+    'this',
+    'those',
+    'through',
+    'too',
+    'two',
+    'under',
+    'until',
+    'used',
+    'uses',
+    'using',
+    'very',
+    'want',
+    'was',
+    'way',
+    'were',
+    'what',
+    'when',
+    'where',
+    'which',
+    'while',
+    'who',
+    'whom',
+    'whose',
+    'will',
+    'with',
+    'would',
+    'yes',
+    'yet',
+    'you',
+    'your'
+])
+
+export function triggersOf(text: string, limit = 8): string[] {
+    const words = text.toLowerCase().match(/[a-z][a-z0-9_-]{3,}/g) ?? []
+    const counts = new Map<string, number>()
+    for (const word of words) {
+        if (STOP.has(word)) continue
+        counts.set(word, (counts.get(word) ?? 0) + 1)
+    }
+    return [...counts.entries()]
+        .toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .slice(0, limit)
+        .map(([word]) => word)
+}
+
+const MIN_TRIGGER_PROMPTS = 2
+
+function plausible(word: string, count: number): boolean {
+    if (count >= MIN_TRIGGER_PROMPTS) return true
+    if (word.length < 8) return false
+    return !/[bcdfghjklmnpqrstvwxz]{4,}/.test(word)
+}
+
+/**
+ * The trigger words are the ones that say the situation, not the ones every
+ * request has. The corpus says which is which: a word the prompts never used
+ * cannot route to anything, so it is not a trigger, and a word most of them
+ * carry would put the tool in every request. A token seen once has to look like
+ * a word, because a typo is the rarest token in the corpus and routes nowhere.
+ */
+export function rareTriggersOf(cue: string, rarity: PromptWords, limit = 6): string[] {
+    const words = [
+        ...new Set((cue.toLowerCase().match(/[a-z][a-z0-9_-]{3,}/g) ?? []).filter((word) => !STOP.has(word)))
+    ]
+    const seen = words
+        .map((word) => ({ word, count: rarity.words.get(word) ?? 0 }))
+        .filter((entry) => entry.count > 0 && plausible(entry.word, entry.count))
+        .toSorted((a, b) => a.count - b.count || a.word.localeCompare(b.word))
+    const rare = seen.filter((entry) => entry.count <= Math.max(1, Math.floor(rarity.count * COMMON_RARITY)))
+    const kept = rare.length > 0 ? rare : seen.slice(0, 3)
+    return kept.slice(0, limit).map((entry) => entry.word)
+}
+
+function rareEnough(word: string, rarity?: PromptWords): boolean {
+    if (rarity === undefined || rarity.count === 0) return word.length >= 6
+    return (rarity.words.get(word) ?? 0) / rarity.count <= 0.005
+}
+
+/**
+ * One word is a coincidence: a request that says "key" is not a request about
+ * credentials. Two triggers, or one long rare one, is a situation.
+ */
+export function matchLessons(lessons: Lesson[], prompt: string, rarity?: PromptWords): Lesson[] {
+    const words = new Set(prompt.toLowerCase().match(/[a-z][a-z0-9_-]{2,}/g) ?? [])
+    const scored = lessons
+        .filter((lesson) => lesson.state !== 'superseded' && lesson.state !== 'rejected' && lesson.state !== 'failed')
+        .map((lesson) => {
+            const triggers = lesson.triggers.length > 0 ? lesson.triggers : triggersOf(lesson.cue)
+            const matched = triggers.filter((word) => words.has(word) && !commonIn(word, rarity))
+            const strong = matched.some((word) => word.length >= 6 && rareEnough(word, rarity))
+            return { lesson, hits: matched.length, strong }
+        })
+        .filter((entry) => entry.hits >= 2 || (entry.hits === 1 && entry.strong))
+        .toSorted((a, b) => b.hits - a.hits || b.lesson.at - a.lesson.at)
+    return scored.map((entry) => entry.lesson)
+}