@tduyng/pi-timbre 0.0.0-stage → 0.0.2

package/src/report.ts

@@ -0,0 +1,205 @@
+import type { Hit } from './scan.js'
+import type { SegmentResult } from './segments.js'
+import { cacheSummary, tallyIds, type Domain, type Effect, type Memory } from './memory.js'
+
+const CAP = 10
+
+export function formatHits(path: string, hits: Hit[]): string {
+    if (hits.length === 0) return ''
+
+    const shown = hits.slice(0, CAP)
+    const lines = shown.map((h) => `  ${h.line}: ${h.id}: ${h.hint} → ${truncate(h.text, 90)}`)
+    const more = hits.length > CAP ? `\n  ... ${hits.length - CAP} more` : ''
+
+    return [
+        `timbre found ${hits.length} tell${hits.length === 1 ? '' : 's'} in ${path}:`,
+        lines.join('\n') + more,
+        'Fix the ones in text you just wrote. Pre-existing tells elsewhere are not yours to fix.'
+    ].join('\n')
+}
+
+export type FixTarget = { path: string; register: string; hits: Hit[]; segments?: SegmentResult }
+
+export function formatFixRequest(targets: FixTarget[], instruction = '', fullRules = ''): string {
+    const total = targets.reduce((sum, t) => sum + t.hits.length, 0)
+    const blocks = targets.map((t) => {
+        const header = `${t.path} (register: ${t.register})`
+        const tells = groupById(t.hits)
+        if (t.segments === undefined) return [header, ...tells].join('\n')
+        const { flaggedFraction, verdict, spans } = t.segments
+        const pct = `${(flaggedFraction * 100).toFixed(1)}%`
+        const spanList =
+            spans.length === 0
+                ? 'no flagged spans'
+                : spans
+                      .map((s) =>
+                          s.startLine === s.endLine ? `  line ${s.startLine}` : `  lines ${s.startLine}-${s.endLine}`
+                      )
+                      .join('\n')
+        const advice =
+            verdict === 'clean'
+                ? 'Light touch only. The scan found scattered tells: fix them, leave the rest.'
+                : verdict === 'heavy'
+                  ? 'Most of this file reads machine-shaped. Restart from your own outline, keep the ideas, rewrite every paragraph.'
+                  : 'Some spans are machine-shaped, the rest reads human. Fix the flagged spans, leave the clean ones alone.'
+        return [
+            header,
+            `${pct} of this file's prose is flagged (${verdict}).`,
+            'Flagged spans:',
+            spanList,
+            ...tells,
+            advice
+        ].join('\n')
+    })
+    const found =
+        total === 0
+            ? 'found no mechanical tell, so this one is a judgment pass'
+            : `found ${total} tell${total === 1 ? '' : 's'}`
+
+    return [
+        `timbre scanned ${targets.length} file${targets.length === 1 ? '' : 's'} on request and ${found}.`,
+        blocks.join('\n\n'),
+        ...(instruction === '' ? [] : [`The user asked for this alongside the paths: ${instruction}`]),
+        `Rewrite these files so they read as human. ${REWRITE_PROCEDURE}`,
+        ...(fullRules === ''
+            ? []
+            : [`The complete constraint set for the rewrite (every rule, not the short version):\n\n${fullRules}`])
+    ].join('\n\n')
+}
+
+export function formatFindRequest(instruction: string, fullRules = ''): string {
+    return [
+        'timbre: a prose fix was requested without a filename the command could resolve.',
+        `The user asked for this: ${instruction}`,
+        `Work out which files they mean from what was said earlier in this conversation, and ask only if it is still ambiguous after looking. Then rewrite them so they read as human. ${REWRITE_PROCEDURE}`,
+        ...(fullRules === ''
+            ? []
+            : [`The complete constraint set for the rewrite (every rule, not the short version):\n\n${fullRules}`])
+    ].join('\n\n')
+}
+
+const REWRITE_PROCEDURE =
+    'Call timbre_review_guide first and work in the order it gives: triage every hit before fixing any, leave the ones the text quotes or is discussing, then read once for what no scanner sees. If the outline came from the model, restart from your own outline, paraphrase cannot save it. Reordering your own sentences is always safe; the test is clause by clause, whose idea each carries, and a borrowed idea in swapped words must go. Edit in place. Removing the tells is half the job: the rewrite also has to leave a voice behind, read naturally, and stay easy to follow, with short paragraphs, one idea per sentence, and concrete specifics over abstractions. It must not drop a claim, condition, or instruction the original carried. There is no timbre binary to run; `timbre_review_guide` is the only timbre tool.'
+
+const EXCERPTS = 2
+
+function groupById(hits: Hit[], excerpt = false): string[] {
+    const groups = new Map<string, { hint: string; lines: number[]; texts: string[] }>()
+    for (const hit of hits) {
+        const group = groups.get(hit.id) ?? { hint: hit.hint, lines: [], texts: [] }
+        if (hit.line > 0) group.lines.push(hit.line)
+        if (hit.text !== '') group.texts.push(`${hit.line}: ${truncate(hit.text, 90)}`)
+        groups.set(hit.id, group)
+    }
+    return [...groups.entries()]
+        .toSorted(([aId, a], [bId, b]) => b.lines.length - a.lines.length || aId.localeCompare(bId))
+        .map(([id, { hint, lines, texts }]) => {
+            const where = lines.length === 0 ? 'whole file' : `line${lines.length === 1 ? '' : 's'} ${lines.join(', ')}`
+            const quoted = excerpt
+                ? texts
+                      .slice(0, EXCERPTS)
+                      .map((t) => `\n    ${t}`)
+                      .join('')
+                : ''
+            return `  ${id} (${where}): ${hint}${quoted}`
+        })
+}
+
+function groupByRule(hits: Hit[]): string[] {
+    const order: string[] = []
+    const grouped = new Map<string, { lines: (number | '?')[]; hint: string }>()
+    for (const hit of hits) {
+        let row = grouped.get(hit.id)
+        if (row === undefined) {
+            row = { lines: [], hint: hit.hint }
+            grouped.set(hit.id, row)
+            order.push(hit.id)
+        }
+        row.lines.push(hit.line || '?')
+    }
+    return order.map((id) => {
+        const row = grouped.get(id)
+        return `  ${row?.lines.join(', ')}: ${id}: ${row?.hint ?? ''}`
+    })
+}
+
+export function formatCodeFindings(hits: Array<Hit & { path: string }>): string {
+    if (hits.length === 0) return ''
+    const byPath = new Map<string, typeof hits>()
+    for (const hit of hits) {
+        const row = byPath.get(hit.path)
+        if (row === undefined) byPath.set(hit.path, [hit])
+        else row.push(hit)
+    }
+    const blocks = [...byPath].map(([path, rows]) => {
+        const shown = rows.slice(0, CAP)
+        const more = rows.length > CAP ? `\n  ... ${rows.length - CAP} more` : ''
+        const noun = rows.length === 1 ? 'code tell' : 'code tells'
+        return [`timbre found ${rows.length} ${noun} in ${path}:`, groupByRule(shown).join('\n') + more].join('\n')
+    })
+    blocks.push(
+        `The message is the rule. Prefer inference, \`as const\`, \`satisfies\`, named domain types, and boundary parsing over widening then asserting. The agreements are in AGENTS.md, section Code.`
+    )
+    return blocks.join('\n\n')
+}
+
+export function formatStats(memory: Memory, effects: Array<[Domain, Effect]>, preventionOn: boolean): string {
+    const out: string[] = [`timbre: prevention ${preventionOn ? 'on' : 'off'}`]
+
+    const totalTells = memory.samples.reduce((a, s) => a + s.hits, 0)
+    if (memory.samples.length === 0) {
+        out.push('No samples recorded yet. Write some prose or code and check back.')
+        return out.join('\n')
+    }
+
+    out.push(`${memory.samples.length} samples, ${totalTells} tells recorded.`)
+
+    for (const [domain, effect] of effects) {
+        const { withPrevention: on, withoutPrevention: off, delta } = effect
+        if (on.samples === 0 && off.samples === 0) continue
+
+        out.push('')
+        out.push(`  ${domain}`)
+        out.push(`    prevention off: ${fmt(off.density)} tells/1k chars over ${off.samples} samples (inferred)`)
+        out.push(`    prevention on:  ${fmt(on.density)} tells/1k chars over ${on.samples} samples (inferred)`)
+
+        if (delta === null) {
+            out.push('    Need samples on both sides to compare. `/timbre off` builds a baseline.')
+        } else {
+            const verdict = delta > 0 ? `${fmt(delta)}% fewer tells` : `${fmt(Math.abs(delta))}% more tells`
+            out.push(`    Effect: ${verdict} with prevention on.`)
+        }
+    }
+
+    const cache = cacheSummary(memory, true)
+    if (cache !== null) {
+        out.push('')
+        out.push(
+            `  cache (prevention on): avg ${Math.round(cache.avgCacheRead)} read / ${Math.round(cache.avgCacheWrite)} write tokens/turn over ${cache.turns} turns`
+        )
+        if (cache.avgCacheWrite > cache.avgCacheRead) {
+            out.push('    High write relative to read: the block is not staying cached turn to turn.')
+        }
+    }
+
+    const top = [...tallyIds(memory.samples)]
+        .toSorted(([aId, a], [bId, b]) => b - a || aId.localeCompare(bId))
+        .slice(0, 5)
+    if (top.length > 0) {
+        out.push('')
+        out.push('  Most frequent:')
+        for (const [id, n] of top) {
+            out.push(`    ${n}x ${id}`)
+        }
+    }
+
+    return out.join('\n')
+}
+
+function fmt(n: number): string {
+    return n.toFixed(1)
+}
+
+function truncate(s: string, max: number): string {
+    return s.length <= max ? s : `${s.slice(0, max - 1)}…`
+}

package/src/rules.ts

@@ -0,0 +1,37 @@
+import { readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const GUIDANCE = join(dirname(fileURLToPath(import.meta.url)), 'guidance')
+
+export const CATALOG_PATH = join(GUIDANCE, 'catalog.md')
+
+const cache = new Map<string, string>()
+
+function load(name: string): string {
+    const cached = cache.get(name)
+    if (cached !== undefined) return cached
+
+    let raw: string
+    try {
+        raw = readFileSync(join(GUIDANCE, `${name}.md`), 'utf-8')
+    } catch {
+        return ''
+    }
+    const text = raw.replace(/^<!--[\s\S]*?-->\s*/, '').trim()
+    cache.set(name, text)
+    return text
+}
+
+/** The one rule set: injected every turn and sent with a rewrite. */
+export function generationRules(): string {
+    return load('catalog')
+}
+
+export function catalogAvailable(): boolean {
+    return load('catalog') !== ''
+}
+
+export function reviewGuide(): string {
+    return load('review')
+}

package/src/scan.ts

@@ -0,0 +1,328 @@
+import { PATTERNS, TIER_2_WORDS, TIER_3_PHRASES, type Pattern, type Tier } from './patterns.js'
+import {
+    burstiness,
+    hasClauseStack,
+    hasLongSentenceRun,
+    hasStaccatoRun,
+    isMonotone,
+    repeatedOpening,
+    wordCount
+} from './stats.js'
+
+export type Hit = {
+    line: number
+    id: string
+    tier: Tier
+    hint: string
+    text: string
+}
+
+export type ScanResult = { hits: Hit[]; chars: number }
+
+const IGNORE_MARKER = /<!--[ \t]*lint-disable[ \t]+--[ \t]*([^\n]*)/
+const MIN_REASON_CHARS = 8
+
+const FENCE = /^\s*(`{3,}|~{3,})/
+const TABLE_ROW = /^\s*\|/
+const CODE_SPAN = /`[^`]*`/g
+const URL_OR_LINK = /https?:\/\/\S+|\]\([^)]*\)/g
+
+export function isIgnored(text: string): boolean {
+    const reason = IGNORE_MARKER.exec(text.slice(0, 400))?.[1]
+    return reason !== undefined && reason.split('-->')[0]!.trim().length >= MIN_REASON_CHARS
+}
+
+type Split = {
+    lines: Array<[number, string]>
+    unclosedFence: number | undefined
+}
+
+function proseLines(text: string): Split {
+    const lines: Array<[number, string]> = []
+    let open: { marker: string; length: number } | undefined
+    let openedAt: number | undefined
+
+    for (const [i, raw] of text.split('\n').entries()) {
+        const indented = raw.startsWith('    ')
+        const m = FENCE.exec(raw)
+        const run = m?.[1]
+
+        if (open === undefined) {
+            if (indented || TABLE_ROW.test(raw)) continue
+            if (run) {
+                open = { marker: run.charAt(0), length: run.length }
+                openedAt = i + 1
+                continue
+            }
+            lines.push([i + 1, raw])
+            continue
+        }
+
+        if (run && run[0] === open.marker && run.length >= open.length && /^\s*(`{3,}|~{3,})\s*$/.test(raw)) {
+            open = undefined
+            openedAt = undefined
+        }
+    }
+    return { lines, unclosedFence: openedAt }
+}
+
+export function scan(text: string, only?: Tier[]): Hit[] {
+    return analyze(text, only).hits
+}
+
+export function scanProse(text: string, only?: Tier[]): ScanResult {
+    return analyze(text, only)
+}
+
+function analyze(text: string, only?: Tier[]): ScanResult {
+    if (isIgnored(text)) return { hits: [], chars: 0 }
+
+    const active: Pattern[] = only ? PATTERNS.filter((p) => only.includes(p.tier)) : PATTERNS
+    const wantCluster = !only || only.includes(2)
+    const wantTier1 = !only || only.includes(1)
+    const hits: Hit[] = []
+    const clusterHits: Hit[] = []
+    const { lines, unclosedFence } = proseLines(text)
+
+    let paraStart: number | undefined
+    let paraText = ''
+
+    const flushPara = (): void => {
+        if (wantCluster && paraStart !== undefined) {
+            if (countTier2(paraText) >= TIER_2_CLUSTER_MIN) {
+                clusterHits.push({
+                    line: paraStart,
+                    id: 'tier-2-cluster',
+                    tier: 2,
+                    hint: 'Several tier-2 words in one paragraph. Use plain words or spread them across sections.',
+                    text: ''
+                })
+            }
+            const t3 = tier3Info(paraText)
+            if (t3.distinct >= TIER_3_CLUSTER_MIN || t3.doubled) {
+                clusterHits.push({
+                    line: paraStart,
+                    id: 'tier-3-phrase-cluster',
+                    tier: 2,
+                    hint: 'Several boilerplate phrases in one paragraph. State the specific claim, or vary genuinely.',
+                    text: ''
+                })
+            }
+        }
+        paraText = ''
+        paraStart = undefined
+    }
+
+    for (const [line, raw] of lines) {
+        const noCode = raw.replaceAll(CODE_SPAN, ' ')
+        const noUrl = noCode.replaceAll(URL_OR_LINK, ' ')
+
+        for (const p of active) {
+            const target = p.scope === 'url' ? noCode : noUrl
+            if (p.re.test(target)) {
+                hits.push({ line, id: p.id, tier: p.tier, hint: p.hint, text: raw.trim() })
+            }
+        }
+
+        if (raw.trim() === '') flushPara()
+        else {
+            if (paraStart === undefined) paraStart = line
+            paraText += ' ' + noUrl
+        }
+    }
+
+    if (wantTier1) {
+        for (const [line, raw] of lines) {
+            if (isTitleCase(raw)) {
+                hits.push({
+                    line,
+                    id: 'title-case-heading',
+                    tier: 1,
+                    hint: 'Sentence case: capitalise the first word and proper nouns only.',
+                    text: raw.trim()
+                })
+            }
+        }
+    }
+
+    flushPara()
+
+    if (wantCluster) {
+        const nonEmpty = lines.filter(([, l]) => l.trim() !== '')
+        const blankLines = lines.filter(([, l]) => l.trim() === '').length
+        const structured =
+            nonEmpty.filter(([, l]) => /^\s*([-*]|\d+\.|#{1,6}\s)/.test(l)).length / Math.max(1, nonEmpty.length)
+        const fullMasked = lines.map(([, l]) => l.replaceAll(CODE_SPAN, ' ').replaceAll(URL_OR_LINK, ' ')).join(' ')
+        const longEnough = wordCount(fullMasked) >= 50
+        if (structured < 0.3 && blankLines >= 1 && longEnough) {
+            if (isMonotone(burstiness(fullMasked))) {
+                clusterHits.push({
+                    line: 1,
+                    id: 'monotone-cadence',
+                    tier: 2,
+                    hint: 'Sentence lengths are unusually uniform across the whole piece. Vary them; human prose wanders.',
+                    text: ''
+                })
+            }
+        }
+        if (structured < 0.3) {
+            if (hasStaccatoRun(fullMasked)) {
+                clusterHits.push({
+                    line: 1,
+                    id: 'staccato-run',
+                    tier: 2,
+                    hint: 'Four or more short sentences in a row. One is emphasis, a run is engineering.',
+                    text: ''
+                })
+            }
+            const opening = repeatedOpening(fullMasked)
+            if (opening !== null) {
+                clusterHits.push({
+                    line: 1,
+                    id: 'repeated-opening',
+                    tier: 2,
+                    hint: 'Three or more consecutive sentences open the same way. Merge sentences or vary the opening.',
+                    text: opening
+                })
+            }
+            if (hasLongSentenceRun(fullMasked)) {
+                clusterHits.push({
+                    line: 1,
+                    id: 'long-sentence-run',
+                    tier: 2,
+                    hint: 'Three or more long sentences in a row. Split one; one idea per sentence.',
+                    text: ''
+                })
+            }
+            if (hasClauseStack(fullMasked)) {
+                clusterHits.push({
+                    line: 1,
+                    id: 'clause-stack',
+                    tier: 2,
+                    hint: 'Several sentences stack three or more subordinate clauses. Split one; one clause, one idea.',
+                    text: ''
+                })
+            }
+        }
+        for (const [start, words] of paragraphs(lines)) {
+            if (words >= WALL_WORDS) {
+                clusterHits.push({
+                    line: start,
+                    id: 'wall-paragraph',
+                    tier: 2,
+                    hint: 'One paragraph carries the whole argument. Split it; short paragraphs are easier to read.',
+                    text: ''
+                })
+            }
+        }
+    }
+
+    if (unclosedFence !== undefined) {
+        hits.push({
+            line: unclosedFence,
+            id: 'unclosed-fence',
+            tier: 'forensic',
+            hint: 'Code fence opened here and never closed, so nothing after it was scanned.',
+            text: ''
+        })
+    }
+
+    const chars = lines.reduce((a, [, l]) => a + l.replaceAll(/\s/g, '').length, 0)
+    return { hits: [...hits, ...clusterHits], chars }
+}
+
+const TIER_2_CLUSTER_MIN = 3
+
+const TIER_3_CLUSTER_MIN = 3
+
+const WALL_WORDS = 120
+
+function paragraphs(lines: Array<[number, string]>): Array<[number, number]> {
+    const out: Array<[number, number]> = []
+    let start = 0
+    let words = 0
+    const flush = (): void => {
+        if (words > 0) out.push([start, words])
+        words = 0
+        start = 0
+    }
+    for (const [line, raw] of lines) {
+        if (raw.trim() === '') {
+            flush()
+            continue
+        }
+        if (words === 0) start = line
+        words += raw
+            .replaceAll(CODE_SPAN, ' ')
+            .replaceAll(URL_OR_LINK, ' ')
+            .split(/\s+/)
+            .filter((w) => w !== '').length
+    }
+    flush()
+    return out
+}
+
+const TIER_2_RE = new RegExp(`\\b(?:${TIER_2_WORDS.join('|')})\\b`, 'gi')
+
+function countTier2(masked: string): number {
+    TIER_2_RE.lastIndex = 0
+    return (masked.match(TIER_2_RE) ?? []).length
+}
+
+const TIER_3_RES = TIER_3_PHRASES.map(
+    (p) => new RegExp('\\b' + p.replace(/-/g, '[- ]').replace(/\\s+/g, '\\s+') + '\\b', 'gi')
+)
+
+type Tier3Info = { distinct: number; doubled: boolean }
+
+function tier3Info(masked: string): Tier3Info {
+    let distinct = 0
+    let doubled = false
+    for (const re of TIER_3_RES) {
+        const n = (masked.match(re) ?? []).length
+        if (n >= 1) distinct++
+        if (n >= 2) doubled = true
+    }
+    return { distinct, doubled }
+}
+
+const SMALL = new Set([
+    'a',
+    'an',
+    'and',
+    'as',
+    'at',
+    'but',
+    'by',
+    'for',
+    'in',
+    'of',
+    'on',
+    'or',
+    'the',
+    'to',
+    'vs',
+    'with'
+])
+
+function isTitleCase(raw: string): boolean {
+    const m = /^#{1,6}\s+(.+)/.exec(raw)
+    if (!m?.[1]) return false
+    const words = m[1].match(/[A-Za-z][\w'-]*/g) ?? []
+    const caps = words
+        .slice(1)
+        .filter((w) => w[0] === w[0]?.toUpperCase() && !SMALL.has(w.toLowerCase()) && w !== w.toUpperCase())
+    return words.length >= 4 && caps.length >= 3
+}
+
+export function proseLineChars(text: string): Array<[number, number]> {
+    if (isIgnored(text)) return []
+    return proseLines(text).lines.map(([line, raw]) => [line, raw.replaceAll(/\s/g, '').length])
+}
+
+export function proseLength(text: string): number {
+    if (isIgnored(text)) return 0
+    let chars = 0
+    for (const [, l] of proseLines(text).lines) chars += l.replaceAll(/\s/g, '').length
+    return chars
+}

package/src/segments.ts

@@ -0,0 +1,81 @@
+import type { Hit } from './scan.js'
+
+export type ProseLine = {
+    line: number
+    chars: number
+    flagged: boolean
+}
+
+export type Span = {
+    startLine: number
+    endLine: number
+    chars: number
+}
+
+export type Verdict = 'clean' | 'mixed' | 'heavy'
+
+export type SegmentResult = {
+    lines: ProseLine[]
+    spans: Span[]
+    flaggedFraction: number
+    verdict: Verdict
+}
+
+const DEFAULT_GAP = 3
+
+const HEAVY_THRESHOLD = 0.8
+const CLEAN_THRESHOLD = 0.1
+
+export function aggregateSpans(lineChars: Array<[number, number]>, hits: Hit[], gap = DEFAULT_GAP): SegmentResult {
+    const totalChars = lineChars.reduce((a, [, c]) => a + c, 0)
+
+    if (lineChars.length === 0 || totalChars === 0) {
+        return { lines: [], spans: [], flaggedFraction: 0, verdict: 'clean' }
+    }
+
+    const flaggedLineSet = new Set<number>()
+    for (const h of hits) {
+        if (h.tier === 'forensic' || h.tier === 1) flaggedLineSet.add(h.line)
+    }
+
+    const lines: ProseLine[] = lineChars.map(([line, chars]) => ({
+        line,
+        chars,
+        flagged: flaggedLineSet.has(line)
+    }))
+
+    const spans: Span[] = []
+    let current: Span | null = null
+    let cleanSince = 0
+
+    const flush = (): void => {
+        if (current !== null) {
+            spans.push(current)
+            current = null
+        }
+        cleanSince = 0
+    }
+
+    for (const l of lines) {
+        if (l.flagged) {
+            if (current === null) {
+                current = { startLine: l.line, endLine: l.line, chars: l.chars }
+            } else {
+                const c: Span = current
+                current = { startLine: c.startLine, endLine: l.line, chars: c.chars + l.chars }
+            }
+            cleanSince = 0
+            continue
+        }
+        cleanSince++
+        if (cleanSince >= gap) flush()
+    }
+    flush()
+
+    const flaggedChars = spans.reduce((a, s) => a + s.chars, 0)
+    const flaggedFraction = totalChars === 0 ? 0 : flaggedChars / totalChars
+    const verdict: Verdict =
+        flaggedFraction >= HEAVY_THRESHOLD ? 'heavy' : flaggedFraction <= CLEAN_THRESHOLD ? 'clean' : 'mixed'
+
+    return { lines, spans, flaggedFraction, verdict }
+}