@iceinvein/agent-skills 0.1.39 → 0.2.0

package/skills/migrate/scripts/leaks.ts

@@ -0,0 +1,184 @@
+import { existsSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { relative } from 'node:path'
+import { storePaths } from './paths.ts'
+import type { Violation } from './types.ts'
+
+const MIN_SECRET_LENGTH = 8
+
+export function parseEnv(text: string): Record<string, string> {
+  const env: Record<string, string> = {}
+  for (const rawLine of text.split('\n')) {
+    const line = rawLine.trim()
+    if (!line || line.startsWith('#')) continue
+    const withoutExport = line.startsWith('export ') ? line.slice(7) : line
+    const at = withoutExport.indexOf('=')
+    if (at === -1) continue
+    const key = withoutExport.slice(0, at).trim()
+    let value = withoutExport.slice(at + 1).trim()
+    if (
+      (value.startsWith('"') && value.endsWith('"') && value.length >= 2) ||
+      (value.startsWith("'") && value.endsWith("'") && value.length >= 2)
+    ) {
+      value = value.slice(1, -1)
+    }
+    env[key] = value
+  }
+  return env
+}
+
+// A URL is excluded because artifacts reference the environment by host on
+// purpose; flagging it would make the gate cry wolf on correct prose.
+//
+// Everything below matches secrets by substring, which has two inherent
+// blind spots that are not worth chasing: a secret that got line-wrapped
+// when it was pasted (a newline now sits inside it) and a paste that differs
+// from the source only in case. Widening the match to catch either would
+// trade a rare miss for false positives common enough to make the gate
+// ignorable, so both are accepted as known limits rather than bugs.
+function isSecretValue(value: string): boolean {
+  if (value.length < MIN_SECRET_LENGTH) return false
+  if (/^https?:\/\//.test(value)) return false
+  return true
+}
+
+export function secretValues(env: Record<string, string>): string[] {
+  return secretEntries(env).map((entry) => entry.value)
+}
+
+type SecretEntry = { value: string; names: string[] }
+
+// Two env vars can legitimately hold the same value (a password reused
+// between a normal and a read-only runtime account, for instance). Grouping
+// by value up front means a shared value is reported once per file, or once
+// per history, naming every variable that holds it, instead of once per
+// variable with the wrong one picked.
+function secretEntries(env: Record<string, string>): SecretEntry[] {
+  const byValue = new Map<string, string[]>()
+  for (const [name, value] of Object.entries(env)) {
+    if (!isSecretValue(value)) continue
+    const names = byValue.get(value)
+    if (names) names.push(name)
+    else byValue.set(value, [name])
+  }
+  return Array.from(byValue, ([value, names]) => ({ value, names }))
+}
+
+function violationFor(entry: SecretEntry, where: string): Violation {
+  return { gate: 'leaks', message: `value of ${entry.names.join(', ')} appears in ${where}` }
+}
+
+async function scanArtifacts(
+  dir: string,
+  root: string,
+  entries: SecretEntry[],
+): Promise<Violation[]> {
+  const violations: Violation[] = []
+  const glob = new Bun.Glob('**/*')
+  for await (const rel of glob.scan({ cwd: dir, onlyFiles: true, dot: true })) {
+    if (rel === '.env') continue
+    const abs = `${dir}/${rel}`
+    // A file the scanner could not read is a file it did not scan. Treating
+    // that as a clean result would be a false negative, so an unreadable
+    // artifact is reported as a leaks violation rather than skipped in silence.
+    let text: string
+    try {
+      text = await readFile(abs, 'utf8')
+    } catch {
+      violations.push({
+        gate: 'leaks',
+        message: `${relative(root, abs)} could not be read, so it was not checked for leaked credentials`,
+      })
+      continue
+    }
+    for (const entry of entries) {
+      if (text.includes(entry.value)) violations.push(violationFor(entry, relative(root, abs)))
+    }
+  }
+  return violations
+}
+
+// Reads a subprocess's stdout in whatever chunks the pipe delivers and tests
+// each chunk for the secrets, without ever buffering the full history into
+// one string. A secret can straddle the boundary between two chunks, so each
+// pass keeps a tail of the combined text at least as long as the longest
+// secret minus one character and prepends it to the next chunk; that is
+// enough overlap for a split match to be whole again on the next check.
+async function findInStream(
+  stream: ReadableStream<Uint8Array>,
+  entries: SecretEntry[],
+): Promise<SecretEntry[]> {
+  const maxLen = Math.max(...entries.map((entry) => entry.value.length))
+  const remaining = new Map(entries.map((entry) => [entry.value, entry] as const))
+  const found: SecretEntry[] = []
+  const decoder = new TextDecoder()
+  let carry = ''
+  const reader = stream.getReader()
+  try {
+    for (;;) {
+      const { done, value } = await reader.read()
+      const chunkText = done ? decoder.decode() : decoder.decode(value, { stream: true })
+      const combined = carry + chunkText
+      for (const [secretValue, entry] of remaining) {
+        if (combined.includes(secretValue)) {
+          found.push(entry)
+          remaining.delete(secretValue)
+        }
+      }
+      if (done || remaining.size === 0) break
+      carry = combined.slice(-(maxLen - 1))
+    }
+  } finally {
+    await reader.cancel().catch(() => {})
+  }
+  return found
+}
+
+// git log -S<secret> would put the raw value on the process argument list,
+// readable by any other local user via ps or /proc for the life of the call.
+// A module whose entire purpose is stopping credentials from escaping must
+// not do that itself, so history is streamed instead: one subprocess for the
+// whole run, no secret in its argv, matched against the streamed diff text
+// in-process. --all covers a secret that only ever landed on a branch other
+// than the one currently checked out, not just an older commit on this one.
+async function scanGitHistory(
+  root: string,
+  gitBin: string,
+  entries: SecretEntry[],
+): Promise<Violation[]> {
+  if (!existsSync(`${root}/.git`)) return []
+
+  let proc: Bun.Subprocess<'ignore', 'pipe', 'ignore'>
+  try {
+    proc = Bun.spawn([gitBin, 'log', '-p', '--all'], {
+      cwd: root,
+      stdin: 'ignore',
+      stdout: 'pipe',
+      stderr: 'ignore',
+    })
+  } catch {
+    return [{ gate: 'leaks', message: 'could not run git to check history for leaked credentials' }]
+  }
+
+  let found: SecretEntry[]
+  try {
+    found = await findInStream(proc.stdout, entries)
+  } finally {
+    proc.kill()
+    await proc.exited.catch(() => {})
+  }
+
+  return found.map((entry) => violationFor(entry, 'git history'))
+}
+
+export async function scanLeaks(opts: { root: string; gitBin?: string }): Promise<Violation[]> {
+  const paths = storePaths(opts.root)
+  if (!existsSync(paths.env)) return []
+  const env = parseEnv(await readFile(paths.env, 'utf8'))
+  const entries = secretEntries(env)
+  if (entries.length === 0) return []
+
+  const artifactViolations = await scanArtifacts(paths.dir, opts.root, entries)
+  const historyViolations = await scanGitHistory(opts.root, opts.gitBin ?? 'git', entries)
+  return [...artifactViolations, ...historyViolations]
+}

package/skills/migrate/scripts/lock.ts

@@ -0,0 +1,188 @@
+import { open, readFile, unlink } from 'node:fs/promises'
+import { storePaths } from './paths.ts'
+
+export type LockHolder = { pid: number; startedAt: string; cmd: string }
+
+export class LockError extends Error {
+  constructor(
+    message: string,
+    readonly kind: 'timeout' | 'stale',
+  ) {
+    super(message)
+    this.name = 'LockError'
+  }
+}
+
+export type LockOptions = {
+  cmd: string
+  timeoutMs?: number
+  force?: boolean
+  alive?: (pid: number) => boolean
+  onWait?: (message: string) => void
+}
+
+// Thirty seconds is roughly two orders of magnitude above the expected hold
+// time for a ten-row batch, so reaching it means something is wrong rather
+// than merely busy.
+const DEFAULT_TIMEOUT_MS = 30_000
+
+// Five consecutive corrupt reads (see readHolder below for what counts as
+// corrupt, as distinct from the merely transient absent state that never
+// reaches this counter at all) is well past what a fluctuating filesystem
+// state could produce by chance, so it means the lock file itself is broken,
+// not merely between holders.
+const UNREADABLE_TOLERANCE = 5
+
+// signal 0 performs the permission and existence checks without delivering
+// anything. ESRCH is the only code that proves the process is gone: EPERM
+// means it exists but belongs to another user, which is still a live holder.
+export function processIsAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch (e) {
+    return (e as { code?: string }).code === 'EPERM'
+  }
+}
+
+export function lockPath(root: string): string {
+  return `${storePaths(root).dir}/.lock`
+}
+
+// A lock file passes through two ordinary transient states that are not
+// evidence of anything wrong: missing (ENOENT, the gap between one holder's
+// unlink and the next one's create) and present-but-empty (the gap between
+// O_EXCL create and the holder record actually being written). Both are
+// reported as 'absent' here and must never spend any of
+// UNREADABLE_TOLERANCE's budget: under real contention with several
+// short-lived holders cycling quickly, a handful of unlucky consecutive
+// polls landing in one gap or the other is ordinary noise, not a stuck lock,
+// and charging it against the same counter a genuinely corrupt file uses
+// produced exactly that false positive. A file that exists, is non-empty,
+// and still fails to parse (or parses to something with no numeric pid) is
+// the only state that means a lock nobody will ever finish writing or
+// clean up, so that is the only state 'corrupt' covers.
+type HolderRead = { status: 'present'; holder: LockHolder } | { status: 'absent' | 'corrupt' }
+
+async function readHolder(path: string): Promise<HolderRead> {
+  let text: string
+  try {
+    text = await readFile(path, 'utf8')
+  } catch {
+    return { status: 'absent' }
+  }
+  if (text.length === 0) return { status: 'absent' }
+  try {
+    const parsed = JSON.parse(text) as LockHolder
+    if (typeof parsed?.pid === 'number') return { status: 'present', holder: parsed }
+  } catch {
+    // falls through to 'corrupt' below
+  }
+  return { status: 'corrupt' }
+}
+
+// Serialises the read-modify-write that `import` and `census` both perform
+// over the whole store file. Without it two subagents importing concurrently
+// each read the same base, and whichever renames last silently discards the
+// other's rows. The lock is deliberately coarse: one lock for the whole
+// store, held for milliseconds, rather than per-file locks that would have to
+// agree on an ordering to stay deadlock-free.
+export async function withStoreLock<T>(
+  root: string,
+  fn: () => Promise<T>,
+  opts: LockOptions,
+): Promise<T> {
+  const path = lockPath(root)
+  const alive = opts.alive ?? processIsAlive
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS
+  const deadline = Date.now() + timeoutMs
+  if (opts.force) await unlink(path).catch(() => {})
+
+  let announced = false
+  let unreadable = 0
+  let delay = 25
+  for (;;) {
+    let handle: Awaited<ReturnType<typeof open>> | null = null
+    try {
+      handle = await open(path, 'wx')
+    } catch (e) {
+      if ((e as { code?: string }).code !== 'EEXIST') throw e
+    }
+    if (handle) {
+      let held = true
+      try {
+        const holder: LockHolder = {
+          pid: process.pid,
+          startedAt: new Date().toISOString(),
+          cmd: opts.cmd,
+        }
+        await handle.writeFile(`${JSON.stringify(holder)}\n`)
+        await handle.close()
+        held = false
+        return await fn()
+      } finally {
+        if (held) await handle.close().catch(() => {})
+        await unlink(path).catch(() => {})
+      }
+    }
+
+    const read = await readHolder(path)
+    if (read.status === 'corrupt') {
+      unreadable += 1
+      if (unreadable >= UNREADABLE_TOLERANCE) {
+        throw new LockError(
+          `store lock at ${path} is unreadable; re-run with --force-unlock after confirming no other agent is writing`,
+          'stale',
+        )
+      }
+    } else if (read.status === 'present') {
+      unreadable = 0
+      const holder = read.holder
+      if (!alive(holder.pid)) {
+        // The holder read above may have already finished, unlinked the lock
+        // and exited by the time this check runs: that is a live release, not
+        // a stale one, and it must not be reported as the holder's pid being
+        // dead. Re-reading before concluding otherwise tells the two apart.
+        // If the file is now gone, or now names a different holder, the lock
+        // was released (or handed off) in the gap between the two reads, and
+        // the right move is to fall through and retry on the next iteration,
+        // not to throw about a holder that no longer holds anything. Only a
+        // second read that still shows the exact same dead pid means the
+        // holder actually exited without releasing.
+        const confirm = await readHolder(path)
+        if (confirm.status === 'present' && confirm.holder.pid === holder.pid) {
+          throw new LockError(
+            `store lock held by pid ${holder.pid}, which is not running. Re-run with --force-unlock after confirming no other agent is writing`,
+            'stale',
+          )
+        }
+      } else if (!announced) {
+        opts.onWait?.(
+          `waiting for store lock (held by pid ${holder.pid} since ${holder.startedAt})`,
+        )
+        announced = true
+      }
+    }
+    // read.status === 'absent' falls straight through to here: see
+    // readHolder's comment for why this must not touch `unreadable`.
+    //
+    // The deadline applies unconditionally, not only while a live holder is
+    // in view: a lock file stuck 'absent' or 'corrupt' for the whole timeout
+    // (its creator crashed after the O_EXCL create but before writing, say)
+    // must still surface as a timeout rather than loop forever, since
+    // neither status alone throws on its own schedule the way a confirmed
+    // dead holder or five straight corrupt reads do. The message names the
+    // holder when one was last seen, and falls back to naming the lock path
+    // when the wait never got that far.
+    if (Date.now() >= deadline) {
+      throw new LockError(
+        read.status === 'present'
+          ? `timed out after ${timeoutMs}ms waiting for the store lock held by pid ${read.holder.pid} since ${read.holder.startedAt}`
+          : `timed out after ${timeoutMs}ms waiting for the store lock at ${path}`,
+        'timeout',
+      )
+    }
+    await Bun.sleep(delay)
+    delay = Math.min(Math.ceil(delay * 1.5), 250)
+  }
+}

package/skills/migrate/scripts/paths.ts

@@ -0,0 +1,103 @@
+import { existsSync } from 'node:fs'
+import { realpath } from 'node:fs/promises'
+import { basename, dirname, join, relative, resolve, sep } from 'node:path'
+
+export const STORE_DIR = '.migrate'
+
+export type StorePaths = {
+  dir: string
+  config: string
+  elements: string
+  requirements: string
+  capabilities: string
+  deltas: string
+  census: string
+  phases: string
+  queueDir: string
+  seamJson: string
+  seamMd: string
+  parityBasis: string
+  env: string
+}
+
+export function storePaths(root: string): StorePaths {
+  const dir = join(root, STORE_DIR)
+  return {
+    dir,
+    config: join(dir, 'config.toml'),
+    elements: join(dir, 'elements.jsonl'),
+    requirements: join(dir, 'requirements.jsonl'),
+    capabilities: join(dir, 'capabilities.jsonl'),
+    deltas: join(dir, 'deltas.jsonl'),
+    census: join(dir, 'census.jsonl'),
+    phases: join(dir, 'phases.json'),
+    queueDir: join(dir, 'queue'),
+    seamJson: join(dir, 'seam.json'),
+    seamMd: join(dir, 'seam.md'),
+    parityBasis: join(dir, 'parity-basis.md'),
+    env: join(dir, '.env'),
+  }
+}
+
+export async function findStoreRoot(startDir: string): Promise<string | null> {
+  let current = resolve(startDir)
+  for (;;) {
+    if (existsSync(join(current, STORE_DIR))) return current
+    const parent = dirname(current)
+    if (parent === current) return null
+    current = parent
+  }
+}
+
+// A shared string prefix is not containment: legacy-notes/ is not inside legacy/.
+// relative() gives '..' as its first segment for anything outside, and '' for the
+// root itself, which counts as inside. This is the one containment predicate
+// both the write side (this file) and the read side (citations.ts) use, so the
+// two cannot drift into disagreeing again about what "inside" means.
+export function isContained(inner: string, outer: string): boolean {
+  const rel = relative(outer, inner)
+  return rel === '' || !(rel.startsWith(`..${sep}`) || rel === '..')
+}
+
+// Resolves a path to its canonical real form -- following symlinks and, on a
+// case-insensitive filesystem, normalizing to the on-disk case -- even when
+// the path itself does not exist yet. A write target is normally exactly
+// that: a file `writeAtomically` is about to create. realpath() alone would
+// fail on it with ENOENT, so this walks up to the nearest ancestor that does
+// exist, resolves that ancestor for real, and reattaches the non-existent
+// remainder unchanged (a path segment that does not exist cannot itself be a
+// symlink or a case variant, so there is nothing further to resolve about it).
+// If no ancestor at all exists (a bogus root), it falls back to the lexically
+// resolved path rather than throwing out of what is meant to be a yes/no check.
+async function realOrNearestAncestor(target: string): Promise<string> {
+  const resolved = resolve(target)
+  let current = resolved
+  const suffix: string[] = []
+  for (;;) {
+    try {
+      const real = await realpath(current)
+      return suffix.length > 0 ? join(real, ...suffix.reverse()) : real
+    } catch {
+      const parent = dirname(current)
+      if (parent === current) return resolved
+      suffix.push(basename(current))
+      current = parent
+    }
+  }
+}
+
+// Lexical comparison alone (the previous implementation) is fooled two ways:
+// a symlink inside the target tree that points into the source tree changes
+// what a path really resolves to without changing its literal text, and on a
+// case-insensitive volume two differently-cased spellings of the same
+// directory compare as unrelated strings even though they are the same
+// physical location. Both are closed by resolving real paths (which follow
+// symlinks and normalize case) before the containment check, exactly as
+// citations.ts already does on the read side.
+export async function assertNotUnderSource(target: string, sourcePath: string): Promise<void> {
+  const realRoot = await realOrNearestAncestor(resolve(sourcePath))
+  const realTarget = await realOrNearestAncestor(resolve(target))
+  if (isContained(realTarget, realRoot)) {
+    throw new Error(`refusing to write inside the read-only source tree: ${target}`)
+  }
+}

package/skills/migrate/scripts/phase-cmd.ts

@@ -0,0 +1,63 @@
+import { loadConfig } from './config.ts'
+import { LockError } from './lock.ts'
+import { isPhase, loadPhases, PHASES, type PhaseState, setPhaseStatus } from './phases.ts'
+
+const STATUSES: readonly PhaseState['status'][] = ['pending', 'running', 'blocked', 'done']
+
+function line(name: string, state: PhaseState): string {
+  return `${name.padEnd(11)} ${state.status.padEnd(8)} ${state.batches.length} batch(es)\n`
+}
+
+// Reading is the default because phase state should be inspectable before it
+// is writable: an orchestrator resuming a run needs to see where it stopped
+// far more often than it needs to move the marker.
+export async function runPhase(opts: {
+  root: string
+  name?: string
+  status?: string
+  forceUnlock?: boolean
+}): Promise<number> {
+  const phases = await loadPhases(opts.root)
+  if (!opts.name) {
+    for (const p of PHASES) process.stdout.write(line(p, phases[p]))
+    return 0
+  }
+  if (!isPhase(opts.name)) {
+    process.stderr.write(`phase: unknown phase ${opts.name}; want one of ${PHASES.join(', ')}\n`)
+    return 2
+  }
+  const name = opts.name
+  if (!opts.status) {
+    process.stdout.write(line(name, phases[name]))
+    return 0
+  }
+  if (!(STATUSES as readonly string[]).includes(opts.status)) {
+    process.stderr.write(
+      `phase: unknown status ${opts.status}; want one of ${STATUSES.join(', ')}\n`,
+    )
+    return 2
+  }
+  const cfg = await loadConfig(opts.root)
+  try {
+    await setPhaseStatus(
+      opts.root,
+      name,
+      opts.status as PhaseState['status'],
+      cfg.source.path,
+      opts.forceUnlock,
+    )
+  } catch (e) {
+    // A lock failure is neither a bad status value nor an unknown phase: the
+    // request is fine and would succeed on retry, so it gets the same
+    // dedicated exit code census-cmd.ts and import-cmd.ts already use for it
+    // (3), not the generic usage-error code (2) a thrown LockError would
+    // otherwise fall through to in main()'s catch-all.
+    if (e instanceof LockError) {
+      process.stderr.write(`phase: ${e.message}\n`)
+      return 3
+    }
+    throw e
+  }
+  process.stdout.write(`phase: ${name} is now ${opts.status}\n`)
+  return 0
+}

package/skills/migrate/scripts/phases.ts

@@ -0,0 +1,113 @@
+import { existsSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { withStoreLock } from './lock.ts'
+import { storePaths } from './paths.ts'
+import { writeAtomically } from './store.ts'
+
+export const PHASES = [
+  'probe',
+  'enumerate',
+  'seam',
+  'extract',
+  'parity',
+  'queue',
+  'adjudicate',
+  'handoff',
+] as const
+
+export type Phase = (typeof PHASES)[number]
+
+export type Batch = { id: string; count: number }
+
+export type PhaseState = {
+  status: 'pending' | 'running' | 'blocked' | 'done'
+  batches: Batch[]
+  pending: string[]
+}
+
+export type PhasesFile = { version: 1; phases: Record<Phase, PhaseState> }
+
+export function isPhase(s: string): s is Phase {
+  return (PHASES as readonly string[]).includes(s)
+}
+
+function empty(): Record<Phase, PhaseState> {
+  const out = {} as Record<Phase, PhaseState>
+  for (const p of PHASES) out[p] = { status: 'pending', batches: [], pending: [] }
+  return out
+}
+
+export async function loadPhases(root: string): Promise<Record<Phase, PhaseState>> {
+  const path = storePaths(root).phases
+  if (!existsSync(path)) return empty()
+  const text = await readFile(path, 'utf8')
+  let parsed: PhasesFile
+  try {
+    parsed = JSON.parse(text) as PhasesFile
+  } catch {
+    throw new Error(`${path}: malformed JSON`)
+  }
+  const state = empty()
+  for (const p of PHASES) {
+    const found = parsed.phases?.[p]
+    if (found)
+      state[p] = {
+        status: found.status,
+        batches: found.batches ?? [],
+        pending: found.pending ?? [],
+      }
+  }
+  return state
+}
+
+// Writes phases.json atomically using temp-plus-rename. Atomicity alone does
+// not make a read-modify-write safe: two callers can still read the same base
+// and one rename still discards the other's batches. That is closed by the
+// store lock, not here. This function deliberately does not take it, because
+// every caller already holds it -- recordBatch runs inside import's and
+// census's critical sections, setPhaseStatus takes it itself, and reset-cmd.ts
+// wraps its whole mutation in it -- and withStoreLock is not reentrant, so a
+// lock taken here would deadlock all three.
+export async function savePhases(
+  root: string,
+  phases: Record<Phase, PhaseState>,
+  sourcePath: string,
+): Promise<void> {
+  const path = storePaths(root).phases
+  const file: PhasesFile = { version: 1, phases }
+  await writeAtomically(path, `${JSON.stringify(file, null, 2)}\n`, sourcePath)
+}
+
+export async function recordBatch(
+  root: string,
+  phase: Phase,
+  batch: Batch,
+  sourcePath: string,
+): Promise<void> {
+  const phases = await loadPhases(root)
+  const state = phases[phase]
+  if (!state.batches.some((b) => b.id === batch.id)) state.batches.push(batch)
+  if (state.status !== 'done') state.status = 'running'
+  await savePhases(root, phases, sourcePath)
+}
+
+export async function setPhaseStatus(
+  root: string,
+  phase: Phase,
+  status: PhaseState['status'],
+  sourcePath: string,
+  forceUnlock?: boolean,
+): Promise<void> {
+  // Same read-modify-write hazard as recordBatch. The orchestrator is the
+  // only expected caller and is serial, but the cost of holding the lock for
+  // a status flip is a few milliseconds and it removes the question.
+  await withStoreLock(
+    root,
+    async () => {
+      const phases = await loadPhases(root)
+      phases[phase].status = status
+      await savePhases(root, phases, sourcePath)
+    },
+    { cmd: 'phase', ...(forceUnlock ? { force: true } : {}) },
+  )
+}