@tokenfactory/acc-runner 0.47.17 → 0.47.18

package/scripts/audit-terminal-claims-core.d.mts

@@ -0,0 +1,66 @@
+// Type surface for audit-terminal-claims-core.mjs, so the fixture test can
+// import it under NodeNext + strict without allowJs.
+export interface AuditSource { read: boolean; error?: string; earliest?: string | null }
+export interface AuditEvent { id: string; target_id: string; created_at: string; verb: string; payload: Record<string, unknown> }
+export interface AuditTaskEvent { task_id: string; created_at: string; kind: string; payload: Record<string, unknown> }
+export interface AuditTask { id: string; status: string; pr_number?: number | null; repo?: string | null; branch?: string | null; runner_id?: string | null }
+export interface AuditLock { task_id: string; runner_id: string; locked_at?: string }
+export interface AuditRunner { id: string; current_task_id: string | null; heartbeat_at?: string | null }
+
+export interface AuditBound {
+  statement: string;
+  reads: string[];
+  cannot_see: string[];
+  earliest_retained: Record<string, string | null>;
+  unread_sources: Array<{ source: string; error: string }>;
+}
+
+export interface AuditRow {
+  task_id: string;
+  pr_number: number | null;
+  terminal_status: string;
+  terminal_at: string;
+  evidence: Array<'claim_lock_held_past_terminal' | 'pane_output_after_terminal'>;
+  claim: { record: string | null; granted_at: string | null; runner_id: string | null; runner_id_source: string };
+  lock: { released_at: string | null; right_censored: boolean; persisted_ms: number } | null;
+  pane_output_after_terminal: { events: number; first_at: string; last_at: string; persisted_ms: number } | null;
+  cancellation_event: { verb: string; at: string; elapsed_ms: unknown; token_spend: unknown; token_spend_status: unknown } | null;
+  claim_persisted_ms: number;
+  pane_active_after_terminal_ms: number;
+  repo: string | null;
+  branch: string | null;
+  commit_after_terminal: boolean | null;
+}
+
+export interface AuditReport {
+  bound: AuditBound;
+  date_range: { from_inclusive: string; to_exclusive: string };
+  terminal_tasks_examined: number;
+  task_rows_examined: number;
+  count: number;
+  total_claim_time_ms: number;
+  total_pane_time_after_terminal_ms: number;
+  rows: AuditRow[];
+  current_terminal_locks: AuditLock[];
+  runners_pointing_at_terminal_tasks: Array<{ runner_id: string; current_task_id: string; heartbeat_at: string | null }>;
+}
+
+export const TERMINAL_AUDIT_STATUSES: string[];
+export const CLAIM_VERB: string;
+export const RELEASE_VERB: string;
+export const STATUS_VERB: string;
+export const CANCELLATION_VERBS: string[];
+export const AUDIT_VERBS: string[];
+export const AUDIT_READS: string[];
+export const AUDIT_BLIND_SPOTS: string[];
+export function auditBound(input: { sources: Record<string, AuditSource> }): AuditBound;
+export function analyzeTerminalClaims(input: {
+  from: string;
+  to: string;
+  events: AuditEvent[];
+  taskEvents: AuditTaskEvent[];
+  tasks: AuditTask[];
+  locks: AuditLock[];
+  runners: AuditRunner[];
+  sources: Record<string, AuditSource>;
+}): AuditReport;

package/scripts/audit-terminal-claims-core.mjs

@@ -0,0 +1,164 @@
+/**
+ * Pure analysis for audit-terminal-claims.mjs: given the retained records the
+ * script read, list the tasks that reached merged/archived while a runner still
+ * held a claim or file lock, and state what those records cannot show.
+ *
+ * No I/O here, so the fixture test can drive it without a database.
+ */
+
+export const TERMINAL_AUDIT_STATUSES = ['merged', 'archived'];
+export const CLAIM_VERB = 'task.claim_locks_granted';
+export const RELEASE_VERB = 'task.locks_released';
+export const STATUS_VERB = 'task.status_changed';
+export const CANCELLATION_VERBS = ['runner.terminal_cancelled', 'runner.write_refused'];
+export const AUDIT_VERBS = [STATUS_VERB, CLAIM_VERB, RELEASE_VERB, ...CANCELLATION_VERBS];
+
+/** The fixed part of the bound. The per-run part (earliest retained rows, unread sources) is added by auditBound. */
+export const AUDIT_READS = [
+  'acc.activity_events and acc.activity_events_archive (attached partitions): task.status_changed to merged/archived; '
+    + 'task.claim_locks_granted, the claim record acc.claim_task_with_locks writes when a runner claims a task (the claim and its file lock are one record); '
+    + 'task.locks_released; runner.terminal_cancelled and runner.write_refused',
+  'acc.task_events kind=log with a stdout/stderr stream, for the terminal tasks only: pane output written after the terminal transition',
+  'acc.tasks: status, pr_number, repo, branch, runner_id (runner_id is attribution kept after completion, not proof of a live claim)',
+  'acc.task_file_locks and acc.runners.current_task_id: the current snapshot only, with no history',
+];
+
+export const AUDIT_BLIND_SPOTS = [
+  'Claims whose claim/release events were rolled off acc.activity_events and whose archive partition was detached; nothing older than the earliest retained event below can be seen.',
+  'Claims taken by any path that wrote no task.claim_locks_granted event.',
+  'A pane that kept running after the terminal transition released its lock (the release happens in the same transaction) and wrote no task_events and no cancellation event: work or spend with no output is invisible.',
+  'task_events removed with their task (on delete cascade) and any row the reading identity cannot see under RLS.',
+  'Past values of acc.runners.current_task_id; only the current pointer is read.',
+  'Commits on deleted or force-pushed branches, and push times (the commit probe reads committer timestamps).',
+];
+
+const ms = (iso) => Date.parse(iso);
+const isPaneOutput = (e) => e.kind === 'log' && (e.payload?.stream === 'stdout' || e.payload?.stream === 'stderr');
+
+/**
+ * @param {{ sources: Record<string, { read: boolean, error?: string, earliest?: string | null }> }} input
+ */
+export function auditBound({ sources }) {
+  const unread = Object.entries(sources)
+    .filter(([, s]) => !s.read)
+    .map(([source, s]) => ({ source, error: s.error ?? 'not read' }));
+  const earliestRetained = Object.fromEntries(
+    Object.entries(sources).filter(([, s]) => s.read && 'earliest' in s).map(([source, s]) => [source, s.earliest ?? null]),
+  );
+  return {
+    statement: 'Bounded audit: it reports only what the retained claim and lock records below can show. '
+      + 'A task missing from rows is not shown to be clean; it may sit in a blind spot. '
+      + 'Intervals with no release are censored at TO_ISO.',
+    reads: AUDIT_READS,
+    cannot_see: AUDIT_BLIND_SPOTS,
+    earliest_retained: earliestRetained,
+    unread_sources: unread,
+  };
+}
+
+/**
+ * @param {{
+ *   from: string, to: string,
+ *   events: Array<{ id: string, target_id: string, created_at: string, verb: string, payload: Record<string, unknown> }>,
+ *   taskEvents: Array<{ task_id: string, created_at: string, kind: string, payload: Record<string, unknown> }>,
+ *   tasks: Array<{ id: string, status: string, pr_number?: number | null, repo?: string | null, branch?: string | null, runner_id?: string | null }>,
+ *   locks: Array<{ task_id: string, runner_id: string, locked_at?: string }>,
+ *   runners: Array<{ id: string, current_task_id: string | null, heartbeat_at?: string | null }>,
+ *   sources: Record<string, { read: boolean, error?: string, earliest?: string | null }>,
+ * }} input
+ */
+export function analyzeTerminalClaims({ from, to, events, taskEvents, tasks, locks, runners, sources }) {
+  const fromMs = ms(from);
+  const toMs = ms(to);
+  const taskById = new Map(tasks.map((t) => [t.id, t]));
+  const byTask = new Map();
+  for (const e of [...events].sort((a, b) => ms(a.created_at) - ms(b.created_at))) {
+    if (!byTask.has(e.target_id)) byTask.set(e.target_id, []);
+    byTask.get(e.target_id).push(e);
+  }
+  const paneByTask = new Map();
+  for (const e of taskEvents.filter(isPaneOutput)) {
+    if (!paneByTask.has(e.task_id)) paneByTask.set(e.task_id, []);
+    paneByTask.get(e.task_id).push(e);
+  }
+
+  // One row per task, anchored at its first merged/archived transition in the window.
+  const terminals = new Map();
+  for (const [taskId, history] of byTask) {
+    const first = history.find((e) => e.verb === STATUS_VERB
+      && TERMINAL_AUDIT_STATUSES.includes(e.payload?.to)
+      && ms(e.created_at) >= fromMs && ms(e.created_at) < toMs);
+    if (first) terminals.set(taskId, first);
+  }
+
+  const rows = [];
+  for (const [taskId, terminal] of terminals) {
+    const at = ms(terminal.created_at);
+    const history = byTask.get(taskId);
+    const grant = history.filter((e) => e.verb === CLAIM_VERB && ms(e.created_at) <= at).at(-1) ?? null;
+    const release = grant
+      ? history.find((e) => e.verb === RELEASE_VERB && ms(e.created_at) >= ms(grant.created_at)) ?? null
+      : null;
+    // The terminal transition releases locks in its own transaction, so a release
+    // stamped at or before the terminal event is zero overrun.
+    const lockHeld = !!grant && !(release && ms(release.created_at) <= at);
+    const paneAfter = (paneByTask.get(taskId) ?? [])
+      .filter((e) => ms(e.created_at) > at && ms(e.created_at) < toMs)
+      .sort((a, b) => ms(a.created_at) - ms(b.created_at));
+    if (!lockHeld && paneAfter.length === 0) continue;
+
+    const task = taskById.get(taskId);
+    const lockPersistedMs = lockHeld ? Math.min(release ? ms(release.created_at) : toMs, toMs) - at : 0;
+    const panePersistedMs = paneAfter.length ? ms(paneAfter.at(-1).created_at) - at : 0;
+    const cancellation = history.find((e) => CANCELLATION_VERBS.includes(e.verb) && ms(e.created_at) >= at) ?? null;
+    rows.push({
+      task_id: taskId,
+      pr_number: task?.pr_number ?? null,
+      terminal_status: terminal.payload.to,
+      terminal_at: terminal.created_at,
+      evidence: [lockHeld ? 'claim_lock_held_past_terminal' : null, paneAfter.length ? 'pane_output_after_terminal' : null].filter(Boolean),
+      claim: grant
+        ? { record: CLAIM_VERB, granted_at: grant.created_at, runner_id: grant.payload?.runner_id ?? null, runner_id_source: 'claim_event' }
+        : { record: null, granted_at: null, runner_id: task?.runner_id ?? null, runner_id_source: 'tasks.runner_id (attribution only; claim record not retained)' },
+      lock: grant
+        ? { released_at: release?.created_at ?? null, right_censored: lockHeld && !release, persisted_ms: lockPersistedMs }
+        : null,
+      pane_output_after_terminal: paneAfter.length
+        ? { events: paneAfter.length, first_at: paneAfter[0].created_at, last_at: paneAfter.at(-1).created_at, persisted_ms: panePersistedMs }
+        : null,
+      cancellation_event: cancellation
+        ? {
+          verb: cancellation.verb, at: cancellation.created_at,
+          elapsed_ms: cancellation.payload?.elapsed_ms ?? null,
+          token_spend: cancellation.payload?.token_spend ?? null,
+          token_spend_status: cancellation.payload?.token_spend_status ?? null,
+        }
+        : null,
+      // Claim time is the lock held past terminal, nothing else: pane output
+      // after the lock was released is work, not a held claim, and is reported
+      // separately so the two are never summed into one figure.
+      claim_persisted_ms: lockPersistedMs,
+      pane_active_after_terminal_ms: panePersistedMs,
+      repo: task?.repo ?? null,
+      branch: task?.branch ?? null,
+      commit_after_terminal: null,
+    });
+  }
+  rows.sort((a, b) => ms(a.terminal_at) - ms(b.terminal_at));
+
+  const isTerminal = (id) => TERMINAL_AUDIT_STATUSES.includes(taskById.get(id)?.status);
+  return {
+    bound: auditBound({ sources }),
+    date_range: { from_inclusive: from, to_exclusive: to },
+    terminal_tasks_examined: terminals.size,
+    task_rows_examined: tasks.length,
+    count: rows.length,
+    total_claim_time_ms: rows.reduce((sum, row) => sum + row.claim_persisted_ms, 0),
+    total_pane_time_after_terminal_ms: rows.reduce((sum, row) => sum + row.pane_active_after_terminal_ms, 0),
+    rows,
+    current_terminal_locks: locks.filter((lock) => isTerminal(lock.task_id)),
+    runners_pointing_at_terminal_tasks: runners
+      .filter((r) => r.current_task_id && isTerminal(r.current_task_id))
+      .map((r) => ({ runner_id: r.id, current_task_id: r.current_task_id, heartbeat_at: r.heartbeat_at ?? null })),
+  };
+}

package/scripts/audit-terminal-claims.mjs

@@ -0,0 +1,98 @@
+/** Report only. Usage: node packages/acc-runner/scripts/audit-terminal-claims.mjs FROM_ISO TO_ISO
+ * Uses the runner's stored login and SELECTs only. No ACC/GitHub mutations.
+ *
+ * BOUND: this audit sees only the claim and lock records that are still retained
+ * and visible to the runner's identity under RLS. It reads task.claim_locks_granted
+ * (the claim record; a claim and its file lock are one row), task.locks_released,
+ * task.status_changed and runner cancellation events from acc.activity_events plus
+ * the attached partitions of acc.activity_events_archive; pane output from
+ * acc.task_events; and the current acc.task_file_locks / acc.runners.current_task_id
+ * snapshot. It cannot see claims whose events were purged or detached, claims taken
+ * by a path that wrote no claim event, a pane that worked silently after its lock was
+ * released, or past current_task_id values. The full statement, with the earliest
+ * retained row per source and any source that could not be read, is printed as
+ * `bound` at the top of the report.
+ */
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { getTokenProvider } from '../dist/token-provider.js';
+import { createRunnerClient } from '../dist/supabase.js';
+import { loadConfig, applyBootstrapConfig } from '../dist/config.js';
+import { AUDIT_VERBS, TERMINAL_AUDIT_STATUSES, STATUS_VERB, analyzeTerminalClaims } from './audit-terminal-claims-core.mjs';
+
+const [from, to] = process.argv.slice(2);
+if (!from || !to || !Number.isFinite(Date.parse(from)) || !Number.isFinite(Date.parse(to)) || Date.parse(from) >= Date.parse(to)) {
+  throw new Error('Supply an increasing FROM_ISO TO_ISO date range');
+}
+const session = await getTokenProvider().load();
+if (!session) throw new Error('Runner login is required; audit not run');
+const cfg = applyBootstrapConfig(loadConfig(), session);
+const db = createRunnerClient(cfg, session.access_token);
+const exec = promisify(execFile);
+
+async function all(table, columns, orderBy, filter = query => query) {
+  const out = [];
+  for (let offset = 0; ; offset += 500) {
+    const result = await filter(db.from(table).select(columns)).order(orderBy).range(offset, offset + 499);
+    if (result.error) throw new Error(`${table} read failed: ${result.error.message}`);
+    out.push(...result.data);
+    if (result.data.length < 500) return out;
+  }
+}
+async function earliestOf(table) {
+  const r = await db.from(table).select('created_at').order('created_at').limit(1);
+  if (r.error) throw new Error(`${table} read failed: ${r.error.message}`);
+  return r.data?.[0]?.created_at ?? null;
+}
+
+// Required sources fail the run; optional ones are recorded as unread in the bound.
+const sources = {};
+async function source(name, read, { required = false, withEarliest = true } = {}) {
+  try {
+    const rows = await read();
+    sources[name] = withEarliest ? { read: true, earliest: await earliestOf(name) } : { read: true };
+    return rows;
+  } catch (err) {
+    if (required) throw err;
+    sources[name] = { read: false, error: err.message };
+    return [];
+  }
+}
+
+const eventColumns = 'id,target_id,created_at,verb,payload';
+const byVerb = query => query.in('verb', AUDIT_VERBS).lt('created_at', to);
+const hot = await source('activity_events', () => all('activity_events', eventColumns, 'id', byVerb), { required: true });
+const archived = await source('activity_events_archive', () => all('activity_events_archive', eventColumns, 'id', byVerb));
+const seen = new Set(hot.map(e => e.id));
+const events = [...hot, ...archived.filter(e => !seen.has(e.id))];
+const tasks = await source('tasks', () => all('tasks', 'id,status,pr_number,repo,branch,runner_id', 'id'), { required: true, withEarliest: false });
+const locks = await source('task_file_locks', () => all('task_file_locks', 'task_id,runner_id,locked_at', 'task_id'), { required: true, withEarliest: false });
+const runners = await source('runners', () => all('runners', 'id,current_task_id,heartbeat_at', 'id'), { withEarliest: false });
+
+const terminalIds = [...new Set(events
+  .filter(e => e.verb === STATUS_VERB && TERMINAL_AUDIT_STATUSES.includes(e.payload?.to)
+    && Date.parse(e.created_at) >= Date.parse(from) && Date.parse(e.created_at) < Date.parse(to))
+  .map(e => e.target_id))];
+const taskEvents = await source('task_events', async () => {
+  const out = [];
+  for (let i = 0; i < terminalIds.length; i += 100) {
+    out.push(...await all('task_events', 'id,task_id,created_at,kind,payload', 'id', query => query
+      .in('task_id', terminalIds.slice(i, i + 100)).eq('kind', 'log').lt('created_at', to)));
+  }
+  return out;
+});
+
+const report = analyzeTerminalClaims({ from, to, events, taskEvents, tasks, locks, runners, sources });
+for (const row of report.rows) {
+  if (!row.repo || !row.branch) { row.commit_probe = 'missing repo/branch'; continue; }
+  const endpoint = `repos/${row.repo}/commits?sha=${encodeURIComponent(row.branch)}&since=${encodeURIComponent(row.terminal_at)}&until=${encodeURIComponent(to)}&per_page=100`;
+  try {
+    const { stdout } = await exec('gh', ['api', '--paginate', endpoint, '--jq', '.[] | [.sha, .commit.committer.date] | @tsv'], { timeout: 60000, maxBuffer: 16 * 1024 * 1024 });
+    const commits = stdout.trim().split('\n').filter(Boolean).map(line => line.split('\t'))
+      .filter(([, date]) => Date.parse(date) > Date.parse(row.terminal_at));
+    row.commit_after_terminal = commits.length > 0;
+    row.commit_shas_after_terminal = commits.map(([sha]) => sha);
+    row.commit_probe = 'branch history by committer timestamp; not a push-time audit';
+  } catch { row.commit_probe = 'unavailable (deleted branch, permissions, or network); not inferred false'; }
+}
+console.log(JSON.stringify({ project: new URL(cfg.supabaseUrl).hostname, ...report }, null, 2));