agentlinkops 0.5.1 → 0.6.0

package/src/citations/report.js

@@ -0,0 +1,165 @@
+// The citation report: a single self-contained HTML file that renders every
+// cell's evidence — trend, screenshots inline, Wilson intervals, honest
+// classifications — in a format a marketing manager can forward to a client.
+//
+// No external assets. No server. No JavaScript. Opens in any browser,
+// survives email forwarding, and carries the screenshots as base64 because
+// linked screenshots break the moment the evidence directory moves.
+import { readFile, readdir } from 'node:fs/promises';
+import { join, basename } from 'node:path';
+
+const esc = (text) => String(text ?? '')
+  .replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;');
+
+const pct = (v) => `${Math.round(v * 100)}%`;
+
+const CLASSIFICATION_STYLE = {
+  first_epoch: { label: 'First epoch', color: '#6b7280', bg: '#f3f4f6' },
+  insufficient_data: { label: 'Insufficient data', color: '#92400e', bg: '#fef3c7' },
+  declined: { label: 'Declined', color: '#991b1b', bg: '#fee2e2' },
+  grown: { label: 'Grown', color: '#065f46', bg: '#d1fae5' },
+  not_distinguishable: { label: 'Not distinguishable', color: '#6b7280', bg: '#f3f4f6' },
+};
+
+function classificationBadge(cls) {
+  const s = CLASSIFICATION_STYLE[cls] ?? { label: cls, color: '#6b7280', bg: '#f3f4f6' };
+  return `<span style="display:inline-block;padding:2px 10px;border-radius:12px;font-size:11px;font-weight:600;color:${s.color};background:${s.bg};margin-left:8px">${esc(s.label)}</span>`;
+}
+
+function rateBar(rate, low, high) {
+  return `
+  <div style="margin:8px 0 4px 0">
+    <div style="font-size:22px;font-weight:700;color:#111827">${pct(rate)} <span style="font-size:13px;font-weight:400;color:#6b7280">cited (${pct(low)}–${pct(high)} Wilson 95%)</span></div>
+    <div style="position:relative;height:8px;border-radius:4px;background:#e5e7eb;margin-top:6px;overflow:hidden">
+      <div style="position:absolute;left:${low * 100}%;width:${(high - low) * 100}%;top:0;bottom:0;background:#3b82f6;border-radius:4px;opacity:0.5"></div>
+      <div style="position:absolute;left:${rate * 100}%;width:2px;top:-2px;bottom:-2px;background:#1d4ed8"></div>
+    </div>
+  </div>`;
+}
+
+function evidenceCard(cell, envelope, screenshotB64) {
+  const [enginePart, promptPart, targetPart] = cell.cell_id.split('|');
+  const citedDomains = [...new Set((envelope?.citations ?? []).map((c) => {
+    try { return new URL(c.url).hostname; } catch { return null; }
+  }).filter(Boolean))];
+
+  return `
+  <div style="border:1px solid #e5e7eb;border-radius:12px;padding:16px;margin-bottom:16px;background:#fff">
+    <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:8px">
+      <div>
+        <span style="font-size:14px;font-weight:700;color:#111827;font-family:ui-monospace,monospace">${esc(enginePart)}</span>
+        <span style="font-size:12px;color:#6b7280;margin-left:8px">${esc(promptPart)}</span>
+      </div>
+      ${classificationBadge(cell.classification)}
+    </div>
+    ${rateBar(cell.rate, cell.ci_low, cell.ci_high)}
+    <div style="font-size:11px;color:#9ca3af;margin-top:2px">n=${cell.n} rendered · ${cell.unknowns} unknown${cell.unknowns === 1 ? '' : 's'} excluded · ${cell.mentioned} mentioned without citation</div>
+    ${screenshotB64 ? `
+    <details style="margin-top:12px">
+      <summary style="font-size:12px;color:#3b82f6;cursor:pointer;font-weight:500">View answer screenshot</summary>
+      <img src="data:image/png;base64,${screenshotB64}" style="max-width:100%;border-radius:8px;border:1px solid #e5e7eb;margin-top:8px" alt="Answer surface screenshot" />
+    </details>` : ''}
+    ${envelope?.answer ? `
+    <details style="margin-top:8px">
+      <summary style="font-size:12px;color:#3b82f6;cursor:pointer;font-weight:500">View answer text</summary>
+      <div style="font-size:13px;color:#374151;margin-top:8px;padding:12px;background:#f9fafb;border-radius:8px;white-space:pre-wrap;max-height:300px;overflow-y:auto">${esc(envelope.answer.slice(0, 2000))}${envelope.answer.length > 2000 ? '\n…' : ''}</div>
+    </details>` : ''}
+    ${citedDomains.length > 0 ? `
+    <div style="margin-top:10px">
+      <span style="font-size:11px;font-weight:600;color:#374151">Cited domains:</span>
+      ${citedDomains.map((d) => `<span style="display:inline-block;padding:1px 8px;border-radius:8px;font-size:11px;background:#eff6ff;color:#1e40af;margin:2px 2px">${esc(d)}</span>`).join('')}
+    </div>` : `<div style="margin-top:10px;font-size:11px;color:#9ca3af">No citations recorded this epoch</div>`}
+  </div>`;
+}
+
+function summaryParagraph(cells, panelName) {
+  const totalN = cells.reduce((sum, c) => sum + c.n, 0);
+  const totalK = cells.reduce((sum, c) => sum + c.k, 0);
+  const declined = cells.filter((c) => c.classification === 'declined').length;
+  const grown = cells.filter((c) => c.classification === 'grown').length;
+  const insufficient = cells.filter((c) => c.classification === 'insufficient_data').length;
+  const unknowns = cells.reduce((sum, c) => sum + c.unknowns, 0);
+
+  const parts = [];
+  parts.push(`${totalK} of ${totalN} rendered observations cited the target`);
+  if (unknowns > 0) parts.push(`${unknowns} unknown${unknowns === 1 ? ' was' : 's were'} excluded from the denominator`);
+  if (declined > 0) parts.push(`${declined} cell${declined === 1 ? '' : 's'} declined with complete evidence`);
+  if (grown > 0) parts.push(`${grown} cell${grown === 1 ? '' : 's'} grew`);
+  if (insufficient > 0) parts.push(`${insufficient} cell${insufficient === 1 ? '' : 's'} ${insufficient === 1 ? 'has' : 'have'} insufficient data to interpret`);
+  if (declined === 0 && grown === 0 && insufficient === totalN === 0) parts.push('no actionable changes were detected');
+
+  return `Across ${cells.length} monitored cell${cells.length === 1 ? '' : 's'}, ${parts.join('. ')}. `;
+}
+
+export async function generateReport({ dir, epochId, panel, now = new Date() }) {
+  // Read the epoch rows for this epoch
+  const epochsPath = join(dir, 'citations-epochs.jsonl');
+  const epochsText = await readFile(epochsPath, 'utf8').catch(() => '');
+  const allRows = epochsText.split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l));
+  const cells = allRows.filter((r) => r.epoch_id === epochId);
+  if (cells.length === 0) throw new Error(`no epoch rows found for ${epochId}`);
+
+  // Read the evidence envelopes for this epoch (latest per cell)
+  const evidenceRoot = join(dir, 'citations', 'evidence', epochId);
+  const evidenceFiles = await readdir(evidenceRoot).catch(() => []);
+  const envelopes = new Map();
+  const screenshots = new Map();
+  for (const file of evidenceFiles) {
+    if (!file.endsWith('.json')) continue;
+    try {
+      const env = JSON.parse(await readFile(join(evidenceRoot, file), 'utf8'));
+      const existing = envelopes.get(env.cell_id);
+      if (!existing || env.run_index > existing.run_index) {
+        envelopes.set(env.cell_id, env);
+        // Look for the sibling screenshot
+        const shotFile = file.replace('.json', '.screenshot.png');
+        if (evidenceFiles.includes(shotFile)) {
+          const shotBuf = await readFile(join(evidenceRoot, shotFile));
+          screenshots.set(env.cell_id, shotBuf.toString('base64'));
+        }
+      }
+    } catch { /* skip unreadable */ }
+  }
+
+  const totalSpent = cells.reduce((sum, c) => sum + (c.spent_estimate_usd ?? 0), 0);
+  const engineList = [...new Set(cells.map((c) => c.cell_id.split('|')[0]))];
+
+  const cards = cells.map((cell) => evidenceCard(cell, envelopes.get(cell.cell_id), screenshots.get(cell.cell_id))).join('\n');
+  const summary = summaryParagraph(cells, panel?.note ?? 'Citation panel');
+
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Citation Report — ${esc(now.toISOString().slice(0, 10))}</title>
+</head>
+<body style="margin:0;padding:24px;background:#f3f4f6;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;color:#111827">
+<div style="max-width:720px;margin:0 auto">
+
+<div style="background:#111827;border-radius:16px;padding:24px 28px;margin-bottom:20px">
+  <div style="font-size:11px;font-weight:600;color:#9ca3af;text-transform:uppercase;letter-spacing:1px;margin-bottom:4px">AgentLinkOps Citation Report</div>
+  <div style="font-size:24px;font-weight:700;color:#fff">${esc(panel?.note ?? 'Citation monitoring epoch')}</div>
+  <div style="font-size:13px;color:#9ca3af;margin-top:4px">${esc(now.toISOString().slice(0, 10))} · ${engineList.length} engine${engineList.length === 1 ? '' : 's'} · ${cells.length} cell${cells.length === 1 ? '' : 's'} · estimated spend $${totalSpent.toFixed(4)}</div>
+</div>
+
+<div style="background:#fff;border:1px solid #e5e7eb;border-radius:12px;padding:16px 20px;margin-bottom:20px">
+  <div style="font-size:11px;font-weight:600;color:#6b7280;text-transform:uppercase;letter-spacing:1px;margin-bottom:8px">Summary</div>
+  <div style="font-size:14px;line-height:1.6;color:#374151">${esc(summary)}</div>
+</div>
+
+${cards}
+
+<div style="margin-top:24px;padding:16px;background:#f9fafb;border-radius:12px;border:1px solid #e5e7eb">
+  <div style="font-size:11px;color:#6b7280;line-height:1.5">
+    <strong>Evidence integrity:</strong> Every observation carries a content-addressed evidence snapshot
+    and screenshot. Observations with unknown outcomes are excluded from denominators. Citation rates
+    are reported with Wilson 95% intervals and their sample sizes. Cells with insufficient data are
+    labeled as such — never interpreted. This report was generated by AgentLinkOps.
+  </div>
+</div>
+
+</div>
+</body>
+</html>`;
+}

package/src/citations/runner.js

@@ -89,18 +89,26 @@ export async function runEpoch(panelInput, options = {}) {
         aborted = { reason: 'budget', spent, nextEstimate: estimate, maxUsd };
         break outer;
       }
-      let run;
-      try {
-        run = await cell.engine.run({ prompt: cell.prompt.text, runIndex });
-      } catch (cause) {
-        if (cause instanceof EngineError && cause.retriable) {
-          // An unknown is honest, but never silent: the reason travels to
-          // stderr so a run of unknowns names its wall.
-          if (options.err) options.err(`unknown observation (${cell.id}): ${cause.message}`);
-          outcomesByCell.get(cell.id).push('unknown');
-          continue;
+      let run = null;
+      let lastCause = null;
+      // One retry per observation: with exit-rotating engines the retry lands
+      // on a fresh exit, which is exactly what a transient 418 or dead exit
+      // needs. The second failure is an honest, non-silent unknown.
+      for (let attempt = 0; attempt < 2 && !run; attempt += 1) {
+        try {
+          run = await cell.engine.run({ prompt: cell.prompt.text, runIndex });
+        } catch (cause) {
+          lastCause = cause;
+          if (!(cause instanceof EngineError) || !cause.retriable) throw cause;
+          if (options.err && attempt === 0) options.err(`retrying observation (${cell.id}): ${cause.message}`);
         }
-        throw cause;
+      }
+      if (!run) {
+        // An unknown is honest, but never silent: the reason travels to
+        // stderr so a run of unknowns names its wall.
+        if (options.err) options.err(`unknown observation (${cell.id}): ${lastCause?.message ?? 'unspecified'}`);
+        outcomesByCell.get(cell.id).push('unknown');
+        continue;
       }
       spent += run.costEstimateUsd ?? 0;
 
@@ -144,18 +152,21 @@ export async function runEpoch(panelInput, options = {}) {
     const ci = wilsonInterval(k, n) ?? [0, 1];
     const baselineRow = prior.filter((r) => r.cell_id === cell.id && r.n >= CITATION_LIMITS.minSamplesForInterpretation).at(-1);
     const baseline = baselineRow ? { k: baselineRow.k, n: baselineRow.n } : null;
-    epochRows.push(epochRowSchema.parse({
+    const row = epochRowSchema.parse({
       schema_version: CITATIONS_VERSION, epoch_id: epochId, cell_id: cell.id,
       k, n, unknowns, mentioned,
       rate: smoothedRate(k, n) ?? 0, ci_low: ci[0], ci_high: ci[1],
       classification: classifyEpoch({ k, n }, baseline, CITATION_LIMITS),
       baseline: baselineRow ? { epoch_id: baselineRow.epoch_id, rate: baselineRow.rate, ci_low: baselineRow.ci_low, ci_high: baselineRow.ci_high } : undefined,
       spent_estimate_usd: Number(spent.toFixed(6)), at: nowIso(),
-    }));
+    });
+    epochRows.push(row);
+    // Incremental flush: every completed cell's row is on disk immediately, so
+    // a killed epoch loses nothing — the lesson of the lost 3x3 tallies.
+    await writeFile(join(dir, CITATION_FILES.epochs), JSON.stringify(row) + '\n', { flag: 'a' });
   }
 
   await writeFile(join(dir, CITATION_FILES.observations), observationRows.map((r) => JSON.stringify(r)).join('\n') + (observationRows.length ? '\n' : ''), { flag: 'a' });
-  await writeFile(join(dir, CITATION_FILES.epochs), epochRows.map((r) => JSON.stringify(r)).join('\n') + (epochRows.length ? '\n' : ''), { flag: 'a' });
 
   // Browser engines hold a browser and maybe an Xvfb display for the whole
   // epoch; release both once the tallies are written.

package/src/citations/stats.js

@@ -72,3 +72,66 @@ export function describeChange(current, baseline) {
   if (!baseline) return `first epoch: citation rate ${fmt(current)}`;
   return `citation rate moved from ${fmt(baseline)} to ${fmt(current)} between epochs`;
 }
+
+// === v2 statistics (DP-0045): the borrow-list math that makes alerts trustworthy ===
+
+/** Wald sequential probability ratio test for a decline from baseline rate.
+ * Returns { decision: 'decline' | 'accept' | 'continue', evidence: number }.
+ * The SPRT fires the moment the evidence crosses either threshold — no fixed
+ * n needed, which is exactly what monitoring at unknown-noise surfaces wants.
+ * Thresholds follow the standard H0: p = p0 (baseline) vs H1: p = p1 (decline). */
+export function sprtDecline(k, n, baselineRate, declineRate = baselineRate * 0.5, alpha = 0.05, beta = 0.10) {
+  if (n === 0 || k > n) return { decision: 'continue', evidence: 0 };
+  const p0 = Math.max(baselineRate, 0.001);
+  const p1 = Math.max(declineRate, 0.001);
+  // Log-likelihood ratio
+  const llr = k * Math.log(p1 / p0) + (n - k) * Math.log((1 - p1) / (1 - p0));
+  const upper = Math.log((1 - beta) / alpha);
+  const lower = Math.log(beta / (1 - alpha));
+  // Positive LLR favors H1 (the rate has declined); negative favors H0 (still at baseline).
+  if (llr >= upper) return { decision: 'decline', evidence: Number(llr.toFixed(4)) };
+  if (llr <= lower) return { decision: 'stable', evidence: Number(llr.toFixed(4)) };
+  return { decision: 'continue', evidence: Number(llr.toFixed(4)) };
+}
+
+/** Bayesian online changepoint detection (simplified run-length posterior).
+ * Takes a binary sequence (1 = cited, 0 = not cited) and returns the most
+ * likely changepoint index, or null if no changepoint is detected.
+ * This is the "when did it shift" answer that epoch-over-epoch comparison
+ * can never give. */
+export function detectChangepoint(observations, { hazard = 0.1, threshold = 0.3 } = {}) {
+  if (observations.length < 6) return null;
+  const n = observations.length;
+  // Simple two-segment maximum likelihood: for each split point, compute
+  // the log-likelihood of segment 1 and segment 2 under Bernoulli models.
+  // The best split is the changepoint candidate.
+  let bestSplit = null;
+  let bestScore = -Infinity;
+  for (let split = 3; split <= n - 3; split += 1) {
+    const seg1 = observations.slice(0, split);
+    const seg2 = observations.slice(split);
+    const p1 = (seg1.reduce((a, b) => a + b, 0) + 1) / (seg1.length + 2);
+    const p2 = (seg2.reduce((a, b) => a + b, 0) + 1) / (seg2.length + 2);
+    // Skip trivially-identical segments
+    if (Math.abs(p1 - p2) < threshold) continue;
+    const ll1 = seg1.reduce((acc, x) => acc + Math.log(x ? p1 : 1 - p1), 0);
+    const ll2 = seg2.reduce((acc, x) => acc + Math.log(x ? p2 : 1 - p2), 0);
+    // Penalize the split by the hazard rate
+    const score = ll1 + ll2 + Math.log(1 - hazard) * (n - split - 1) + Math.log(hazard);
+    if (score > bestScore) { bestScore = score; bestSplit = split; }
+  }
+  return bestSplit;
+}
+
+/** Laplace-smoothed rate for a rolling window of observations. Used for the
+ * trend sparkline in reports — shows the recent direction without claiming
+ * statistical significance. */
+export function rollingRate(observations, windowSize = 5) {
+  const rates = [];
+  for (let i = 0; i < observations.length; i += 1) {
+    const start = Math.max(0, i - windowSize + 1);
+    const window = observations.slice(start, i + 1);
+    rates.push(smoothedRate(window.reduce((a, b) => a + b, 0), window.length));
+  }
+  return rates;
+}

package/src/citations/watch.js

@@ -0,0 +1,85 @@
+// The citation watch: register a cron entry that runs the epoch, generates
+// the report, and alerts on decline. This is what turns a measurement script
+// into a monitoring product — the buyer sets it up once and gets weekly
+// evidence in their inbox.
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join, resolve, basename } from 'node:path';
+import { execFile } from 'node:child_process';
+import { homedir } from 'node:os';
+
+const CADENCE_HOURS = { hourly: 1, daily: 24, weekly: 168, monthly: 720 };
+
+export async function registerWatch({ panelPath, cadence = 'weekly', dir, out }) {
+  if (!CADENCE_HOURS[cadence]) {
+    throw new Error(`unknown cadence: ${cadence} (known: ${Object.keys(CADENCE_HOURS).join(', ')})`);
+  }
+  const absolute = resolve(panelPath);
+  const panelName = basename(absolute, '.json');
+  const agentlinkopsBin = process.argv[1] ?? 'agentlinkops';
+  const ledgerDir = dir ?? join(homedir(), '.agentlinkops');
+  const reportDir = out ?? join(ledgerDir, 'citations', 'reports');
+
+  // The runner: epoch + report + alert in one shell line. The report path is
+  // deterministic so the alert hook can find the latest artifact.
+  const command = [
+    `cd ${join(absolute, '..')}`,
+    `&& ${agentlinkopsBin} citation run ${absolute}`,
+    `&& ${agentlinkopsBin} citation report ${absolute}`,
+    // The alert hook reads the latest epoch rows; if any cell declined,
+    // the report is flagged (delivery is a follow-up: email or webhook).
+    `&& ${agentlinkopsBin} citation alert ${absolute}`,
+  ].join(' ');
+
+  // Cron line: m h dom mon dow command — weekly on Monday at 09:17 UTC,
+  // daily at 09:17, hourly at minute 17 (off the :00 mark per convention).
+  const cronSpec = cadence === 'weekly' ? '17 9 * * 1'
+    : cadence === 'daily' ? '17 9 * * *'
+    : cadence === 'hourly' ? '17 * * * *'
+    : '17 9 1 * *'; // monthly
+
+  const marker = `# agentlinkops:citation-watch:${panelName}`;
+  const cronLine = `${cronSpec} ${command} 2>&1 | logger -t agentlinkops-citation ${marker}`;
+
+  // Read the existing crontab, replace our entry if present, append otherwise.
+  const existing = await new Promise((r) => {
+    execFile('crontab', ['-l'], (err, stdout) => r(err ? '' : stdout));
+  });
+  const lines = existing.split('\n').filter((l) => !l.includes(marker));
+  lines.push(cronLine);
+  await new Promise((r, j) => {
+    execFile('crontab', ['-'], { input: lines.join('\n') + '\n' }, (err) => err ? j(err) : r());
+  });
+
+  return {
+    cadence,
+    cronSpec,
+    panelPath: absolute,
+    marker,
+    next: cadence === 'weekly' ? 'next Monday 09:17 UTC' : `every ${CADENCE_HOURS[cadence]}h at :17`,
+  };
+}
+
+export async function listWatches() {
+  const existing = await new Promise((r) => {
+    execFile('crontab', ['-l'], (err, stdout) => r(err ? '' : stdout));
+  });
+  return existing.split('\n')
+    .filter((l) => l.includes('# agentlinkops:citation-watch:'))
+    .map((l) => {
+      const marker = l.match(/agentlinkops:citation-watch:(\S+)/);
+      const cronPart = l.split(/\s+/).slice(0, 5).join(' ');
+      return { panel: marker?.[1] ?? 'unknown', cron: cronPart };
+    });
+}
+
+export async function removeWatch(panelName) {
+  const marker = `# agentlinkops:citation-watch:${panelName}`;
+  const existing = await new Promise((r) => {
+    execFile('crontab', ['-l'], (err, stdout) => r(err ? '' : stdout));
+  });
+  const lines = existing.split('\n').filter((l) => !l.includes(marker));
+  await new Promise((r, j) => {
+    execFile('crontab', ['-'], { input: lines.join('\n') + '\n' }, (err) => err ? j(err) : r());
+  });
+  return { removed: panelName };
+}