@aria_asi/cli 0.2.40 → 0.2.41

package/scripts/session-quality-backfill.mjs

@@ -0,0 +1,253 @@
+#!/usr/bin/env node
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { analyzeDomainOutputQuality } from '../hooks/lib/domain-output-quality.mjs';
+
+const repoRoot = process.cwd();
+const runId = new Date().toISOString().replace(/[:.]/g, '-');
+const outputRoot = path.join(repoRoot, 'artifacts', 'session-quality-backfill', runId);
+
+function argNumber(name, fallback) {
+  const prefix = `--${name}=`;
+  const raw = process.argv.find((arg) => arg.startsWith(prefix));
+  if (!raw) return fallback;
+  const value = Number(raw.slice(prefix.length));
+  return Number.isFinite(value) ? value : fallback;
+}
+
+const recentDays = argNumber('recent-days', 14);
+const priorDays = argNumber('prior-days', 14);
+const maxFiles = argNumber('max-files', 600);
+const maxMessages = argNumber('max-messages', 1600);
+const now = Date.now();
+const recentStart = now - recentDays * 24 * 60 * 60 * 1000;
+const priorStart = recentStart - priorDays * 24 * 60 * 60 * 1000;
+
+function sha256(value) {
+  return createHash('sha256').update(String(value ?? '')).digest('hex');
+}
+
+function ensureDir(dir) {
+  mkdirSync(dir, { recursive: true, mode: 0o755 });
+}
+
+function walkFiles(root, predicate, out = []) {
+  if (!existsSync(root) || out.length >= maxFiles) return out;
+  for (const entry of readdirSync(root, { withFileTypes: true })) {
+    if (out.length >= maxFiles) break;
+    const full = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      walkFiles(full, predicate, out);
+    } else if (predicate(full)) {
+      out.push(full);
+    }
+  }
+  return out;
+}
+
+function parseJsonLine(line) {
+  try {
+    return JSON.parse(line);
+  } catch {
+    return null;
+  }
+}
+
+function textFromContent(content) {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  return content
+    .map((part) => {
+      if (!part || typeof part !== 'object') return '';
+      if (typeof part.text === 'string') return part.text;
+      if (part.type === 'output_text' && typeof part.text === 'string') return part.text;
+      if (part.type === 'text' && typeof part.text === 'string') return part.text;
+      return '';
+    })
+    .filter(Boolean)
+    .join('\n')
+    .trim();
+}
+
+function extractAssistantText(record, surface) {
+  if (!record || typeof record !== 'object') return '';
+  if (surface === 'claude') {
+    if (record.type !== 'assistant' || record.message?.role !== 'assistant') return '';
+    return textFromContent(record.message.content);
+  }
+  if (surface === 'codex') {
+    const payload = record.payload || {};
+    if (record.type !== 'response_item' || payload.role !== 'assistant') return '';
+    return textFromContent(payload.content);
+  }
+  return '';
+}
+
+function bucketFor(ts) {
+  const time = Date.parse(ts || '');
+  if (!Number.isFinite(time)) return null;
+  if (time >= recentStart && time <= now) return 'recent';
+  if (time >= priorStart && time < recentStart) return 'prior';
+  return null;
+}
+
+function scoreText(text) {
+  const source = String(text || '');
+  const runtimeBlockedOutput = /\bAria runtime blocked this output\b|\bRemaining blockers:\b|\bRecovery attempts:\b/i.test(source);
+  const checks = {
+    notRuntimeBlock: !runtimeBlockedOutput,
+    concreteNextAction: /\b(next action|next step|do this|implement|verify|run|inspect|measure|continue)\b/i.test(source),
+    evidenceBoundary: /\b(evidence|observed|verified|unverified|bounded|not measured|proof|artifact|metric|status)\b/i.test(source),
+    expectedObserved: /\b(expected|observed|predicate|actual|pass|fail|threshold|exit=0)\b/i.test(source),
+    qaCorrection: /\b(QA|finding|correct|correction|repair|harden|verify|re-test|rescore)\b/i.test(source),
+    riskTradeoff: /\b(risk|tradeoff|failure mode|blast radius|cost|latency|scope|danger)\b/i.test(source),
+    ownerReadable: source.length >= 120 && source.length <= 3000 && /\n|:/.test(source),
+    avoidsAskOnly: !/\bwhat would you like me to do next\b|\bhow would you like me to proceed\b/i.test(source),
+    avoidsFalseClosure: !/\b(production ready|fully done|guaranteed|complete)\b/i.test(source),
+    avoidsRawInternals: !/\b(skillExecutionReceipt|phaseReceipts|executedOperatorIdsHash|raw json)\b/i.test(source),
+    learningLoop: /\b(lesson|learning|next selection|feedback|loop|reflexion|record)\b/i.test(source),
+    specificVerification: /\b(exit=0|100\/100|\d+\/\d+|status=|sha256|screenshot|metric|test|smoke|ledger)\b/i.test(source),
+  };
+  const domain = analyzeDomainOutputQuality(source, {});
+  const base = Object.values(checks).filter(Boolean).length;
+  const penalty = (runtimeBlockedOutput ? Object.keys(checks).length : 0) + domain.blockers.length * 2 + domain.warnings.length * 0.5;
+  return {
+    score: Math.max(0, Number((base - penalty).toFixed(2))),
+    maxScore: Object.keys(checks).length,
+    checks,
+    blockerCount: domain.blockers.length,
+    warningCount: domain.warnings.length,
+    blockers: domain.blockers.slice(0, 6),
+    warnings: domain.warnings.slice(0, 6),
+    chars: source.length,
+    sha256: sha256(source),
+  };
+}
+
+function scanFile(filePath, surface, rows, bucketCounts) {
+  const raw = readFileSync(filePath, 'utf8');
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    const record = parseJsonLine(line);
+    if (!record) continue;
+    const bucket = bucketFor(record.timestamp || record.payload?.timestamp || record.message?.timestamp);
+    if (!bucket) continue;
+    if ((bucketCounts[bucket] || 0) >= maxMessages) continue;
+    const text = extractAssistantText(record, surface);
+    if (!text || text.length < 80) continue;
+    const score = scoreText(text);
+    rows.push({
+      bucket,
+      surface,
+      timestamp: record.timestamp || record.message?.timestamp || null,
+      file: filePath,
+      sessionId: record.sessionId || record.payload?.session_meta?.payload?.id || record.payload?.sessionId || record.message?.sessionId || null,
+      textSha256: score.sha256,
+      textPreview: text.slice(0, 260),
+      score,
+    });
+    bucketCounts[bucket] = (bucketCounts[bucket] || 0) + 1;
+  }
+}
+
+function mean(values) {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+}
+
+function summarize(rows) {
+  const scores = rows.map((row) => row.score.score);
+  const checkRate = (name) => rows.length ? rows.filter((row) => row.score.checks[name]).length / rows.length : 0;
+  return {
+    count: rows.length,
+    meanScore: Number(mean(scores).toFixed(3)),
+    blockerRate: Number((mean(rows.map((row) => row.score.blockerCount > 0 ? 1 : 0)) * 100).toFixed(2)),
+    warningRate: Number((mean(rows.map((row) => row.score.warningCount > 0 ? 1 : 0)) * 100).toFixed(2)),
+    runtimeBlockRate: Number(((1 - checkRate('notRuntimeBlock')) * 100).toFixed(2)),
+    concreteNextActionRate: Number((checkRate('concreteNextAction') * 100).toFixed(2)),
+    evidenceBoundaryRate: Number((checkRate('evidenceBoundary') * 100).toFixed(2)),
+    qaCorrectionRate: Number((checkRate('qaCorrection') * 100).toFixed(2)),
+    learningLoopRate: Number((checkRate('learningLoop') * 100).toFixed(2)),
+    specificVerificationRate: Number((checkRate('specificVerification') * 100).toFixed(2)),
+  };
+}
+
+function renderMarkdown(summary, rows) {
+  const examples = [...rows]
+    .sort((a, b) => a.score.score - b.score.score)
+    .slice(0, 8)
+    .map((row) => `| ${row.bucket} | ${row.surface} | ${row.score.score} | ${row.score.blockers.join('; ') || '-'} | ${row.textPreview.replace(/\|/g, '/').replace(/\n/g, ' ')} |`);
+  return [
+    '# Session Quality Backfill',
+    '',
+    `- run_id: ${summary.runId}`,
+    `- recent_window: ${summary.windows.recent}`,
+    `- prior_window: ${summary.windows.prior}`,
+    `- evidence_boundary: ${summary.evidenceBoundary}`,
+    '',
+    '## Summary',
+    '',
+    '| Bucket | Count | Mean | Blocker % | Runtime block % | Next action % | Evidence % | QA % | Learning % | Specific verification % |',
+    '|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|',
+    `| prior | ${summary.buckets.prior.count} | ${summary.buckets.prior.meanScore} | ${summary.buckets.prior.blockerRate} | ${summary.buckets.prior.runtimeBlockRate} | ${summary.buckets.prior.concreteNextActionRate} | ${summary.buckets.prior.evidenceBoundaryRate} | ${summary.buckets.prior.qaCorrectionRate} | ${summary.buckets.prior.learningLoopRate} | ${summary.buckets.prior.specificVerificationRate} |`,
+    `| recent | ${summary.buckets.recent.count} | ${summary.buckets.recent.meanScore} | ${summary.buckets.recent.blockerRate} | ${summary.buckets.recent.runtimeBlockRate} | ${summary.buckets.recent.concreteNextActionRate} | ${summary.buckets.recent.evidenceBoundaryRate} | ${summary.buckets.recent.qaCorrectionRate} | ${summary.buckets.recent.learningLoopRate} | ${summary.buckets.recent.specificVerificationRate} |`,
+    '',
+    '## Lowest Scored Samples',
+    '',
+    '| Bucket | Surface | Score | Blockers | Preview |',
+    '|---|---|---:|---|---|',
+    ...examples,
+    '',
+  ].join('\n');
+}
+
+function main() {
+  ensureDir(outputRoot);
+  const rows = [];
+  const bucketCounts = { recent: 0, prior: 0 };
+  const codexRoot = path.join(os.homedir(), '.codex', 'sessions');
+  const claudeRoot = path.join(os.homedir(), '.claude', 'projects');
+  const codexFiles = walkFiles(codexRoot, (file) => file.endsWith('.jsonl'));
+  const claudeFiles = walkFiles(claudeRoot, (file) => file.endsWith('.jsonl'));
+  for (const file of [...codexFiles].sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)) {
+    if (bucketCounts.recent >= maxMessages && bucketCounts.prior >= maxMessages) break;
+    scanFile(file, 'codex', rows, bucketCounts);
+  }
+  for (const file of [...claudeFiles].sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)) {
+    if (bucketCounts.recent >= maxMessages && bucketCounts.prior >= maxMessages) break;
+    scanFile(file, 'claude', rows, bucketCounts);
+  }
+  const recent = rows.filter((row) => row.bucket === 'recent');
+  const prior = rows.filter((row) => row.bucket === 'prior');
+  const summary = {
+    schema: 'aria.session_quality_backfill.v1',
+    runId,
+    generatedAt: new Date().toISOString(),
+    evidenceBoundary: 'Backfill scores existing Claude/Codex assistant messages with deterministic heuristics. It is longitudinal directional evidence, not a controlled provider A/B.',
+    scanned: {
+      codexFiles: codexFiles.length,
+      claudeFiles: claudeFiles.length,
+      messages: rows.length,
+      maxFiles,
+      maxMessagesPerBucket: maxMessages,
+    },
+    windows: {
+      recent: `${new Date(recentStart).toISOString()}..${new Date(now).toISOString()}`,
+      prior: `${new Date(priorStart).toISOString()}..${new Date(recentStart).toISOString()}`,
+    },
+    buckets: {
+      prior: summarize(prior),
+      recent: summarize(recent),
+    },
+  };
+  const metricsPath = path.join(outputRoot, 'metrics.json');
+  const rowsPath = path.join(outputRoot, 'rows.jsonl');
+  const summaryPath = path.join(outputRoot, 'summary.md');
+  writeFileSync(metricsPath, JSON.stringify({ summary, rows }, null, 2));
+  writeFileSync(rowsPath, rows.map((row) => JSON.stringify(row)).join('\n') + (rows.length ? '\n' : ''));
+  writeFileSync(summaryPath, renderMarkdown(summary, rows));
+  console.log(JSON.stringify({ ok: true, summary, artifacts: { outputRoot, metricsPath, rowsPath, summaryPath } }, null, 2));
+}
+
+main();