@fitlab-ai/agent-infra 0.9.2 → 0.9.4

package/lib/process-data/normalize.ts

@@ -0,0 +1,202 @@
+import { sha256 } from './store.ts';
+
+import type { CapturedObject, JsonValue, NormalizedKind, NormalizedRecord } from './types.ts';
+
+const TASK_ID_RE = /TASK-\d{8}-\d{6}/;
+const ACTIVITY_RE = /^- (\d{4}-\d{2}-\d{2} [^ ]+) — \*\*(.+?)\*\* by ([^ ]+) — (.+)$/gm;
+const LEDGER_RE = /^\|\s*((?:AN|PL|CD)-\d+)\s*\|\s*(analysis|plan|code)\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|$/gm;
+const RULING_RE = /^###\s+(HDR-\d+)\s*$([\s\S]*?)(?=^###\s+|^##\s+|(?![\s\S]))/gm;
+const CONFLICT_STATUSES = new Set(['open', 'accepted', 'adjusted', 'refuted', 'cannot-judge', 'needs-human-decision']);
+
+function recordId(kind: NormalizedKind, identity: string, suffix = ''): string {
+  return `${kind}:${sha256(`${identity}\0${suffix}`).slice(0, 24)}`;
+}
+
+function canonicalActor(raw: string): string | null {
+  const lowered = raw.toLowerCase();
+  const aliases: Record<string, string> = {
+    'claude-code': 'claude',
+    'antigravity-cli': 'antigravity',
+    'gemini-cli': 'gemini',
+    gemini: 'gemini'
+  };
+  if (['claude', 'codex', 'antigravity', 'opencode', 'cursor', 'human'].includes(lowered)) return lowered;
+  return aliases[lowered] ?? null;
+}
+
+function nextOccurrence(occurrences: Map<string, number>, identity: string): string {
+  const occurrence = (occurrences.get(identity) ?? 0) + 1;
+  occurrences.set(identity, occurrence);
+  return occurrence === 1 ? '' : `occurrence=${occurrence}`;
+}
+
+function normalizePlatformPage(object: CapturedObject): NormalizedRecord[] {
+  if (!object.content) return [];
+  try {
+    const page = JSON.parse(object.content) as unknown;
+    if (!Array.isArray(page)) return [];
+    return page.map((entry, index) => {
+      const value = entry as Record<string, unknown>;
+      const identity = String(value.id ?? value.node_id ?? value.sha ?? value.number ?? `${object.sourceIdentity}:${index}`);
+      const platformIdentity = /\/issues\?/.test(object.sourceIdentity) && typeof value.number === 'number'
+        ? `issue:${value.number}`
+        : /\/pulls\?/.test(object.sourceIdentity) && typeof value.number === 'number'
+          ? `pr:${value.number}`
+          : /\/issues\/\d+\/comments/.test(object.sourceIdentity)
+            ? `issue-comment:${identity}`
+            : /\/pulls\/\d+\/reviews/.test(object.sourceIdentity)
+              ? `review:${identity}`
+              : /\/pulls\/\d+\/commits/.test(object.sourceIdentity)
+                ? `commit:${identity}`
+                : `platform:${identity}`;
+      const binding = typeof value.body === 'string' ? value.body.match(TASK_ID_RE)?.[0] : undefined;
+      return {
+        recordId: recordId('platform-resource', object.sourceIdentity, identity),
+        kind: 'platform-resource',
+        sourceIdentity: platformIdentity,
+        sourceSha256: object.sha256,
+        ...(binding ? { binding } : {}),
+        data: entry as JsonValue
+      };
+    });
+  } catch {
+    return [];
+  }
+}
+
+function normalizeObjects(objects: CapturedObject[]): NormalizedRecord[] {
+  const records: NormalizedRecord[] = [];
+  for (const object of objects) {
+    if (object.disposition?.state && object.disposition.state !== 'included') {
+      records.push({
+        recordId: recordId('missing', object.sourceIdentity),
+        kind: 'missing',
+        sourceIdentity: object.sourceIdentity,
+        sourceSha256: object.sha256,
+        data: object.disposition.state === 'excluded-sensitive'
+          ? { reason: 'privacy-excluded', ruleId: object.disposition.ruleId }
+          : { reason: object.disposition.reason }
+      });
+      continue;
+    }
+    if (object.sourceKind === 'github-rest') {
+      records.push(...normalizePlatformPage(object));
+      continue;
+    }
+    if (object.sourceKind === 'operational-report') {
+      records.push({
+        recordId: recordId('operational-report', object.sourceIdentity),
+        kind: 'operational-report',
+        sourceIdentity: object.sourceIdentity,
+        sourceSha256: object.sha256
+      });
+      continue;
+    }
+    if (object.sourceKind === 'structured-telemetry') {
+      let data: JsonValue = { availability: 'unknown' };
+      try {
+        data = JSON.parse(object.content ?? '{}') as JsonValue;
+      } catch {
+        data = { availability: 'unavailable', reason: 'invalid receipt JSON' };
+      }
+      records.push({
+        recordId: recordId('telemetry', object.sourceIdentity),
+        kind: 'telemetry',
+        sourceIdentity: object.sourceIdentity,
+        sourceSha256: object.sha256,
+        data
+      });
+      continue;
+    }
+
+    const taskId = object.sourceIdentity.match(TASK_ID_RE)?.[0];
+    if (/\/task\.md$/.test(object.sourceIdentity) && taskId) {
+      const activityOccurrences = new Map<string, number>();
+      const findingOccurrences = new Map<string, number>();
+      const issueNumber = object.content?.match(/^issue_number:\s*(\d+)$/m)?.[1];
+      const agentInfraVersion = object.content?.match(/^agent_infra_version:\s*(\S+)$/m)?.[1];
+      records.push({
+        recordId: recordId('task', taskId),
+        kind: 'task',
+        sourceIdentity: taskId,
+        sourceSha256: object.sha256,
+        ...(issueNumber ? { binding: `issue:${issueNumber}` } : {}),
+        data: {
+          agentInfraVersion: agentInfraVersion
+            ? { state: 'known', value: agentInfraVersion }
+            : { state: 'unknown', reason: 'legacy-task-missing-agent-infra-version' }
+        }
+      });
+      for (const match of object.content?.matchAll(ACTIVITY_RE) ?? []) {
+        const rawActor = match[3]!;
+        const identity = `${match[1]}:${match[2]}`;
+        const occurrence = nextOccurrence(activityOccurrences, identity);
+        records.push({
+          recordId: recordId('lifecycle-event', taskId, occurrence ? `${identity}\0${occurrence}` : identity),
+          kind: 'lifecycle-event',
+          sourceIdentity: `${taskId}#activity=${match[1]}`,
+          sourceSha256: object.sha256,
+          data: {
+            timestamp: match[1]!,
+            action: match[2]!,
+            actorRaw: rawActor,
+            actorCanonical: canonicalActor(rawActor),
+            note: match[4]!
+          }
+        });
+      }
+      for (const match of object.content?.matchAll(LEDGER_RE) ?? []) {
+        const findingId = match[1]!.trim();
+        const stage = match[2]!.trim();
+        const round = match[3]!.trim();
+        const severity = match[4]!.trim();
+        const status = match[5]!.trim();
+        const evidence = match[6]!.trim();
+        const identity = `${taskId}#finding=${findingId}`;
+        const occurrence = nextOccurrence(findingOccurrences, identity);
+        const data = { findingId, stage, round, severity, status, evidence };
+        records.push({
+          recordId: recordId('review-finding', identity, occurrence),
+          kind: 'review-finding',
+          sourceIdentity: identity,
+          sourceSha256: object.sha256,
+          binding: taskId,
+          data
+        });
+        if (status && CONFLICT_STATUSES.has(status)) {
+          records.push({
+            recordId: recordId('conflict', identity, occurrence),
+            kind: 'conflict',
+            sourceIdentity: `${identity}#status=${status}`,
+            sourceSha256: object.sha256,
+            binding: taskId,
+            data
+          });
+        }
+      }
+      for (const match of object.content?.matchAll(RULING_RE) ?? []) {
+        const rulingId = match[1]!;
+        const identity = `${taskId}#ruling=${rulingId}`;
+        records.push({
+          recordId: recordId('human-ruling', identity),
+          kind: 'human-ruling',
+          sourceIdentity: identity,
+          sourceSha256: object.sha256,
+          binding: taskId,
+          data: { rulingId, evidence: match[2]!.trim() }
+        });
+      }
+      continue;
+    }
+    records.push({
+      recordId: recordId('artifact', object.sourceIdentity),
+      kind: 'artifact',
+      sourceIdentity: object.sourceIdentity,
+      sourceSha256: object.sha256,
+      ...(taskId ? { binding: taskId } : {})
+    });
+  }
+  return records.sort((left, right) => left.recordId.localeCompare(right.recordId));
+}
+
+export { normalizeObjects };

package/lib/process-data/privacy.ts

@@ -0,0 +1,41 @@
+import { sha256 } from './store.ts';
+
+import type { CapturedObject, PrivacyDisposition, SourceKind } from './types.ts';
+
+type ScannedDisposition =
+  | Extract<PrivacyDisposition, { state: 'included' }>
+  | Extract<PrivacyDisposition, { state: 'excluded-sensitive' }>;
+
+const RULES = [
+  { id: 'private-key', pattern: /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/ },
+  { id: 'github-token', pattern: /\b(?:gh[opusr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/ },
+  { id: 'bearer-token', pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i },
+  { id: 'aws-access-key', pattern: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
+  { id: 'slack-token', pattern: /\bxox[abprs]-[A-Za-z0-9-]{10,}\b/ },
+  { id: 'slack-webhook', pattern: /https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]{20,}/i },
+  { id: 'credential-field', pattern: /(?:api[_-]?key|password|secret)\s*[:=]\s*["']?[^\s"']{8,}/i }
+] as const;
+
+function inspectPrivacy(content: string): ScannedDisposition {
+  for (const rule of RULES) {
+    if (rule.pattern.test(content)) return { state: 'excluded-sensitive', ruleId: rule.id };
+  }
+  return { state: 'included' };
+}
+
+function captureText(sourceKind: SourceKind, sourceIdentity: string, content: string): CapturedObject {
+  const bytes = Buffer.byteLength(content);
+  const digest = sha256(Buffer.from(content, 'utf8'));
+  const disposition = inspectPrivacy(content);
+  return disposition.state === 'included'
+    ? { sourceKind, sourceIdentity, sha256: digest, bytes, content, disposition }
+    : { sourceKind, sourceIdentity, sha256: digest, bytes, disposition };
+}
+
+function redactExcerpt(content: string, maxBytes = 4096): string | null {
+  if (inspectPrivacy(content).state !== 'included') return null;
+  const buffer = Buffer.from(content, 'utf8');
+  return buffer.subarray(0, maxBytes).toString('utf8').replace(/\b(token|password|secret)=\S+/gi, '$1=[REDACTED]');
+}
+
+export { captureText, inspectPrivacy, redactExcerpt };

package/lib/process-data/reconcile.ts

@@ -0,0 +1,110 @@
+import { sha256 } from './store.ts';
+
+import type { NormalizedRecord, QualityFinding, RepairAction, SnapshotManifest } from './types.ts';
+
+function reconcileRecords(
+  records: NormalizedRecord[],
+  scope: SnapshotManifest['scope']
+): { findings: QualityFinding[]; repairs: RepairAction[] } {
+  const findings: QualityFinding[] = [];
+  const repairs: RepairAction[] = [];
+  const findingKeys = new Set<string>();
+  const addFinding = (
+    category: QualityFinding['category'],
+    severity: QualityFinding['severity'],
+    identities: string[],
+    repairable: boolean,
+    reason: string
+  ) => {
+    const sortedIdentities = [...identities].sort();
+    const key = `${category}:${sortedIdentities.join('\0')}`;
+    if (findingKeys.has(key)) return;
+    findingKeys.add(key);
+    findings.push({
+      findingId: `quality:${sha256(key).slice(0, 20)}`,
+      category,
+      severity,
+      identities: sortedIdentities,
+      repairable,
+      reason
+    });
+  };
+  const byIdentity = new Map<string, NormalizedRecord[]>();
+  for (const record of records) {
+    const group = byIdentity.get(record.sourceIdentity) ?? [];
+    group.push(record);
+    byIdentity.set(record.sourceIdentity, group);
+    if (
+      record.kind === 'missing' &&
+      record.data !== null &&
+      typeof record.data === 'object' &&
+      !Array.isArray(record.data) &&
+      record.data.reason === 'privacy-excluded'
+    ) {
+      addFinding('privacy-excluded', 'warning', [record.sourceIdentity], false, 'Sensitive content was excluded by policy');
+    } else if (record.kind === 'missing') {
+      addFinding('unrecoverable', 'info', [record.sourceIdentity], false, 'Source body is unavailable under the capture policy');
+    }
+  }
+
+  for (const [identity, group] of byIdentity) {
+    if (group.length < 2) continue;
+    addFinding('duplicate-identity', 'error', group.map((record) => record.recordId), false, 'Multiple records claim the same stable identity');
+    if (group.some((record) => record.kind === 'platform-resource') && new Set(group.map((record) => record.sourceSha256)).size > 1) {
+      addFinding('mutable-remote', 'warning', group.map((record) => record.recordId), false, `Remote identity changed content: ${identity}`);
+    }
+  }
+
+  const tasks = records.filter((record) => record.kind === 'task');
+  const resources = records.filter((record) => record.kind === 'platform-resource');
+  const taskByIdentity = new Map(tasks.map((task) => [task.sourceIdentity, task]));
+
+  for (const task of tasks) {
+    const data = task.data && typeof task.data === 'object' && !Array.isArray(task.data) ? task.data : null;
+    const version = data?.agentInfraVersion;
+    if (!version || typeof version !== 'object' || Array.isArray(version) || version.state !== 'known') {
+      addFinding('schema-difference', 'info', [task.recordId], false, 'Task uses a legacy schema without a known agent_infra_version');
+    }
+    if (scope !== 'all' || !task.binding) continue;
+    const identityCandidates = resources.filter((record) => record.sourceIdentity === task.binding);
+    if (identityCandidates.length === 0) {
+      addFinding('missing-remote', 'error', [task.recordId, task.binding], false, 'Task binding has no captured platform resource');
+    }
+    if (identityCandidates.length > 1) {
+      addFinding('binding-conflict', 'error', [task.recordId, ...identityCandidates.map((record) => record.recordId)], false, 'Multiple platform resources bind to one task');
+    }
+    for (const remote of identityCandidates) {
+      if (remote.binding && remote.binding !== task.sourceIdentity) {
+        addFinding('content-mismatch', 'warning', [task.recordId, remote.recordId], false, 'Local and remote binding identities disagree');
+      }
+    }
+    const candidates = identityCandidates.filter((record) => record.binding === task.sourceIdentity);
+    if (candidates.length === 1 && (byIdentity.get(candidates[0]!.sourceIdentity)?.length ?? 0) === 1) {
+      const target = candidates[0]!;
+      repairs.push({
+        repairId: `repair:${sha256(`${task.recordId}:${target.recordId}`).slice(0, 20)}`,
+        operation: 'link',
+        sourceRecordId: task.recordId,
+        targetRecordId: target.recordId,
+        preconditionSha256: sha256(`${task.sourceSha256}:${target.sourceSha256}`)
+      });
+    }
+  }
+
+  for (const resource of scope === 'all' ? resources : []) {
+    if (resource.binding?.startsWith('TASK-') && !taskByIdentity.has(resource.binding)) {
+      addFinding('missing-local', 'error', [resource.recordId, resource.binding], false, 'Platform resource binds to a task that is not captured locally');
+    }
+    const localClaims = tasks.filter((task) => task.binding === resource.sourceIdentity);
+    if (localClaims.length > 1 || (resource.binding && localClaims.length > 0 && !localClaims.some((task) => task.sourceIdentity === resource.binding))) {
+      addFinding('binding-conflict', 'error', [resource.recordId, ...localClaims.map((task) => task.recordId)], false, 'Local and remote records claim incompatible bindings');
+    }
+  }
+
+  return {
+    findings: findings.sort((left, right) => left.findingId.localeCompare(right.findingId)),
+    repairs: repairs.sort((left, right) => left.repairId.localeCompare(right.repairId))
+  };
+}
+
+export { reconcileRecords };

package/lib/process-data/repair.ts

@@ -0,0 +1,64 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { canonicalJsonBytes, sha256, verifySnapshot } from './store.ts';
+
+import type { RepairAction } from './types.ts';
+import type { NormalizedRecord } from './types.ts';
+
+type RepairResult = { applied: boolean; repairId: string | null; actionCount: number; path?: string };
+
+function readRepairActions(snapshotPath: string): RepairAction[] {
+  const filePath = path.join(snapshotPath, 'repair-manifest.jsonl');
+  if (!fs.existsSync(filePath)) return [];
+  return fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean).map((line) => {
+    const { schema: _schema, ...action } = JSON.parse(line) as { schema: string } & RepairAction;
+    return action;
+  });
+}
+
+function repairSnapshot(root: string, snapshotId: string, apply: boolean): RepairResult {
+  const verified = verifySnapshot(root, snapshotId);
+  if (!verified.ok || !verified.snapshotPath) throw new Error(`snapshot verification failed: ${verified.errors.join('; ')}`);
+  const actions = readRepairActions(verified.snapshotPath);
+  const records = new Map(
+    fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8').split('\n').filter(Boolean).map((line) => {
+      const record = JSON.parse(line) as { schema: string } & NormalizedRecord;
+      return [record.recordId, record] as const;
+    })
+  );
+  for (const action of actions) {
+    const source = records.get(action.sourceRecordId);
+    const target = records.get(action.targetRecordId);
+    if (!source || !target) throw new Error(`repair references a missing record: ${action.repairId}`);
+    if (sha256(`${source.sourceSha256}:${target.sourceSha256}`) !== action.preconditionSha256) {
+      throw new Error(`repair precondition failed: ${action.repairId}`);
+    }
+  }
+  if (!apply || actions.length === 0) return { applied: false, repairId: null, actionCount: actions.length };
+
+  const digest = sha256(canonicalJsonBytes(actions));
+  const repairId = `repair-${digest.slice(0, 16)}`;
+  const repairDir = path.join(root, 'repairs', snapshotId, repairId);
+  const manifest = `${canonicalJsonBytes({ schema: 'repair-application/v1', snapshotId, repairId, actionCount: actions.length, overlaySha256: digest }).toString('utf8')}\n`;
+  const overlay = actions.map((action) => `${canonicalJsonBytes({ schema: 'normalized-overlay/v1', ...action }).toString('utf8')}\n`).join('');
+  if (fs.existsSync(repairDir)) {
+    if (fs.readFileSync(path.join(repairDir, 'manifest.json'), 'utf8') !== manifest) throw new Error(`repair collision: ${repairId}`);
+    return { applied: false, repairId, actionCount: actions.length, path: repairDir };
+  }
+  fs.mkdirSync(repairDir, { recursive: true, mode: 0o700 });
+  fs.writeFileSync(path.join(repairDir, 'manifest.json'), manifest, { mode: 0o600, flag: 'wx' });
+  fs.writeFileSync(path.join(repairDir, 'overlay.jsonl'), overlay, { mode: 0o600, flag: 'wx' });
+  return { applied: true, repairId, actionCount: actions.length, path: repairDir };
+}
+
+function readAppliedOverlays(root: string, snapshotId: string): string {
+  const base = path.join(root, 'repairs', snapshotId);
+  if (!fs.existsSync(base)) return '';
+  return fs.readdirSync(base).sort().map((entry) => {
+    const overlay = path.join(base, entry, 'overlay.jsonl');
+    return fs.existsSync(overlay) ? fs.readFileSync(overlay, 'utf8') : '';
+  }).join('');
+}
+
+export { readAppliedOverlays, repairSnapshot };