@fitlab-ai/agent-infra 0.9.5 → 0.9.7

package/lib/process-data/index.ts

@@ -1,24 +1,26 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { execFileSync } from 'node:child_process';
 
 import { detectRepoRoot } from '../task/resolve-ref.ts';
-import { normalizeObjects } from './normalize.ts';
-import { reconcileRecords } from './reconcile.ts';
+import { acquireGitHubCheckpoint, commitGitHubCheckpoint, readGitHubCheckpoint, releaseGitHubCheckpoint } from './checkpoint.ts';
+import { materializeJson, materializeSnapshot } from './materialize.ts';
+import { normalizeResources } from './normalize.ts';
+import { reconcileRecords, reconcileResourceRecords } from './reconcile.ts';
 import { readAppliedOverlays, repairSnapshot } from './repair.ts';
-import { collectGitHubObjects, collectLocalObjects } from './sources.ts';
-import { canonicalJsonBytes, createObjectStore, publishSnapshot, verifySnapshot } from './store.ts';
+import { collectGitHubBoundary, resolveGitHubRepository } from './sources.ts';
+import { canonicalJsonBytes, createObjectStore, findSnapshot, publishSnapshotV2, verifySnapshot } from './store.ts';
+import { createGitHubClient } from '../platform/github-client.ts';
 
-import type { CapturedObject, NormalizedRecord, RestCollectionEvidence } from './types.ts';
+import type { NormalizedRecord } from './types.ts';
 
 const USAGE = `Usage: ai data <command> [options]
 
 Commands:
-  capture [--source all|local|github] [--root <dir>] [--include-excerpts]
+  capture [--source github] [--root <dir>] [--full-reconcile]
   verify <snapshot-id> [--root <dir>]
   audit <snapshot-id> [--root <dir>] [--format json|text]
   repair <snapshot-id> [--root <dir>] [--apply]
-  export <snapshot-id> [--root <dir>] [--repairs none|applied] [--output <file|->]
+  export <snapshot-id> [--root <dir>] [--repairs none|applied] [--as-of <ISO>] [--output <file|->]
 `;
 
 type ParsedArgs = { positionals: string[]; options: Map<string, string | true> };
@@ -64,14 +66,6 @@ function dataRoot(repoRoot: string, requested?: string): string {
   return resolved;
 }
 
-function repositoryIdentity(repoRoot: string): string {
-  try {
-    return execFileSync('git', ['config', '--get', 'remote.origin.url'], { cwd: repoRoot, encoding: 'utf8' }).trim() || path.basename(repoRoot);
-  } catch {
-    return path.basename(repoRoot);
-  }
-}
-
 function writeJson(value: unknown): void {
   process.stdout.write(`${JSON.stringify(value)}\n`);
 }
@@ -82,54 +76,106 @@ function optionString(parsed: ParsedArgs, name: string): string | undefined {
 }
 
 function capture(args: string[]): number {
-  const parsed = parseArgs(args, new Set(['--source', '--root']), new Set(['--include-excerpts']));
+  const parsed = parseArgs(args, new Set(['--source', '--root']), new Set(['--full-reconcile', '--include-excerpts']));
   if (parsed.positionals.length > 0) throw new Error('capture does not accept positional arguments');
-  const scope = optionString(parsed, '--source') ?? 'all';
-  if (!['all', 'local', 'github'].includes(scope)) throw new Error('--source must be all, local, or github');
+  const scope = optionString(parsed, '--source') ?? 'github';
+  if (scope !== 'github') throw new Error('new process-data capture only supports --source github; local data is already normalized and --source all is retired');
+  if (parsed.options.has('--include-excerpts')) throw new Error('--include-excerpts is not supported for GitHub-only capture');
   const repoRoot = detectRepoRoot();
   const root = dataRoot(repoRoot, optionString(parsed, '--root'));
-  const observedFrom = new Date().toISOString();
-  const objects: CapturedObject[] = [];
-  const endpoints: RestCollectionEvidence[] = [];
-  let repository = repositoryIdentity(repoRoot);
-
-  if (scope === 'all' || scope === 'local') {
-    const local = collectLocalObjects(repoRoot, { includeExcerpts: parsed.options.has('--include-excerpts') });
-    if (!local.ok) { writeJson({ ok: false, error: local.error }); return 2; }
-    objects.push(...local.value);
-  }
-  if (scope === 'all' || scope === 'github') {
-    const github = collectGitHubObjects(repoRoot);
+  const client = createGitHubClient();
+  const repositoryResult = resolveGitHubRepository(repoRoot, client);
+  if (!repositoryResult.ok) { writeJson({ ok: false, error: repositoryResult.error }); return 2; }
+  const repository = repositoryResult.value;
+  const leaseResult = acquireGitHubCheckpoint(root, repository);
+  if (!leaseResult.ok) { writeJson({ ok: false, error: leaseResult.error }); return 2; }
+  const lease = leaseResult.value;
+  try {
+    const checkpointResult = readGitHubCheckpoint(root, repository);
+    if (!checkpointResult.ok) { writeJson({ ok: false, error: checkpointResult.error }); return 2; }
+    const checkpoint = checkpointResult.value;
+    const full = parsed.options.has('--full-reconcile');
+    const observedFrom = new Date().toISOString();
+    const github = collectGitHubBoundary(repoRoot, {
+      client,
+      fromInclusive: full ? null : checkpoint?.watermark ?? null,
+      reconciliation: full ? 'full' : 'incremental'
+    });
     if (!github.ok) { writeJson({ ok: false, error: github.error }); return 2; }
-    objects.push(...github.value.objects);
-    endpoints.push(...github.value.endpoints);
-    repository = github.value.repository;
-  }
-
-  const store = createObjectStore(root);
-  for (const object of objects) {
-    if (object.content !== undefined && object.disposition?.state === 'included') {
-      const stored = store.put(Buffer.from(object.content, 'utf8'));
-      if (stored.sha256 !== object.sha256) throw new Error(`source hash mismatch: ${object.sourceIdentity}`);
+    const store = createObjectStore(root);
+    for (const object of github.value.objects) {
+      if (object.content !== undefined && object.disposition?.state === 'included') {
+        const stored = store.put(Buffer.from(object.content, 'utf8'));
+        if (stored.sha256 !== object.sha256) throw new Error(`source hash mismatch: ${object.sourceIdentity}`);
+      }
+    }
+    let parent: NormalizedRecord[] = [];
+    if (checkpoint) {
+      const parentPath = findSnapshot(root, checkpoint.snapshotId);
+      if (!parentPath) throw new Error(`checkpoint head snapshot is missing: ${checkpoint.snapshotId}`);
+      const materialized = materializeSnapshot(root, checkpoint.snapshotId);
+      if (!materialized.manifest || materialized.manifest.manifestSha256 !== checkpoint.manifestSha256 || materialized.manifest.watermark !== checkpoint.watermark) {
+        throw new Error('checkpoint does not match its verified snapshot head');
+      }
+      parent = materialized.records;
+    }
+    const current = normalizeResources(github.value.objects);
+    const reconciliation = reconcileResourceRecords(current, {
+      parent,
+      full,
+      deferred: github.value.deferred,
+      unavailable: github.value.unavailable
+    });
+    const quality = reconcileRecords(reconciliation.records, 'github');
+    const fromInclusive = checkpoint?.watermark ?? null;
+    const hasStrictSinceEndpoint = github.value.endpoints.some((endpoint) => endpoint.queryMode === 'strict-since');
+    const queryAfter = !full && fromInclusive && hasStrictSinceEndpoint
+      ? new Date(new Date(fromInclusive).getTime() - 1000).toISOString()
+      : null;
+    const responseDates = github.value.responseDates;
+    const published = publishSnapshotV2(root, {
+      scope: 'github',
+      repository,
+      observedFrom,
+      observedTo: github.value.watermark,
+      objects: github.value.objects.map(({ content: _content, ...object }) => object),
+      endpoints: github.value.endpoints,
+      excerptsEnabled: false,
+      records: reconciliation.records,
+      quality: quality.findings,
+      repairs: quality.repairs,
+      snapshotKind: checkpoint ? 'delta' : 'base',
+      parentSnapshotId: checkpoint?.snapshotId ?? null,
+      checkpointBefore: checkpoint?.snapshotId ?? null,
+      watermark: github.value.watermark,
+      window: { fromInclusive, queryAfter, toExclusive: github.value.watermark, precision: 'second' },
+      observation: {
+        cutoffSource: 'github-response-date',
+        preflightDate: github.value.preflightDate,
+        responseDates
+      },
+      coverage: { mode: 'boundary-reread-with-full-reconcile', absoluteCompleteness: false },
+      reconciliation: full ? 'full' : 'incremental',
+      operations: reconciliation.operations
+    });
+    const verified = verifySnapshot(root, published.snapshotId);
+    if (!verified.ok || !verified.manifest || verified.manifest.schema !== 'raw-manifest/v2') {
+      throw new Error(`published snapshot failed verification: ${verified.errors.join('; ')}`);
     }
+    const committed = commitGitHubCheckpoint(lease, {
+      schema: 'github-checkpoint/v1',
+      repository,
+      snapshotId: published.snapshotId,
+      watermark: github.value.watermark,
+      manifestSha256: verified.manifest.manifestSha256,
+      committedAt: new Date().toISOString()
+    });
+    if (!committed.ok) { writeJson({ ok: false, error: committed.error }); return 2; }
+    writeJson({ ok: true, snapshotId: published.snapshotId, scope, snapshotKind: checkpoint ? 'delta' : 'base', reconciliation: full ? 'full' : 'incremental', created: published.created, objects: github.value.objects.length, records: reconciliation.records.length, watermark: github.value.watermark, deferred: github.value.deferred.length });
+    return 0;
+  } finally {
+    releaseGitHubCheckpoint(lease);
   }
-  const records = normalizeObjects(objects);
-  const reconciliation = reconcileRecords(records, scope as 'all' | 'local' | 'github');
-  const observedTo = new Date().toISOString();
-  const published = publishSnapshot(root, {
-    scope: scope as 'all' | 'local' | 'github',
-    repository,
-    observedFrom,
-    observedTo,
-    objects: objects.map(({ content: _content, ...object }) => object),
-    endpoints,
-    excerptsEnabled: parsed.options.has('--include-excerpts'),
-    records,
-    quality: reconciliation.findings,
-    repairs: reconciliation.repairs
-  });
-  writeJson({ ok: true, snapshotId: published.snapshotId, scope, created: published.created, objects: objects.length, records: records.length });
-  return 0;
 }
 
 function verify(args: string[]): number {
@@ -175,7 +221,7 @@ function repair(args: string[]): number {
 }
 
 function exportSnapshot(args: string[]): number {
-  const parsed = parseArgs(args, new Set(['--root', '--repairs', '--output']), new Set());
+  const parsed = parseArgs(args, new Set(['--root', '--repairs', '--output', '--as-of']), new Set());
   if (parsed.positionals.length !== 1) throw new Error('export requires one snapshot id');
   const repairMode = optionString(parsed, '--repairs') ?? 'none';
   if (!['none', 'applied'].includes(repairMode)) throw new Error('--repairs must be none or applied');
@@ -184,7 +230,13 @@ function exportSnapshot(args: string[]): number {
   const root = dataRoot(repoRoot, optionString(parsed, '--root'));
   const verified = verifySnapshot(root, parsed.positionals[0]!);
   if (!verified.ok || !verified.snapshotPath) { writeJson(verified); return 1; }
-  let content = fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8');
+  let content: string;
+  if (verified.manifest?.schema === 'raw-manifest/v2') {
+    content = materializeJson(root, parsed.positionals[0]!, { asOf: optionString(parsed, '--as-of') });
+  } else {
+    if (optionString(parsed, '--as-of')) throw new Error('--as-of is only supported for v2 GitHub snapshots');
+    content = fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8');
+  }
   if (repairMode === 'applied') content += readAppliedOverlays(root, parsed.positionals[0]!);
   if (output === '-') process.stdout.write(content);
   else {

package/lib/process-data/materialize.ts

@@ -0,0 +1,83 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { findSnapshot, verifySnapshot } from './store.ts';
+import type { NormalizedRecord, SnapshotManifestV2 } from './types.ts';
+
+type MaterializeOptions = { asOf?: string };
+type MaterializeResult = { snapshotId: string; records: NormalizedRecord[]; manifest: SnapshotManifestV2 | null };
+
+function readRecords(snapshotPath: string, schema: 'normalized/v1' | 'normalized/v2'): NormalizedRecord[] {
+  const file = path.join(snapshotPath, 'normalized.jsonl');
+  if (!fs.existsSync(file)) return [];
+  return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
+    const value = JSON.parse(line) as { schema?: string } & NormalizedRecord;
+    if (value.schema !== schema) return [];
+    const { schema: _schema, ...record } = value;
+    return [record];
+  });
+}
+
+function materializeSnapshot(root: string, snapshotId: string, options: MaterializeOptions = {}): MaterializeResult {
+  const target = options.asOf ? new Date(options.asOf) : null;
+  if (target && !Number.isFinite(target.getTime())) throw new Error('as-of must be an ISO timestamp');
+  const chain: Array<{ id: string; path: string; manifest: SnapshotManifestV2 }> = [];
+  let currentId: string | null = snapshotId;
+  let repository: string | null = null;
+  while (currentId) {
+    const currentPath = findSnapshot(root, currentId);
+    if (!currentPath) throw new Error(`snapshot not found: ${currentId}`);
+    const verified = verifySnapshot(root, currentId);
+    if (!verified.ok || !verified.manifest) throw new Error(`snapshot is not verified: ${currentId}`);
+    if (verified.manifest.schema !== 'raw-manifest/v2') {
+      if (chain.length > 0) throw new Error('v1 and v2 snapshots cannot share a lineage');
+      return { snapshotId, records: readRecords(currentPath, 'normalized/v1'), manifest: null };
+    }
+    const manifest = verified.manifest;
+    if (repository && manifest.repository !== repository) throw new Error('snapshot lineage repository mismatch');
+    repository = manifest.repository;
+    chain.push({ id: currentId, path: currentPath, manifest });
+    currentId = manifest.parentSnapshotId;
+  }
+  if (chain.length === 0) throw new Error('snapshot lineage is empty');
+  const chronological = [...chain].reverse();
+  for (let index = 1; index < chronological.length; index += 1) {
+    if (new Date(chronological[index - 1]!.manifest.watermark).getTime() >= new Date(chronological[index]!.manifest.watermark).getTime()) {
+      throw new Error('snapshot watermark is not monotonic');
+    }
+  }
+  const view = new Map<string, NormalizedRecord>();
+  let appliedManifest: SnapshotManifestV2 | null = null;
+  for (const entry of chronological) {
+    if (target && new Date(entry.manifest.watermark).getTime() > target.getTime()) continue;
+    appliedManifest = entry.manifest;
+    for (const record of readRecords(entry.path, 'normalized/v2')) {
+      const identity = record.resourceIdentity ?? record.sourceIdentity;
+      switch (record.operation) {
+        case 'tombstone':
+          view.delete(identity);
+          break;
+        case 'unavailable':
+          view.set(identity, record);
+          break;
+        default:
+          view.set(identity, { ...record, operation: undefined });
+          break;
+      }
+    }
+  }
+  if (target && !appliedManifest) throw new Error('no verified snapshot is available at the requested as-of time');
+  return {
+    snapshotId,
+    records: [...view.values()].sort((left, right) => left.recordId.localeCompare(right.recordId)),
+    manifest: appliedManifest
+  };
+}
+
+function materializeJson(root: string, snapshotId: string, options: MaterializeOptions = {}): string {
+  const result = materializeSnapshot(root, snapshotId, options);
+  return result.records.map((record) => JSON.stringify({ schema: 'normalized/v2', ...record })).join('\n') + (result.records.length ? '\n' : '');
+}
+
+export { materializeJson, materializeSnapshot };
+export type { MaterializeOptions, MaterializeResult };

package/lib/process-data/normalize.ts

@@ -199,4 +199,32 @@ function normalizeObjects(objects: CapturedObject[]): NormalizedRecord[] {
   return records.sort((left, right) => left.recordId.localeCompare(right.recordId));
 }
 
-export { normalizeObjects };
+function normalizeResources(objects: CapturedObject[]): NormalizedRecord[] {
+  const records: NormalizedRecord[] = [];
+  for (const object of objects) {
+    if (object.role !== 'resource') continue;
+    const identity = object.resourceIdentity ?? object.sourceIdentity;
+    let data: JsonValue = { availability: 'unknown' };
+    try { data = JSON.parse(object.content ?? '{}') as JsonValue; } catch { data = { availability: 'unavailable', reason: 'invalid resource JSON' }; }
+    const binding = typeof data === 'object' && data !== null && !Array.isArray(data) && typeof data.body === 'string'
+      ? data.body.match(TASK_ID_RE)?.[0]
+      : undefined;
+    records.push({
+      recordId: recordId('platform-resource', identity),
+      kind: 'platform-resource',
+      sourceIdentity: identity,
+      resourceIdentity: identity,
+      sourceSha256: object.sha256,
+      ...(binding ? { binding } : {}),
+      ...(object.eventTime ? { observedAt: { state: 'known', value: object.eventTime } } : {}),
+      ...(object.parentIdentity ? { parentIdentity: object.parentIdentity } : {}),
+      ...(object.pageSha256 ? {
+        evidence: [{ resourceIdentity: identity, resourceSha256: object.sha256, pageSha256: object.pageSha256 }]
+      } : {}),
+      data
+    });
+  }
+  return records.sort((left, right) => left.recordId.localeCompare(right.recordId));
+}
+
+export { normalizeObjects, normalizeResources };

package/lib/process-data/reconcile.ts

@@ -1,6 +1,6 @@
 import { sha256 } from './store.ts';
 
-import type { NormalizedRecord, QualityFinding, RepairAction, SnapshotManifest } from './types.ts';
+import type { NormalizedRecord, QualityFinding, RepairAction, SnapshotManifest, SnapshotOperations } from './types.ts';
 
 function reconcileRecords(
   records: NormalizedRecord[],
@@ -107,4 +107,66 @@ function reconcileRecords(
   };
 }
 
-export { reconcileRecords };
+type ResourceReconcileOptions = {
+  parent?: NormalizedRecord[];
+  full: boolean;
+  deferred?: Iterable<string>;
+  unavailable?: Iterable<string>;
+};
+
+function reconcileResourceRecords(
+  current: NormalizedRecord[],
+  options: ResourceReconcileOptions
+): { records: NormalizedRecord[]; operations: SnapshotOperations } {
+  const parent = new Map((options.parent ?? []).filter((record) => record.resourceIdentity).map((record) => [record.resourceIdentity!, record]));
+  const currentMap = new Map(current.filter((record) => record.resourceIdentity).map((record) => [record.resourceIdentity!, record]));
+  const deferred = new Set(options.deferred ?? []);
+  const unavailable = new Set(options.unavailable ?? []);
+  const records: NormalizedRecord[] = [];
+  const operations: SnapshotOperations = { upsert: 0, supersede: 0, tombstone: 0, unavailable: 0 };
+  for (const record of currentMap.values()) {
+    const identity = record.resourceIdentity!;
+    const previous = parent.get(identity);
+    if (!previous) {
+      records.push({ ...record, operation: 'upsert' });
+      operations.upsert += 1;
+    } else if (previous.sourceSha256 !== record.sourceSha256) {
+      records.push({ ...record, operation: 'supersede' });
+      operations.supersede += 1;
+    }
+  }
+  if (options.full) {
+    for (const [identity, previous] of parent) {
+      if (currentMap.has(identity) || deferred.has(identity) || unavailable.has(identity)
+        || (previous.parentIdentity && deferred.has(previous.parentIdentity))
+        || (previous.parentIdentity && unavailable.has(previous.parentIdentity))) continue;
+      records.push({
+        recordId: previous.recordId,
+        kind: 'platform-resource',
+        sourceIdentity: identity,
+        resourceIdentity: identity,
+        sourceSha256: previous.sourceSha256,
+        operation: 'tombstone',
+        data: { availability: 'tombstone', previousSha256: previous.sourceSha256 }
+      });
+      operations.tombstone += 1;
+    }
+  }
+  for (const identity of unavailable) {
+    if (!parent.has(identity) && !currentMap.has(identity)) continue;
+    const previous = parent.get(identity) ?? currentMap.get(identity)!;
+    records.push({
+      recordId: previous.recordId,
+      kind: 'platform-resource',
+      sourceIdentity: identity,
+      resourceIdentity: identity,
+      sourceSha256: previous.sourceSha256,
+      operation: 'unavailable',
+      data: { availability: 'unavailable', reason: 'source-unavailable' }
+    });
+    operations.unavailable += 1;
+  }
+  return { records: records.sort((left, right) => left.recordId.localeCompare(right.recordId)), operations };
+}
+
+export { reconcileRecords, reconcileResourceRecords };