@fitlab-ai/agent-infra 0.9.6 → 0.9.7

package/lib/process-data/checkpoint.ts

@@ -0,0 +1,335 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { verifySnapshot } from './store.ts';
+import type { GitHubCheckpoint, CheckpointOwner, ProcessResult, RecoveryReport } from './types.ts';
+
+type CheckpointPaths = { directory: string; checkpoint: string; lock: string };
+type CheckpointCandidate = {
+  schema: 'github-checkpoint/v1';
+  candidateSchema: 'github-checkpoint-candidate/v1';
+  ownerId: string;
+  expectedOld: { snapshotId: string | null; manifestSha256: string | null };
+  repository: string;
+  snapshotId: string;
+  watermark: string;
+  manifestSha256: string;
+  committedAt: string;
+};
+type CheckpointLease = {
+  paths: CheckpointPaths;
+  repository: string;
+  owner: CheckpointOwner;
+  recovery: RecoveryReport;
+  expectedOld: GitHubCheckpoint | null;
+};
+
+function repositoryKey(repository: string): string {
+  return crypto.createHash('sha256').update(repository).digest('hex').slice(0, 32);
+}
+
+function checkpointPaths(root: string, repository: string): CheckpointPaths {
+  const directory = path.join(root, 'checkpoints', 'github');
+  const base = path.join(directory, repositoryKey(repository));
+  return { directory, checkpoint: `${base}.json`, lock: `${base}.lock` };
+}
+
+function processStartToken(pid: number): string | null {
+  if (process.platform === 'linux') {
+    try {
+      const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
+      const closing = stat.lastIndexOf(')');
+      const fields = stat.slice(closing + 2).split(' ');
+      return fields[19] ?? null;
+    } catch {
+      return null;
+    }
+  }
+  return pid === process.pid ? `self:${process.pid}` : null;
+}
+
+function fsyncDirectory(directory: string): void {
+  if (process.platform === 'win32') return;
+  const fd = fs.openSync(directory, 'r');
+  try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
+}
+
+function ensurePrivateDirectory(directory: string): void {
+  const missing: string[] = [];
+  let current = path.resolve(directory);
+  while (!fs.existsSync(current)) {
+    missing.push(current);
+    const parent = path.dirname(current);
+    if (parent === current) throw new Error(`cannot create directory tree: ${directory}`);
+    current = parent;
+  }
+  for (const target of missing.reverse()) {
+    fs.mkdirSync(target, { mode: 0o700 });
+    if (process.platform !== 'win32') fs.chmodSync(target, 0o700);
+    fsyncDirectory(path.dirname(target));
+  }
+}
+
+function writeTemporary(filePath: string, value: unknown): string {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
+  const temporary = `${filePath}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`;
+  const fd = fs.openSync(temporary, 'wx', 0o600);
+  try {
+    fs.writeFileSync(fd, `${JSON.stringify(value)}\n`);
+    fs.fsyncSync(fd);
+  } finally {
+    fs.closeSync(fd);
+  }
+  return temporary;
+}
+
+function validCheckpoint(value: unknown, repository: string): value is GitHubCheckpoint {
+  if (!value || typeof value !== 'object') return false;
+  const checkpoint = value as GitHubCheckpoint;
+  return checkpoint.schema === 'github-checkpoint/v1'
+    && checkpoint.repository === repository
+    && Boolean(checkpoint.snapshotId)
+    && Boolean(checkpoint.watermark)
+    && Boolean(checkpoint.committedAt)
+    && /^[a-f0-9]{64}$/.test(checkpoint.manifestSha256);
+}
+
+function checkpointMatchesSnapshot(root: string, value: GitHubCheckpoint): boolean {
+  const verified = verifySnapshot(root, value.snapshotId);
+  return verified.ok
+    && verified.manifest?.schema === 'raw-manifest/v2'
+    && verified.manifest.snapshotId === value.snapshotId
+    && verified.manifest.repository === value.repository
+    && verified.manifest.manifestSha256 === value.manifestSha256
+    && verified.manifest.watermark === value.watermark;
+}
+
+function checkpointState(value: GitHubCheckpoint | null): { snapshotId: string | null; manifestSha256: string | null } {
+  return { snapshotId: value?.snapshotId ?? null, manifestSha256: value?.manifestSha256 ?? null };
+}
+
+function readGitHubCheckpoint(root: string, repository: string): ProcessResult<GitHubCheckpoint | null> {
+  const target = checkpointPaths(root, repository).checkpoint;
+  if (!fs.existsSync(target)) return { ok: true, value: null };
+  try {
+    const parsed = JSON.parse(fs.readFileSync(target, 'utf8')) as GitHubCheckpoint;
+    const value: GitHubCheckpoint = {
+      schema: parsed.schema,
+      repository: parsed.repository,
+      snapshotId: parsed.snapshotId,
+      watermark: parsed.watermark,
+      manifestSha256: parsed.manifestSha256,
+      committedAt: parsed.committedAt
+    };
+    if (!validCheckpoint(value, repository)) {
+      return { ok: false, error: { code: 'CHECKPOINT_INVALID', message: 'GitHub checkpoint schema or identity is invalid' } };
+    }
+    return { ok: true, value };
+  } catch {
+    return { ok: false, error: { code: 'CHECKPOINT_INVALID', message: 'GitHub checkpoint is not valid JSON' } };
+  }
+}
+
+function quarantine(lockPath: string): string {
+  const target = `${lockPath}.orphaned-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
+  fs.renameSync(lockPath, target);
+  return target;
+}
+
+function acquireGitHubCheckpoint(root: string, repository: string): ProcessResult<CheckpointLease> {
+  const paths = checkpointPaths(root, repository);
+  ensurePrivateDirectory(paths.directory);
+  const recovery: RecoveryReport = { recovered: false, quarantined: [] };
+  const owner: CheckpointOwner = {
+    schema: 'checkpoint-owner/v1',
+    ownerId: crypto.randomUUID(),
+    pid: process.pid,
+    host: os.hostname(),
+    processStartToken: processStartToken(process.pid) ?? '',
+    createdAt: new Date().toISOString()
+  };
+  if (!owner.processStartToken) {
+    return { ok: false, error: { code: 'CHECKPOINT_TOPOLOGY_UNSUPPORTED', message: 'Process liveness cannot be verified on this host' } };
+  }
+  let recoveredOwnerId: string | null = null;
+  if (fs.existsSync(paths.lock)) {
+    let previous: CheckpointOwner;
+    try {
+      previous = JSON.parse(fs.readFileSync(paths.lock, 'utf8')) as CheckpointOwner;
+    } catch {
+      return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint lock is unreadable' } };
+    }
+    if (previous.host !== owner.host || !previous.ownerId || !previous.processStartToken) {
+      return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint lock owner topology is unknown' } };
+    }
+    const currentToken = processStartToken(previous.pid);
+    if (currentToken && currentToken === previous.processStartToken) {
+      return { ok: false, error: { code: 'CHECKPOINT_BUSY', message: 'Checkpoint is held by a live process' } };
+    }
+    if (process.platform !== 'linux') {
+      return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint owner liveness cannot be proven on this host' } };
+    }
+    try {
+      recoveredOwnerId = previous.ownerId;
+      recovery.quarantined.push(quarantine(paths.lock));
+      recovery.recovered = true;
+    } catch {
+      return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Unable to quarantine an orphaned checkpoint lock' } };
+    }
+  }
+  try {
+    const fd = fs.openSync(paths.lock, 'wx', 0o600);
+    try { fs.writeFileSync(fd, `${JSON.stringify(owner)}\n`); fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
+    fsyncDirectory(paths.directory);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'EEXIST') {
+      return { ok: false, error: { code: 'CHECKPOINT_BUSY', message: 'Checkpoint lock was acquired by another process' } };
+    }
+    return { ok: false, error: { code: 'CHECKPOINT_LOCK_FAILED', message: String(error) } };
+  }
+
+  const releaseOwnedLock = () => {
+    try {
+      fs.unlinkSync(paths.lock);
+      fsyncDirectory(paths.directory);
+    } catch {
+      // Preserve the original recovery error; a remaining lock will fail closed next time.
+    }
+  };
+  const beforeResult = readGitHubCheckpoint(root, repository);
+  if (!beforeResult.ok) {
+    releaseOwnedLock();
+    return beforeResult;
+  }
+  let checkpointBefore = beforeResult.value;
+  const temporaryCandidates = fs.readdirSync(paths.directory)
+    .filter((entry) => entry.startsWith(path.basename(paths.checkpoint) + '.') && entry.endsWith('.tmp'))
+    .map((entry) => path.join(paths.directory, entry));
+  if (temporaryCandidates.length > 1) {
+    releaseOwnedLock();
+    return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Multiple checkpoint recovery candidates exist' } };
+  }
+  if (temporaryCandidates.length === 1) {
+    const candidate = temporaryCandidates[0]!;
+    try {
+      const parsed = JSON.parse(fs.readFileSync(candidate, 'utf8')) as CheckpointCandidate | GitHubCheckpoint;
+      const isCandidate = parsed && typeof parsed === 'object' && 'candidateSchema' in parsed
+        && parsed.candidateSchema === 'github-checkpoint-candidate/v1';
+      if (isCandidate) {
+        const candidateValue = parsed as CheckpointCandidate;
+        const value: GitHubCheckpoint = candidateValue;
+        const expected = checkpointState(checkpointBefore);
+        const candidateExpected = candidateValue.expectedOld;
+        if (!recoveredOwnerId || candidateValue.ownerId !== recoveredOwnerId) {
+          recovery.quarantined.push(quarantine(candidate));
+          releaseOwnedLock();
+          return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint candidate owner proof does not match the recovered lock owner' } };
+        }
+        if (!validCheckpoint(value, repository)
+          || candidateExpected?.snapshotId !== expected.snapshotId
+          || candidateExpected?.manifestSha256 !== expected.manifestSha256) {
+          releaseOwnedLock();
+          return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint candidate does not match the current head' } };
+        }
+        if (!checkpointMatchesSnapshot(root, value)) {
+          recovery.quarantined.push(quarantine(candidate));
+          releaseOwnedLock();
+          return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint candidate snapshot failed full verification' } };
+        }
+        if (process.platform === 'win32' && checkpointBefore) {
+          releaseOwnedLock();
+          return { ok: false, error: { code: 'CHECKPOINT_TOPOLOGY_UNSUPPORTED', message: 'Checkpoint replacement durability cannot be proven on this host' } };
+        }
+        fs.renameSync(candidate, paths.checkpoint);
+        recovery.recovered = true;
+        fsyncDirectory(paths.directory);
+        const recovered = readGitHubCheckpoint(root, repository);
+        if (!recovered.ok) {
+          releaseOwnedLock();
+          return recovered;
+        }
+        checkpointBefore = recovered.value;
+      } else {
+        recovery.quarantined.push(quarantine(candidate));
+        releaseOwnedLock();
+        return { ok: false, error: { code: 'CHECKPOINT_RECOVERY_REQUIRED', message: 'Checkpoint candidate lacks owner proof' } };
+      }
+    } catch {
+      recovery.quarantined.push(quarantine(candidate));
+    }
+  }
+  return { ok: true, value: { paths, repository, owner, recovery, expectedOld: checkpointBefore } };
+}
+
+function commitGitHubCheckpoint(lease: CheckpointLease, value: GitHubCheckpoint): ProcessResult<void> {
+  try {
+    const current = JSON.parse(fs.readFileSync(lease.paths.lock, 'utf8')) as CheckpointOwner;
+    if (current.ownerId !== lease.owner.ownerId) {
+      return { ok: false, error: { code: 'CHECKPOINT_OWNER_MISMATCH', message: 'Checkpoint lock ownership changed' } };
+    }
+    if (value.repository !== lease.repository || !validCheckpoint(value, lease.repository)) {
+      return { ok: false, error: { code: 'CHECKPOINT_INVALID', message: 'Checkpoint value is invalid' } };
+    }
+    const root = path.dirname(path.dirname(lease.paths.directory));
+    if (!checkpointMatchesSnapshot(root, value)) {
+      return { ok: false, error: { code: 'CHECKPOINT_INVALID', message: 'Checkpoint snapshot does not match a verified v2 manifest' } };
+    }
+    const currentCheckpoint = readGitHubCheckpoint(root, lease.repository);
+    if (!currentCheckpoint.ok) return currentCheckpoint;
+    if (checkpointState(currentCheckpoint.value).snapshotId !== checkpointState(lease.expectedOld).snapshotId
+      || checkpointState(currentCheckpoint.value).manifestSha256 !== checkpointState(lease.expectedOld).manifestSha256) {
+      return { ok: false, error: { code: 'CHECKPOINT_CONFLICT', message: 'Checkpoint head changed before commit' } };
+    }
+    const candidate: CheckpointCandidate = {
+      ...value,
+      candidateSchema: 'github-checkpoint-candidate/v1',
+      ownerId: lease.owner.ownerId,
+      expectedOld: checkpointState(lease.expectedOld)
+    };
+    const temporary = writeTemporary(lease.paths.checkpoint, candidate);
+    const rechecked = readGitHubCheckpoint(root, lease.repository);
+    if (!rechecked.ok) {
+      fs.unlinkSync(temporary);
+      return rechecked;
+    }
+    if (checkpointState(rechecked.value).snapshotId !== checkpointState(lease.expectedOld).snapshotId
+      || checkpointState(rechecked.value).manifestSha256 !== checkpointState(lease.expectedOld).manifestSha256) {
+      fs.unlinkSync(temporary);
+      return { ok: false, error: { code: 'CHECKPOINT_CONFLICT', message: 'Checkpoint head changed during commit' } };
+    }
+    if (process.platform === 'win32' && fs.existsSync(lease.paths.checkpoint)) {
+      fs.unlinkSync(temporary);
+      return { ok: false, error: { code: 'CHECKPOINT_TOPOLOGY_UNSUPPORTED', message: 'Checkpoint replacement durability cannot be proven on this host' } };
+    }
+    fs.renameSync(temporary, lease.paths.checkpoint);
+    if (process.platform !== 'win32') fs.chmodSync(lease.paths.checkpoint, 0o600);
+    fsyncDirectory(lease.paths.directory);
+    return { ok: true, value: undefined };
+  } catch (error) {
+    return { ok: false, error: { code: 'CHECKPOINT_COMMIT_FAILED', message: String(error) } };
+  }
+}
+
+function releaseGitHubCheckpoint(lease: CheckpointLease): void {
+  try {
+    const current = JSON.parse(fs.readFileSync(lease.paths.lock, 'utf8')) as CheckpointOwner;
+    if (current.ownerId === lease.owner.ownerId) {
+      fs.unlinkSync(lease.paths.lock);
+      fsyncDirectory(lease.paths.directory);
+    }
+  } catch {
+    // The next invocation will fail closed and report recovery if the lock remains.
+  }
+}
+
+export {
+  acquireGitHubCheckpoint,
+  checkpointPaths,
+  commitGitHubCheckpoint,
+  processStartToken,
+  readGitHubCheckpoint,
+  releaseGitHubCheckpoint
+};
+export type { CheckpointLease, CheckpointPaths };

package/lib/process-data/index.ts

@@ -1,24 +1,26 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { execFileSync } from 'node:child_process';
 
 import { detectRepoRoot } from '../task/resolve-ref.ts';
-import { normalizeObjects } from './normalize.ts';
-import { reconcileRecords } from './reconcile.ts';
+import { acquireGitHubCheckpoint, commitGitHubCheckpoint, readGitHubCheckpoint, releaseGitHubCheckpoint } from './checkpoint.ts';
+import { materializeJson, materializeSnapshot } from './materialize.ts';
+import { normalizeResources } from './normalize.ts';
+import { reconcileRecords, reconcileResourceRecords } from './reconcile.ts';
 import { readAppliedOverlays, repairSnapshot } from './repair.ts';
-import { collectGitHubObjects, collectLocalObjects } from './sources.ts';
-import { canonicalJsonBytes, createObjectStore, publishSnapshot, verifySnapshot } from './store.ts';
+import { collectGitHubBoundary, resolveGitHubRepository } from './sources.ts';
+import { canonicalJsonBytes, createObjectStore, findSnapshot, publishSnapshotV2, verifySnapshot } from './store.ts';
+import { createGitHubClient } from '../platform/github-client.ts';
 
-import type { CapturedObject, NormalizedRecord, RestCollectionEvidence } from './types.ts';
+import type { NormalizedRecord } from './types.ts';
 
 const USAGE = `Usage: ai data <command> [options]
 
 Commands:
-  capture [--source all|local|github] [--root <dir>] [--include-excerpts]
+  capture [--source github] [--root <dir>] [--full-reconcile]
   verify <snapshot-id> [--root <dir>]
   audit <snapshot-id> [--root <dir>] [--format json|text]
   repair <snapshot-id> [--root <dir>] [--apply]
-  export <snapshot-id> [--root <dir>] [--repairs none|applied] [--output <file|->]
+  export <snapshot-id> [--root <dir>] [--repairs none|applied] [--as-of <ISO>] [--output <file|->]
 `;
 
 type ParsedArgs = { positionals: string[]; options: Map<string, string | true> };
@@ -64,14 +66,6 @@ function dataRoot(repoRoot: string, requested?: string): string {
   return resolved;
 }
 
-function repositoryIdentity(repoRoot: string): string {
-  try {
-    return execFileSync('git', ['config', '--get', 'remote.origin.url'], { cwd: repoRoot, encoding: 'utf8' }).trim() || path.basename(repoRoot);
-  } catch {
-    return path.basename(repoRoot);
-  }
-}
-
 function writeJson(value: unknown): void {
   process.stdout.write(`${JSON.stringify(value)}\n`);
 }
@@ -82,54 +76,106 @@ function optionString(parsed: ParsedArgs, name: string): string | undefined {
 }
 
 function capture(args: string[]): number {
-  const parsed = parseArgs(args, new Set(['--source', '--root']), new Set(['--include-excerpts']));
+  const parsed = parseArgs(args, new Set(['--source', '--root']), new Set(['--full-reconcile', '--include-excerpts']));
   if (parsed.positionals.length > 0) throw new Error('capture does not accept positional arguments');
-  const scope = optionString(parsed, '--source') ?? 'all';
-  if (!['all', 'local', 'github'].includes(scope)) throw new Error('--source must be all, local, or github');
+  const scope = optionString(parsed, '--source') ?? 'github';
+  if (scope !== 'github') throw new Error('new process-data capture only supports --source github; local data is already normalized and --source all is retired');
+  if (parsed.options.has('--include-excerpts')) throw new Error('--include-excerpts is not supported for GitHub-only capture');
   const repoRoot = detectRepoRoot();
   const root = dataRoot(repoRoot, optionString(parsed, '--root'));
-  const observedFrom = new Date().toISOString();
-  const objects: CapturedObject[] = [];
-  const endpoints: RestCollectionEvidence[] = [];
-  let repository = repositoryIdentity(repoRoot);
-
-  if (scope === 'all' || scope === 'local') {
-    const local = collectLocalObjects(repoRoot, { includeExcerpts: parsed.options.has('--include-excerpts') });
-    if (!local.ok) { writeJson({ ok: false, error: local.error }); return 2; }
-    objects.push(...local.value);
-  }
-  if (scope === 'all' || scope === 'github') {
-    const github = collectGitHubObjects(repoRoot);
+  const client = createGitHubClient();
+  const repositoryResult = resolveGitHubRepository(repoRoot, client);
+  if (!repositoryResult.ok) { writeJson({ ok: false, error: repositoryResult.error }); return 2; }
+  const repository = repositoryResult.value;
+  const leaseResult = acquireGitHubCheckpoint(root, repository);
+  if (!leaseResult.ok) { writeJson({ ok: false, error: leaseResult.error }); return 2; }
+  const lease = leaseResult.value;
+  try {
+    const checkpointResult = readGitHubCheckpoint(root, repository);
+    if (!checkpointResult.ok) { writeJson({ ok: false, error: checkpointResult.error }); return 2; }
+    const checkpoint = checkpointResult.value;
+    const full = parsed.options.has('--full-reconcile');
+    const observedFrom = new Date().toISOString();
+    const github = collectGitHubBoundary(repoRoot, {
+      client,
+      fromInclusive: full ? null : checkpoint?.watermark ?? null,
+      reconciliation: full ? 'full' : 'incremental'
+    });
     if (!github.ok) { writeJson({ ok: false, error: github.error }); return 2; }
-    objects.push(...github.value.objects);
-    endpoints.push(...github.value.endpoints);
-    repository = github.value.repository;
-  }
-
-  const store = createObjectStore(root);
-  for (const object of objects) {
-    if (object.content !== undefined && object.disposition?.state === 'included') {
-      const stored = store.put(Buffer.from(object.content, 'utf8'));
-      if (stored.sha256 !== object.sha256) throw new Error(`source hash mismatch: ${object.sourceIdentity}`);
+    const store = createObjectStore(root);
+    for (const object of github.value.objects) {
+      if (object.content !== undefined && object.disposition?.state === 'included') {
+        const stored = store.put(Buffer.from(object.content, 'utf8'));
+        if (stored.sha256 !== object.sha256) throw new Error(`source hash mismatch: ${object.sourceIdentity}`);
+      }
+    }
+    let parent: NormalizedRecord[] = [];
+    if (checkpoint) {
+      const parentPath = findSnapshot(root, checkpoint.snapshotId);
+      if (!parentPath) throw new Error(`checkpoint head snapshot is missing: ${checkpoint.snapshotId}`);
+      const materialized = materializeSnapshot(root, checkpoint.snapshotId);
+      if (!materialized.manifest || materialized.manifest.manifestSha256 !== checkpoint.manifestSha256 || materialized.manifest.watermark !== checkpoint.watermark) {
+        throw new Error('checkpoint does not match its verified snapshot head');
+      }
+      parent = materialized.records;
+    }
+    const current = normalizeResources(github.value.objects);
+    const reconciliation = reconcileResourceRecords(current, {
+      parent,
+      full,
+      deferred: github.value.deferred,
+      unavailable: github.value.unavailable
+    });
+    const quality = reconcileRecords(reconciliation.records, 'github');
+    const fromInclusive = checkpoint?.watermark ?? null;
+    const hasStrictSinceEndpoint = github.value.endpoints.some((endpoint) => endpoint.queryMode === 'strict-since');
+    const queryAfter = !full && fromInclusive && hasStrictSinceEndpoint
+      ? new Date(new Date(fromInclusive).getTime() - 1000).toISOString()
+      : null;
+    const responseDates = github.value.responseDates;
+    const published = publishSnapshotV2(root, {
+      scope: 'github',
+      repository,
+      observedFrom,
+      observedTo: github.value.watermark,
+      objects: github.value.objects.map(({ content: _content, ...object }) => object),
+      endpoints: github.value.endpoints,
+      excerptsEnabled: false,
+      records: reconciliation.records,
+      quality: quality.findings,
+      repairs: quality.repairs,
+      snapshotKind: checkpoint ? 'delta' : 'base',
+      parentSnapshotId: checkpoint?.snapshotId ?? null,
+      checkpointBefore: checkpoint?.snapshotId ?? null,
+      watermark: github.value.watermark,
+      window: { fromInclusive, queryAfter, toExclusive: github.value.watermark, precision: 'second' },
+      observation: {
+        cutoffSource: 'github-response-date',
+        preflightDate: github.value.preflightDate,
+        responseDates
+      },
+      coverage: { mode: 'boundary-reread-with-full-reconcile', absoluteCompleteness: false },
+      reconciliation: full ? 'full' : 'incremental',
+      operations: reconciliation.operations
+    });
+    const verified = verifySnapshot(root, published.snapshotId);
+    if (!verified.ok || !verified.manifest || verified.manifest.schema !== 'raw-manifest/v2') {
+      throw new Error(`published snapshot failed verification: ${verified.errors.join('; ')}`);
     }
+    const committed = commitGitHubCheckpoint(lease, {
+      schema: 'github-checkpoint/v1',
+      repository,
+      snapshotId: published.snapshotId,
+      watermark: github.value.watermark,
+      manifestSha256: verified.manifest.manifestSha256,
+      committedAt: new Date().toISOString()
+    });
+    if (!committed.ok) { writeJson({ ok: false, error: committed.error }); return 2; }
+    writeJson({ ok: true, snapshotId: published.snapshotId, scope, snapshotKind: checkpoint ? 'delta' : 'base', reconciliation: full ? 'full' : 'incremental', created: published.created, objects: github.value.objects.length, records: reconciliation.records.length, watermark: github.value.watermark, deferred: github.value.deferred.length });
+    return 0;
+  } finally {
+    releaseGitHubCheckpoint(lease);
   }
-  const records = normalizeObjects(objects);
-  const reconciliation = reconcileRecords(records, scope as 'all' | 'local' | 'github');
-  const observedTo = new Date().toISOString();
-  const published = publishSnapshot(root, {
-    scope: scope as 'all' | 'local' | 'github',
-    repository,
-    observedFrom,
-    observedTo,
-    objects: objects.map(({ content: _content, ...object }) => object),
-    endpoints,
-    excerptsEnabled: parsed.options.has('--include-excerpts'),
-    records,
-    quality: reconciliation.findings,
-    repairs: reconciliation.repairs
-  });
-  writeJson({ ok: true, snapshotId: published.snapshotId, scope, created: published.created, objects: objects.length, records: records.length });
-  return 0;
 }
 
 function verify(args: string[]): number {
@@ -175,7 +221,7 @@ function repair(args: string[]): number {
 }
 
 function exportSnapshot(args: string[]): number {
-  const parsed = parseArgs(args, new Set(['--root', '--repairs', '--output']), new Set());
+  const parsed = parseArgs(args, new Set(['--root', '--repairs', '--output', '--as-of']), new Set());
   if (parsed.positionals.length !== 1) throw new Error('export requires one snapshot id');
   const repairMode = optionString(parsed, '--repairs') ?? 'none';
   if (!['none', 'applied'].includes(repairMode)) throw new Error('--repairs must be none or applied');
@@ -184,7 +230,13 @@ function exportSnapshot(args: string[]): number {
   const root = dataRoot(repoRoot, optionString(parsed, '--root'));
   const verified = verifySnapshot(root, parsed.positionals[0]!);
   if (!verified.ok || !verified.snapshotPath) { writeJson(verified); return 1; }
-  let content = fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8');
+  let content: string;
+  if (verified.manifest?.schema === 'raw-manifest/v2') {
+    content = materializeJson(root, parsed.positionals[0]!, { asOf: optionString(parsed, '--as-of') });
+  } else {
+    if (optionString(parsed, '--as-of')) throw new Error('--as-of is only supported for v2 GitHub snapshots');
+    content = fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8');
+  }
   if (repairMode === 'applied') content += readAppliedOverlays(root, parsed.positionals[0]!);
   if (output === '-') process.stdout.write(content);
   else {

package/lib/process-data/materialize.ts

@@ -0,0 +1,83 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { findSnapshot, verifySnapshot } from './store.ts';
+import type { NormalizedRecord, SnapshotManifestV2 } from './types.ts';
+
+type MaterializeOptions = { asOf?: string };
+type MaterializeResult = { snapshotId: string; records: NormalizedRecord[]; manifest: SnapshotManifestV2 | null };
+
+function readRecords(snapshotPath: string, schema: 'normalized/v1' | 'normalized/v2'): NormalizedRecord[] {
+  const file = path.join(snapshotPath, 'normalized.jsonl');
+  if (!fs.existsSync(file)) return [];
+  return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
+    const value = JSON.parse(line) as { schema?: string } & NormalizedRecord;
+    if (value.schema !== schema) return [];
+    const { schema: _schema, ...record } = value;
+    return [record];
+  });
+}
+
+function materializeSnapshot(root: string, snapshotId: string, options: MaterializeOptions = {}): MaterializeResult {
+  const target = options.asOf ? new Date(options.asOf) : null;
+  if (target && !Number.isFinite(target.getTime())) throw new Error('as-of must be an ISO timestamp');
+  const chain: Array<{ id: string; path: string; manifest: SnapshotManifestV2 }> = [];
+  let currentId: string | null = snapshotId;
+  let repository: string | null = null;
+  while (currentId) {
+    const currentPath = findSnapshot(root, currentId);
+    if (!currentPath) throw new Error(`snapshot not found: ${currentId}`);
+    const verified = verifySnapshot(root, currentId);
+    if (!verified.ok || !verified.manifest) throw new Error(`snapshot is not verified: ${currentId}`);
+    if (verified.manifest.schema !== 'raw-manifest/v2') {
+      if (chain.length > 0) throw new Error('v1 and v2 snapshots cannot share a lineage');
+      return { snapshotId, records: readRecords(currentPath, 'normalized/v1'), manifest: null };
+    }
+    const manifest = verified.manifest;
+    if (repository && manifest.repository !== repository) throw new Error('snapshot lineage repository mismatch');
+    repository = manifest.repository;
+    chain.push({ id: currentId, path: currentPath, manifest });
+    currentId = manifest.parentSnapshotId;
+  }
+  if (chain.length === 0) throw new Error('snapshot lineage is empty');
+  const chronological = [...chain].reverse();
+  for (let index = 1; index < chronological.length; index += 1) {
+    if (new Date(chronological[index - 1]!.manifest.watermark).getTime() >= new Date(chronological[index]!.manifest.watermark).getTime()) {
+      throw new Error('snapshot watermark is not monotonic');
+    }
+  }
+  const view = new Map<string, NormalizedRecord>();
+  let appliedManifest: SnapshotManifestV2 | null = null;
+  for (const entry of chronological) {
+    if (target && new Date(entry.manifest.watermark).getTime() > target.getTime()) continue;
+    appliedManifest = entry.manifest;
+    for (const record of readRecords(entry.path, 'normalized/v2')) {
+      const identity = record.resourceIdentity ?? record.sourceIdentity;
+      switch (record.operation) {
+        case 'tombstone':
+          view.delete(identity);
+          break;
+        case 'unavailable':
+          view.set(identity, record);
+          break;
+        default:
+          view.set(identity, { ...record, operation: undefined });
+          break;
+      }
+    }
+  }
+  if (target && !appliedManifest) throw new Error('no verified snapshot is available at the requested as-of time');
+  return {
+    snapshotId,
+    records: [...view.values()].sort((left, right) => left.recordId.localeCompare(right.recordId)),
+    manifest: appliedManifest
+  };
+}
+
+function materializeJson(root: string, snapshotId: string, options: MaterializeOptions = {}): string {
+  const result = materializeSnapshot(root, snapshotId, options);
+  return result.records.map((record) => JSON.stringify({ schema: 'normalized/v2', ...record })).join('\n') + (result.records.length ? '\n' : '');
+}
+
+export { materializeJson, materializeSnapshot };
+export type { MaterializeOptions, MaterializeResult };