@fitlab-ai/agent-infra 0.9.6 → 0.9.7

package/dist/lib/process-data/index.js

@@ -1,20 +1,22 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { execFileSync } from 'node:child_process';
 import { detectRepoRoot } from '../task/resolve-ref.js';
-import { normalizeObjects } from './normalize.js';
-import { reconcileRecords } from './reconcile.js';
+import { acquireGitHubCheckpoint, commitGitHubCheckpoint, readGitHubCheckpoint, releaseGitHubCheckpoint } from './checkpoint.js';
+import { materializeJson, materializeSnapshot } from './materialize.js';
+import { normalizeResources } from './normalize.js';
+import { reconcileRecords, reconcileResourceRecords } from './reconcile.js';
 import { readAppliedOverlays, repairSnapshot } from './repair.js';
-import { collectGitHubObjects, collectLocalObjects } from './sources.js';
-import { canonicalJsonBytes, createObjectStore, publishSnapshot, verifySnapshot } from './store.js';
+import { collectGitHubBoundary, resolveGitHubRepository } from './sources.js';
+import { canonicalJsonBytes, createObjectStore, findSnapshot, publishSnapshotV2, verifySnapshot } from './store.js';
+import { createGitHubClient } from '../platform/github-client.js';
 const USAGE = `Usage: ai data <command> [options]
 
 Commands:
-  capture [--source all|local|github] [--root <dir>] [--include-excerpts]
+  capture [--source github] [--root <dir>] [--full-reconcile]
   verify <snapshot-id> [--root <dir>]
   audit <snapshot-id> [--root <dir>] [--format json|text]
   repair <snapshot-id> [--root <dir>] [--apply]
-  export <snapshot-id> [--root <dir>] [--repairs none|applied] [--output <file|->]
+  export <snapshot-id> [--root <dir>] [--repairs none|applied] [--as-of <ISO>] [--output <file|->]
 `;
 function parseArgs(args, valueOptions, booleanOptions) {
     const positionals = [];
@@ -61,14 +63,6 @@ function dataRoot(repoRoot, requested) {
     }
     return resolved;
 }
-function repositoryIdentity(repoRoot) {
-    try {
-        return execFileSync('git', ['config', '--get', 'remote.origin.url'], { cwd: repoRoot, encoding: 'utf8' }).trim() || path.basename(repoRoot);
-    }
-    catch {
-        return path.basename(repoRoot);
-    }
-}
 function writeJson(value) {
     process.stdout.write(`${JSON.stringify(value)}\n`);
 }
@@ -77,61 +71,127 @@ function optionString(parsed, name) {
     return typeof value === 'string' ? value : undefined;
 }
 function capture(args) {
-    const parsed = parseArgs(args, new Set(['--source', '--root']), new Set(['--include-excerpts']));
+    const parsed = parseArgs(args, new Set(['--source', '--root']), new Set(['--full-reconcile', '--include-excerpts']));
     if (parsed.positionals.length > 0)
         throw new Error('capture does not accept positional arguments');
-    const scope = optionString(parsed, '--source') ?? 'all';
-    if (!['all', 'local', 'github'].includes(scope))
-        throw new Error('--source must be all, local, or github');
+    const scope = optionString(parsed, '--source') ?? 'github';
+    if (scope !== 'github')
+        throw new Error('new process-data capture only supports --source github; local data is already normalized and --source all is retired');
+    if (parsed.options.has('--include-excerpts'))
+        throw new Error('--include-excerpts is not supported for GitHub-only capture');
     const repoRoot = detectRepoRoot();
     const root = dataRoot(repoRoot, optionString(parsed, '--root'));
-    const observedFrom = new Date().toISOString();
-    const objects = [];
-    const endpoints = [];
-    let repository = repositoryIdentity(repoRoot);
-    if (scope === 'all' || scope === 'local') {
-        const local = collectLocalObjects(repoRoot, { includeExcerpts: parsed.options.has('--include-excerpts') });
-        if (!local.ok) {
-            writeJson({ ok: false, error: local.error });
+    const client = createGitHubClient();
+    const repositoryResult = resolveGitHubRepository(repoRoot, client);
+    if (!repositoryResult.ok) {
+        writeJson({ ok: false, error: repositoryResult.error });
+        return 2;
+    }
+    const repository = repositoryResult.value;
+    const leaseResult = acquireGitHubCheckpoint(root, repository);
+    if (!leaseResult.ok) {
+        writeJson({ ok: false, error: leaseResult.error });
+        return 2;
+    }
+    const lease = leaseResult.value;
+    try {
+        const checkpointResult = readGitHubCheckpoint(root, repository);
+        if (!checkpointResult.ok) {
+            writeJson({ ok: false, error: checkpointResult.error });
             return 2;
         }
-        objects.push(...local.value);
-    }
-    if (scope === 'all' || scope === 'github') {
-        const github = collectGitHubObjects(repoRoot);
+        const checkpoint = checkpointResult.value;
+        const full = parsed.options.has('--full-reconcile');
+        const observedFrom = new Date().toISOString();
+        const github = collectGitHubBoundary(repoRoot, {
+            client,
+            fromInclusive: full ? null : checkpoint?.watermark ?? null,
+            reconciliation: full ? 'full' : 'incremental'
+        });
         if (!github.ok) {
             writeJson({ ok: false, error: github.error });
             return 2;
         }
-        objects.push(...github.value.objects);
-        endpoints.push(...github.value.endpoints);
-        repository = github.value.repository;
-    }
-    const store = createObjectStore(root);
-    for (const object of objects) {
-        if (object.content !== undefined && object.disposition?.state === 'included') {
-            const stored = store.put(Buffer.from(object.content, 'utf8'));
-            if (stored.sha256 !== object.sha256)
-                throw new Error(`source hash mismatch: ${object.sourceIdentity}`);
+        const store = createObjectStore(root);
+        for (const object of github.value.objects) {
+            if (object.content !== undefined && object.disposition?.state === 'included') {
+                const stored = store.put(Buffer.from(object.content, 'utf8'));
+                if (stored.sha256 !== object.sha256)
+                    throw new Error(`source hash mismatch: ${object.sourceIdentity}`);
+            }
+        }
+        let parent = [];
+        if (checkpoint) {
+            const parentPath = findSnapshot(root, checkpoint.snapshotId);
+            if (!parentPath)
+                throw new Error(`checkpoint head snapshot is missing: ${checkpoint.snapshotId}`);
+            const materialized = materializeSnapshot(root, checkpoint.snapshotId);
+            if (!materialized.manifest || materialized.manifest.manifestSha256 !== checkpoint.manifestSha256 || materialized.manifest.watermark !== checkpoint.watermark) {
+                throw new Error('checkpoint does not match its verified snapshot head');
+            }
+            parent = materialized.records;
         }
+        const current = normalizeResources(github.value.objects);
+        const reconciliation = reconcileResourceRecords(current, {
+            parent,
+            full,
+            deferred: github.value.deferred,
+            unavailable: github.value.unavailable
+        });
+        const quality = reconcileRecords(reconciliation.records, 'github');
+        const fromInclusive = checkpoint?.watermark ?? null;
+        const hasStrictSinceEndpoint = github.value.endpoints.some((endpoint) => endpoint.queryMode === 'strict-since');
+        const queryAfter = !full && fromInclusive && hasStrictSinceEndpoint
+            ? new Date(new Date(fromInclusive).getTime() - 1000).toISOString()
+            : null;
+        const responseDates = github.value.responseDates;
+        const published = publishSnapshotV2(root, {
+            scope: 'github',
+            repository,
+            observedFrom,
+            observedTo: github.value.watermark,
+            objects: github.value.objects.map(({ content: _content, ...object }) => object),
+            endpoints: github.value.endpoints,
+            excerptsEnabled: false,
+            records: reconciliation.records,
+            quality: quality.findings,
+            repairs: quality.repairs,
+            snapshotKind: checkpoint ? 'delta' : 'base',
+            parentSnapshotId: checkpoint?.snapshotId ?? null,
+            checkpointBefore: checkpoint?.snapshotId ?? null,
+            watermark: github.value.watermark,
+            window: { fromInclusive, queryAfter, toExclusive: github.value.watermark, precision: 'second' },
+            observation: {
+                cutoffSource: 'github-response-date',
+                preflightDate: github.value.preflightDate,
+                responseDates
+            },
+            coverage: { mode: 'boundary-reread-with-full-reconcile', absoluteCompleteness: false },
+            reconciliation: full ? 'full' : 'incremental',
+            operations: reconciliation.operations
+        });
+        const verified = verifySnapshot(root, published.snapshotId);
+        if (!verified.ok || !verified.manifest || verified.manifest.schema !== 'raw-manifest/v2') {
+            throw new Error(`published snapshot failed verification: ${verified.errors.join('; ')}`);
+        }
+        const committed = commitGitHubCheckpoint(lease, {
+            schema: 'github-checkpoint/v1',
+            repository,
+            snapshotId: published.snapshotId,
+            watermark: github.value.watermark,
+            manifestSha256: verified.manifest.manifestSha256,
+            committedAt: new Date().toISOString()
+        });
+        if (!committed.ok) {
+            writeJson({ ok: false, error: committed.error });
+            return 2;
+        }
+        writeJson({ ok: true, snapshotId: published.snapshotId, scope, snapshotKind: checkpoint ? 'delta' : 'base', reconciliation: full ? 'full' : 'incremental', created: published.created, objects: github.value.objects.length, records: reconciliation.records.length, watermark: github.value.watermark, deferred: github.value.deferred.length });
+        return 0;
+    }
+    finally {
+        releaseGitHubCheckpoint(lease);
     }
-    const records = normalizeObjects(objects);
-    const reconciliation = reconcileRecords(records, scope);
-    const observedTo = new Date().toISOString();
-    const published = publishSnapshot(root, {
-        scope: scope,
-        repository,
-        observedFrom,
-        observedTo,
-        objects: objects.map(({ content: _content, ...object }) => object),
-        endpoints,
-        excerptsEnabled: parsed.options.has('--include-excerpts'),
-        records,
-        quality: reconciliation.findings,
-        repairs: reconciliation.repairs
-    });
-    writeJson({ ok: true, snapshotId: published.snapshotId, scope, created: published.created, objects: objects.length, records: records.length });
-    return 0;
 }
 function verify(args) {
     const parsed = parseArgs(args, new Set(['--root']), new Set());
@@ -182,7 +242,7 @@ function repair(args) {
     return 0;
 }
 function exportSnapshot(args) {
-    const parsed = parseArgs(args, new Set(['--root', '--repairs', '--output']), new Set());
+    const parsed = parseArgs(args, new Set(['--root', '--repairs', '--output', '--as-of']), new Set());
     if (parsed.positionals.length !== 1)
         throw new Error('export requires one snapshot id');
     const repairMode = optionString(parsed, '--repairs') ?? 'none';
@@ -196,7 +256,15 @@ function exportSnapshot(args) {
         writeJson(verified);
         return 1;
     }
-    let content = fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8');
+    let content;
+    if (verified.manifest?.schema === 'raw-manifest/v2') {
+        content = materializeJson(root, parsed.positionals[0], { asOf: optionString(parsed, '--as-of') });
+    }
+    else {
+        if (optionString(parsed, '--as-of'))
+            throw new Error('--as-of is only supported for v2 GitHub snapshots');
+        content = fs.readFileSync(path.join(verified.snapshotPath, 'normalized.jsonl'), 'utf8');
+    }
     if (repairMode === 'applied')
         content += readAppliedOverlays(root, parsed.positionals[0]);
     if (output === '-')

package/dist/lib/process-data/materialize.js

@@ -0,0 +1,84 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { findSnapshot, verifySnapshot } from './store.js';
+function readRecords(snapshotPath, schema) {
+    const file = path.join(snapshotPath, 'normalized.jsonl');
+    if (!fs.existsSync(file))
+        return [];
+    return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
+        const value = JSON.parse(line);
+        if (value.schema !== schema)
+            return [];
+        const { schema: _schema, ...record } = value;
+        return [record];
+    });
+}
+function materializeSnapshot(root, snapshotId, options = {}) {
+    const target = options.asOf ? new Date(options.asOf) : null;
+    if (target && !Number.isFinite(target.getTime()))
+        throw new Error('as-of must be an ISO timestamp');
+    const chain = [];
+    let currentId = snapshotId;
+    let repository = null;
+    while (currentId) {
+        const currentPath = findSnapshot(root, currentId);
+        if (!currentPath)
+            throw new Error(`snapshot not found: ${currentId}`);
+        const verified = verifySnapshot(root, currentId);
+        if (!verified.ok || !verified.manifest)
+            throw new Error(`snapshot is not verified: ${currentId}`);
+        if (verified.manifest.schema !== 'raw-manifest/v2') {
+            if (chain.length > 0)
+                throw new Error('v1 and v2 snapshots cannot share a lineage');
+            return { snapshotId, records: readRecords(currentPath, 'normalized/v1'), manifest: null };
+        }
+        const manifest = verified.manifest;
+        if (repository && manifest.repository !== repository)
+            throw new Error('snapshot lineage repository mismatch');
+        repository = manifest.repository;
+        chain.push({ id: currentId, path: currentPath, manifest });
+        currentId = manifest.parentSnapshotId;
+    }
+    if (chain.length === 0)
+        throw new Error('snapshot lineage is empty');
+    const chronological = [...chain].reverse();
+    for (let index = 1; index < chronological.length; index += 1) {
+        if (new Date(chronological[index - 1].manifest.watermark).getTime() >= new Date(chronological[index].manifest.watermark).getTime()) {
+            throw new Error('snapshot watermark is not monotonic');
+        }
+    }
+    const view = new Map();
+    let appliedManifest = null;
+    for (const entry of chronological) {
+        if (target && new Date(entry.manifest.watermark).getTime() > target.getTime())
+            continue;
+        appliedManifest = entry.manifest;
+        for (const record of readRecords(entry.path, 'normalized/v2')) {
+            const identity = record.resourceIdentity ?? record.sourceIdentity;
+            switch (record.operation) {
+                case 'tombstone':
+                    view.delete(identity);
+                    break;
+                case 'unavailable':
+                    view.set(identity, record);
+                    break;
+                default:
+                    view.set(identity, { ...record, operation: undefined });
+                    break;
+            }
+        }
+    }
+    if (target && !appliedManifest)
+        throw new Error('no verified snapshot is available at the requested as-of time');
+    return {
+        snapshotId,
+        records: [...view.values()].sort((left, right) => left.recordId.localeCompare(right.recordId)),
+        manifest: appliedManifest
+    };
+}
+function materializeJson(root, snapshotId, options = {}) {
+    const result = materializeSnapshot(root, snapshotId, options);
+    return result.records.map((record) => JSON.stringify({ schema: 'normalized/v2', ...record })).join('\n') + (result.records.length ? '\n' : '');
+}
+export { materializeJson, materializeSnapshot };
+//# sourceMappingURL=materialize.js.map

package/dist/lib/process-data/normalize.js

@@ -194,5 +194,38 @@ function normalizeObjects(objects) {
     }
     return records.sort((left, right) => left.recordId.localeCompare(right.recordId));
 }
-export { normalizeObjects };
+function normalizeResources(objects) {
+    const records = [];
+    for (const object of objects) {
+        if (object.role !== 'resource')
+            continue;
+        const identity = object.resourceIdentity ?? object.sourceIdentity;
+        let data = { availability: 'unknown' };
+        try {
+            data = JSON.parse(object.content ?? '{}');
+        }
+        catch {
+            data = { availability: 'unavailable', reason: 'invalid resource JSON' };
+        }
+        const binding = typeof data === 'object' && data !== null && !Array.isArray(data) && typeof data.body === 'string'
+            ? data.body.match(TASK_ID_RE)?.[0]
+            : undefined;
+        records.push({
+            recordId: recordId('platform-resource', identity),
+            kind: 'platform-resource',
+            sourceIdentity: identity,
+            resourceIdentity: identity,
+            sourceSha256: object.sha256,
+            ...(binding ? { binding } : {}),
+            ...(object.eventTime ? { observedAt: { state: 'known', value: object.eventTime } } : {}),
+            ...(object.parentIdentity ? { parentIdentity: object.parentIdentity } : {}),
+            ...(object.pageSha256 ? {
+                evidence: [{ resourceIdentity: identity, resourceSha256: object.sha256, pageSha256: object.pageSha256 }]
+            } : {}),
+            data
+        });
+    }
+    return records.sort((left, right) => left.recordId.localeCompare(right.recordId));
+}
+export { normalizeObjects, normalizeResources };
 //# sourceMappingURL=normalize.js.map

package/dist/lib/process-data/reconcile.js

@@ -91,5 +91,59 @@ function reconcileRecords(records, scope) {
         repairs: repairs.sort((left, right) => left.repairId.localeCompare(right.repairId))
     };
 }
-export { reconcileRecords };
+function reconcileResourceRecords(current, options) {
+    const parent = new Map((options.parent ?? []).filter((record) => record.resourceIdentity).map((record) => [record.resourceIdentity, record]));
+    const currentMap = new Map(current.filter((record) => record.resourceIdentity).map((record) => [record.resourceIdentity, record]));
+    const deferred = new Set(options.deferred ?? []);
+    const unavailable = new Set(options.unavailable ?? []);
+    const records = [];
+    const operations = { upsert: 0, supersede: 0, tombstone: 0, unavailable: 0 };
+    for (const record of currentMap.values()) {
+        const identity = record.resourceIdentity;
+        const previous = parent.get(identity);
+        if (!previous) {
+            records.push({ ...record, operation: 'upsert' });
+            operations.upsert += 1;
+        }
+        else if (previous.sourceSha256 !== record.sourceSha256) {
+            records.push({ ...record, operation: 'supersede' });
+            operations.supersede += 1;
+        }
+    }
+    if (options.full) {
+        for (const [identity, previous] of parent) {
+            if (currentMap.has(identity) || deferred.has(identity) || unavailable.has(identity)
+                || (previous.parentIdentity && deferred.has(previous.parentIdentity))
+                || (previous.parentIdentity && unavailable.has(previous.parentIdentity)))
+                continue;
+            records.push({
+                recordId: previous.recordId,
+                kind: 'platform-resource',
+                sourceIdentity: identity,
+                resourceIdentity: identity,
+                sourceSha256: previous.sourceSha256,
+                operation: 'tombstone',
+                data: { availability: 'tombstone', previousSha256: previous.sourceSha256 }
+            });
+            operations.tombstone += 1;
+        }
+    }
+    for (const identity of unavailable) {
+        if (!parent.has(identity) && !currentMap.has(identity))
+            continue;
+        const previous = parent.get(identity) ?? currentMap.get(identity);
+        records.push({
+            recordId: previous.recordId,
+            kind: 'platform-resource',
+            sourceIdentity: identity,
+            resourceIdentity: identity,
+            sourceSha256: previous.sourceSha256,
+            operation: 'unavailable',
+            data: { availability: 'unavailable', reason: 'source-unavailable' }
+        });
+        operations.unavailable += 1;
+    }
+    return { records: records.sort((left, right) => left.recordId.localeCompare(right.recordId)), operations };
+}
+export { reconcileRecords, reconcileResourceRecords };
 //# sourceMappingURL=reconcile.js.map