@gmickel/gno 1.27.0 → 1.28.0

package/src/ingestion/record-sync.ts

@@ -0,0 +1,70 @@
+import type { StoredRecordState } from "../store/types";
+import type { CanonicalRecord, RecordAdapterRunResult } from "./record-adapter";
+
+export type RecordSyncAction =
+  | { type: "add"; record: CanonicalRecord }
+  | { type: "update"; previous: StoredRecordState; record: CanonicalRecord }
+  | {
+      type: "reactivate";
+      previous: StoredRecordState;
+      record: CanonicalRecord;
+    }
+  | { type: "unchanged"; previous: StoredRecordState; record?: CanonicalRecord }
+  | { type: "deactivate"; previous: StoredRecordState }
+  | { type: "preserve"; previous: StoredRecordState };
+
+export interface RecordSyncPlan {
+  authoritative: boolean;
+  actions: RecordSyncAction[];
+}
+
+/**
+ * Pure snapshot reconciliation. Persistence/transactions are intentionally
+ * deferred until the virtual-record lineage schema exists.
+ */
+export function reconcileRecordSnapshot(
+  priorRecords: readonly StoredRecordState[],
+  snapshot: RecordAdapterRunResult
+): RecordSyncPlan {
+  const priorByKey = new Map(
+    priorRecords.map((record) => [record.recordKey, record])
+  );
+  const seen = new Set<string>();
+  const failed = new Set(snapshot.failedRecordKeys);
+  const actions: RecordSyncAction[] = [];
+
+  for (const record of snapshot.records) {
+    seen.add(record.recordKey);
+    const previous = priorByKey.get(record.recordKey);
+    if (!previous) {
+      actions.push({ type: "add", record });
+      continue;
+    }
+    if (!previous.active) {
+      actions.push({ type: "reactivate", previous, record });
+      continue;
+    }
+    if (
+      previous.sourceHash === record.sourceHash &&
+      previous.adapterVersion === record.adapterVersion &&
+      previous.adapterFingerprint === record.adapterFingerprint
+    ) {
+      actions.push({ type: "unchanged", previous, record });
+      continue;
+    }
+    actions.push({ type: "update", previous, record });
+  }
+
+  for (const previous of priorRecords) {
+    if (seen.has(previous.recordKey)) continue;
+    if (failed.has(previous.recordKey) || !snapshot.authoritative) {
+      actions.push({ type: "preserve", previous });
+    } else if (previous.active) {
+      actions.push({ type: "deactivate", previous });
+    } else {
+      actions.push({ type: "unchanged", previous });
+    }
+  }
+
+  return { authoritative: snapshot.authoritative, actions };
+}

package/src/ingestion/sync.ts

@@ -5,10 +5,10 @@
  * @module src/ingestion/sync
  */
 
-// node:fs/promises for stat (no Bun equivalent for file stats)
-import { stat } from "node:fs/promises";
+// node:fs/promises for realpath/stat (no Bun equivalent for canonical paths or file stats)
+import { realpath, stat } from "node:fs/promises";
 // node:path for join (no Bun path utils)
-import { join } from "node:path";
+import { isAbsolute, join, relative, sep } from "node:path";
 
 import type { NormalizedContentTypeRule } from "../config";
 import type { Collection } from "../config/types";
@@ -16,6 +16,7 @@ import type {
   ChunkInput,
   DocEdgeInput,
   DocLinkInput,
+  DocumentInput,
   DocumentRow,
   IngestErrorInput,
   StorePort,
@@ -37,12 +38,14 @@ import {
   fingerprintContentTypeMetadataRules,
   resolveContentTypeRule,
 } from "../config";
+import { createJsonlAdapter } from "../converters/adapters/jsonl/adapter";
+import { createTranscriptAdapter } from "../converters/adapters/transcript/adapter";
 import { getDefaultMimeDetector, type MimeDetector } from "../converters/mime";
 import {
   type ConversionPipeline,
   getDefaultPipeline,
 } from "../converters/pipeline";
-import { DEFAULT_LIMITS } from "../converters/types";
+import { DEFAULT_LIMITS, type RecordAdapter } from "../converters/types";
 import {
   diffDocumentStructure,
   extractDocumentStructure,
@@ -64,6 +67,7 @@ import {
   stripFrontmatter,
 } from "./frontmatter";
 import { buildLineOffsets } from "./position";
+import { processRecordContainer } from "./record-container";
 import { getExcludedRanges } from "./strip";
 import { collectionToWalkConfig, DEFAULT_CHUNK_PARAMS } from "./types";
 import { defaultWalker } from "./walker";
@@ -542,6 +546,44 @@ function mustOk<T>(
   return result.value;
 }
 
+const preserveDocumentWithError = (
+  existing: DocumentRow,
+  code: string,
+  message: string
+): DocumentInput => ({
+  collection: existing.collection,
+  relPath: existing.relPath,
+  sourceHash: existing.sourceHash,
+  sourceMime: existing.sourceMime,
+  sourceExt: existing.sourceExt,
+  sourceSize: existing.sourceSize,
+  sourceMtime: existing.sourceMtime,
+  sourceCtime: existing.sourceCtime ?? existing.sourceMtime,
+  title: existing.title ?? undefined,
+  mirrorHash: existing.mirrorHash ?? undefined,
+  converterId: existing.converterId ?? undefined,
+  converterVersion: existing.converterVersion ?? undefined,
+  languageHint: existing.languageHint ?? undefined,
+  contentType: existing.contentType ?? undefined,
+  contentTypeSource: existing.contentTypeSource ?? undefined,
+  categories: existing.categories ?? undefined,
+  author: existing.author ?? undefined,
+  frontmatterDate: existing.frontmatterDate ?? undefined,
+  dateFields: existing.dateFields ?? undefined,
+  recordKey: existing.recordKey ?? undefined,
+  recordSourcePath: existing.recordSourcePath ?? undefined,
+  recordSourceLocator: existing.recordSourceLocator ?? undefined,
+  recordMetadata: existing.recordMetadata ?? undefined,
+  recordAnchors: existing.recordAnchors ?? undefined,
+  recordAdapterFingerprint: existing.recordAdapterFingerprint ?? undefined,
+  lastErrorCode: code,
+  lastErrorMessage: message,
+  ingestVersion: existing.ingestVersion ?? undefined,
+  contentTypeRulesFingerprint:
+    existing.contentTypeRulesFingerprint ?? undefined,
+  changeJournal: false,
+});
+
 /**
  * Simple semaphore for bounded concurrency.
  */
@@ -594,6 +636,30 @@ export class SyncService {
     this.pipeline = pipeline ?? getDefaultPipeline();
   }
 
+  private async selectRecordAdapter(
+    collection: Collection,
+    mime: string,
+    ext: string
+  ): Promise<RecordAdapter | undefined> {
+    const transcriptConfig = collection.recordAdapters?.transcript;
+    if (transcriptConfig) {
+      const adapter = createTranscriptAdapter(transcriptConfig);
+      if (adapter.canHandle(mime, ext)) return adapter;
+    }
+    const jsonlConfig = collection.recordAdapters?.jsonl;
+    if (jsonlConfig) {
+      const adapter = createJsonlAdapter(jsonlConfig.fieldMapping);
+      if (adapter.canHandle(mime, ext)) return adapter;
+    }
+    const selectDefault = (
+      this.pipeline as ConversionPipeline & {
+        selectRecordAdapter?: ConversionPipeline["selectRecordAdapter"];
+      }
+    ).selectRecordAdapter;
+    if (!selectDefault) return undefined;
+    return selectDefault.call(this.pipeline, mime, ext);
+  }
+
   /**
    * Process a single file through the ingestion pipeline.
    * All store operations are checked and errors are propagated.
@@ -661,17 +727,57 @@ export class SyncService {
         };
       }
 
-      // 2. Read file bytes
+      const extensionMime = this.mimeDetector.detect(
+        entry.relPath,
+        new Uint8Array()
+      );
+      const recordAdapter = await this.selectRecordAdapter(
+        collection,
+        extensionMime.mime,
+        extensionMime.ext
+      );
+      const contentTypeRules = options.contentTypeRules ?? [];
+      const contentTypeRulesFingerprint =
+        options.contentTypeRulesFingerprint ??
+        fingerprintContentTypeMetadataRules(contentTypeRules);
+      if (recordAdapter) {
+        return await processRecordContainer({
+          adapter: recordAdapter,
+          chunker: this.chunker,
+          collection,
+          contentTypeRules,
+          contentTypeRulesFingerprint,
+          entry,
+          ext: extensionMime.ext,
+          extractMetadata: extractDocumentMetadata,
+          ingestVersion: INGEST_VERSION,
+          mime: extensionMime.mime,
+          options,
+          sourceCtime,
+          sourceMtime,
+          sourceSize,
+          store,
+        });
+      }
+
+      const sniffBytes = new Uint8Array(
+        await Bun.file(entry.absPath).slice(0, 512).arrayBuffer()
+      );
+      const mime = this.mimeDetector.detect(entry.relPath, sniffBytes);
+
+      const priorRecordDocuments = mustOk(
+        await store.listRecordDocuments(collection.name, entry.relPath),
+        "listRecordDocuments",
+        { collection: collection.name, relPath: entry.relPath }
+      );
+
+      // 2. Read byte-oriented files only after record-adapter selection.
       const bytes = await Bun.file(entry.absPath).bytes();
 
       // 3. Compute sourceHash
       const hasher = new Bun.CryptoHasher("sha256");
       hasher.update(bytes);
       const sourceHash = hasher.digest("hex");
-      const contentTypeRules = options.contentTypeRules ?? [];
-      const contentTypeRulesFingerprint =
-        options.contentTypeRulesFingerprint ??
-        fingerprintContentTypeMetadataRules(contentTypeRules);
 
       // 4. Check existing doc for skip/repair decision
       const existingResult = await store.getDocument(
@@ -686,12 +792,20 @@ export class SyncService {
       );
 
       if (decision.kind === "skip") {
+        const activeRecordPaths = priorRecordDocuments
+          .filter((document) => document.active)
+          .map((document) => document.relPath);
+        if (activeRecordPaths.length > 0) {
+          mustOk(
+            await store.markInactive(collection.name, activeRecordPaths),
+            "markInactive",
+            { collection: collection.name, relPath: entry.relPath }
+          );
+          return { relPath: entry.relPath, status: "updated" };
+        }
         return { relPath: entry.relPath, status: "unchanged" };
       }
 
-      // 5. Detect MIME (bytes is already Uint8Array from Bun.file().bytes())
-      const mime = this.mimeDetector.detect(entry.absPath, bytes);
-
       // 6. Convert via pipeline
       const convertResult = await this.pipeline.convert({
         sourcePath: entry.absPath,
@@ -938,7 +1052,19 @@ export class SyncService {
           linkCount: linkInputs.length,
         });
 
-        const status = existing ? "updated" : "added";
+        const activeRecordPaths = priorRecordDocuments
+          .filter((document) => document.active)
+          .map((document) => document.relPath);
+        if (activeRecordPaths.length > 0) {
+          mustOk(
+            await store.markInactive(collection.name, activeRecordPaths),
+            "markInactive",
+            { collection: collection.name, relPath: entry.relPath }
+          );
+        }
+
+        const status =
+          existing || priorRecordDocuments.length > 0 ? "updated" : "added";
         return {
           relPath: entry.relPath,
           status,
@@ -982,22 +1108,10 @@ export class SyncService {
           collection.name,
           entry.relPath
         );
-        if (existingResult.ok && existingResult.value) {
-          await store.upsertDocument({
-            collection: collection.name,
-            relPath: entry.relPath,
-            sourceHash: existingResult.value.sourceHash,
-            sourceMime: existingResult.value.sourceMime,
-            sourceExt: existingResult.value.sourceExt,
-            sourceSize: existingResult.value.sourceSize,
-            sourceMtime: existingResult.value.sourceMtime,
-            sourceCtime:
-              existingResult.value.sourceCtime ??
-              existingResult.value.sourceMtime,
-            lastErrorCode: code,
-            lastErrorMessage: message,
-            changeJournal: false,
-          });
+        if (existingResult.ok && existingResult.value?.active) {
+          await store.upsertDocument(
+            preserveDocumentWithError(existingResult.value, code, message)
+          );
         }
       } catch {
         // Best-effort error recording
@@ -1065,6 +1179,20 @@ export class SyncService {
     let markedInactive = 0;
 
     for (const relPath of relPaths) {
+      const recordDocumentsResult = await store.listRecordDocuments(
+        collection.name,
+        relPath
+      );
+      if (!recordDocumentsResult.ok) {
+        results.push({
+          relPath,
+          status: "error",
+          errorCode: recordDocumentsResult.error.code,
+          errorMessage: recordDocumentsResult.error.message,
+        });
+        continue;
+      }
+      const recordDocuments = recordDocumentsResult.value;
       const existingResult = await store.getDocument(collection.name, relPath);
       const existingDoc = existingResult.ok ? existingResult.value : null;
       if (existingDoc) {
@@ -1074,6 +1202,13 @@ export class SyncService {
           projectionSourceIds
         );
       }
+      for (const recordDocument of recordDocuments) {
+        await this.collectProjectionSourceIds(
+          store,
+          recordDocument.id,
+          projectionSourceIds
+        );
+      }
 
       const absPath = join(collection.path, relPath);
       let stats: Awaited<ReturnType<typeof stat>>;
@@ -1094,10 +1229,17 @@ export class SyncService {
           });
           continue;
         }
-        if (existingDoc?.active) {
-          const inactiveResult = await store.markInactive(collection.name, [
-            relPath,
-          ]);
+        const activePaths = [
+          ...(existingDoc?.active ? [relPath] : []),
+          ...recordDocuments
+            .filter((document) => document.active)
+            .map((document) => document.relPath),
+        ];
+        if (activePaths.length > 0) {
+          const inactiveResult = await store.markInactive(
+            collection.name,
+            activePaths
+          );
           if (!inactiveResult.ok) {
             results.push({
               relPath,
@@ -1111,7 +1253,7 @@ export class SyncService {
           results.push({
             relPath,
             status: "updated",
-            docid: existingDoc.docid,
+            docid: existingDoc?.docid,
           });
           continue;
         }
@@ -1133,8 +1275,39 @@ export class SyncService {
         continue;
       }
 
+      let canonicalSourcePath: string;
+      try {
+        const [collectionRoot, sourcePath] = await Promise.all([
+          realpath(collection.path),
+          realpath(absPath),
+        ]);
+        const sourceRelative = relative(collectionRoot, sourcePath);
+        if (
+          sourceRelative === ".." ||
+          sourceRelative.startsWith(`..${sep}`) ||
+          isAbsolute(sourceRelative)
+        ) {
+          results.push({
+            relPath,
+            status: "error",
+            errorCode: "PATH_OUTSIDE_COLLECTION",
+            errorMessage: "Source path resolves outside the collection root.",
+          });
+          continue;
+        }
+        canonicalSourcePath = sourcePath;
+      } catch {
+        results.push({
+          relPath,
+          status: "error",
+          errorCode: "PATH_UNRESOLVED",
+          errorMessage: "Source path could not be resolved safely.",
+        });
+        continue;
+      }
+
       const entry: WalkEntry = {
-        absPath,
+        absPath: canonicalSourcePath,
         relPath,
         size: stats.size,
         mtime: stats.mtime.toISOString(),
@@ -1157,6 +1330,21 @@ export class SyncService {
           projectionSourceIds
         );
       }
+      const currentDocuments = await store.listRecordDocuments(
+        collection.name,
+        relPath
+      );
+      if (currentDocuments.ok) {
+        for (const recordDocument of currentDocuments.value) {
+          if (recordDocument.active) {
+            await this.collectProjectionSourceIds(
+              store,
+              recordDocument.id,
+              projectionSourceIds
+            );
+          }
+        }
+      }
     }
 
     const errors =
@@ -1600,7 +1788,11 @@ export class SyncService {
     const existingDocsResult = await store.listDocuments(collection.name);
     if (existingDocsResult.ok) {
       const missingPaths = existingDocsResult.value
-        .filter((d) => d.active && !seenPaths.has(d.relPath))
+        .filter(
+          (document) =>
+            document.active &&
+            !seenPaths.has(document.recordSourcePath ?? document.relPath)
+        )
         .map((d) => d.relPath);
 
       if (missingPaths.length > 0) {

package/src/ingestion/types.ts

@@ -7,6 +7,10 @@
 
 import type { NormalizedContentTypeRule } from "../config";
 import type { Collection } from "../config/types";
+import type {
+  RecordAdapterFailure,
+  RecordAttachmentInventoryItem,
+} from "../converters/types";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Walker Types
@@ -32,8 +36,10 @@ export interface WalkConfig {
   root: string;
   /** Glob pattern (default: **\/*) */
   pattern: string;
-  /** Extension allowlist (empty = all) */
+  /** Extension allowlist (empty = supported defaults) */
   include: string[];
+  /** Adapter-configured extensions added only to the supported defaults. */
+  additionalDefaultExtensions?: string[];
   /** Paths/patterns to exclude */
   exclude: string[];
   /** Max file size in bytes (files larger are skipped) */
@@ -147,6 +153,34 @@ export type ContentTypeSource =
   | "path-ext"
   | "fallback";
 
+/** Maximum per-record actions retained in one sync receipt. */
+export const MAX_RECORD_IMPORT_RECEIPT_ITEMS = 1_000;
+
+export type RecordImportOutcome =
+  | "added"
+  | "updated"
+  | "reactivated"
+  | "unchanged"
+  | "deactivated"
+  | "preserved";
+
+/** Bounded, privacy-safe identity and provenance for one reconciled record. */
+export interface RecordImportItemReceipt {
+  outcome: RecordImportOutcome;
+  recordKey: string;
+  sourceLocator: string;
+  sourceHash: string;
+  mirrorHash?: string;
+  adapterFingerprint: string;
+  attachments: RecordAttachmentInventoryItem[];
+}
+
+export interface RecordImportWarning {
+  code: "PARTIAL_SNAPSHOT";
+  message: string;
+  retryable: boolean;
+}
+
 /** Per-file sync status */
 export type FileSyncStatus =
   | "added"
@@ -165,6 +199,29 @@ export interface FileSyncResult {
   contentTypeSource?: ContentTypeSource;
   errorCode?: string;
   errorMessage?: string;
+  recordImport?: {
+    adapterId: string;
+    adapterVersion: string;
+    adapterFingerprint: string;
+    snapshotState: "complete" | "partial";
+    authoritative: boolean;
+    stoppedByCap: boolean;
+    sourceBytesRead: number;
+    records: {
+      accepted: number;
+      added: number;
+      updated: number;
+      reactivated: number;
+      unchanged: number;
+      deactivated: number;
+      preserved: number;
+      failed: number;
+    };
+    items: RecordImportItemReceipt[];
+    itemsTruncated: number;
+    warnings: RecordImportWarning[];
+    failures: RecordAdapterFailure[];
+  };
 }
 
 /** Collection sync summary */
@@ -221,6 +278,13 @@ export interface LanguageDetectorPort {
 // Helper to create WalkConfig from Collection
 // ─────────────────────────────────────────────────────────────────────────────
 
+const TRANSCRIPT_EXTENSION_BY_FORMAT = {
+  json: ".json",
+  srt: ".srt",
+  text: ".txt",
+  vtt: ".vtt",
+} as const;
+
 /**
  * Create WalkConfig from Collection with maxBytes override.
  */
@@ -228,10 +292,14 @@ export function collectionToWalkConfig(
   collection: Collection,
   maxBytes: number
 ): WalkConfig {
+  const transcriptFormat = collection.recordAdapters?.transcript?.format;
   return {
     root: collection.path,
     pattern: collection.pattern,
     include: collection.include,
+    additionalDefaultExtensions: transcriptFormat
+      ? [TRANSCRIPT_EXTENSION_BY_FORMAT[transcriptFormat]]
+      : [],
     exclude: collection.exclude,
     maxBytes,
   };

package/src/ingestion/walker.ts

@@ -21,6 +21,7 @@ import type { SkippedEntry, WalkConfig, WalkEntry, WalkerPort } from "./types";
 
 import { SUPPORTED_EXTENSIONS } from "../converters/mime";
 import { matchesCollectionExclusion } from "../core/path-rules";
+import { isRecordVirtualPath } from "./record-path";
 
 /**
  * Regex to detect dangerous patterns with parent directory traversal.
@@ -124,7 +125,7 @@ function scanPatterns(pattern: string): string[] {
 async function safeRelPath(
   rootReal: string,
   absPath: string
-): Promise<string | null> {
+): Promise<{ absPath: string; relPath: string } | null> {
   try {
     const fileReal = await realpath(absPath);
     const rel = relative(rootReal, fileReal);
@@ -135,7 +136,7 @@ async function safeRelPath(
       return null;
     }
 
-    return toPosixPath(rel);
+    return { absPath: fileReal, relPath: toPosixPath(rel) };
   } catch {
     // Can't resolve path (e.g., broken symlink)
     return null;
@@ -145,18 +146,25 @@ async function safeRelPath(
 /**
  * Check if a file extension matches the include list.
  * Include list contains extensions like ".md" or "md" (normalized).
- * When include is empty, falls back to SUPPORTED_EXTENSIONS to avoid
- * walking files that can't be converted.
+ * When include is empty, falls back to SUPPORTED_EXTENSIONS plus extensions
+ * made convertible by explicit adapter configuration.
  */
-function matchesInclude(relPath: string, include: string[]): boolean {
+function matchesInclude(
+  relPath: string,
+  include: string[],
+  additionalDefaultExtensions: string[]
+): boolean {
   const ext = extname(relPath).toLowerCase();
   if (!ext) {
     return false;
   }
 
-  // Fallback to supported extensions when no explicit include list
+  if (include.length === 0 && SUPPORTED_EXTENSIONS.includes(ext)) {
+    return true;
+  }
+
   const effectiveInclude =
-    include.length === 0 ? SUPPORTED_EXTENSIONS : include;
+    include.length === 0 ? additionalDefaultExtensions : include;
 
   return effectiveInclude.some((inc) => {
     const normalizedInc = inc.startsWith(".")
@@ -211,13 +219,19 @@ export class FileWalker implements WalkerPort {
       }
     }
 
-    for (const absPath of [...matches].sort()) {
+    for (const matchedPath of [...matches].sort()) {
       // Security: Compute safe relative path (validates file is within root)
-      const relPath = await safeRelPath(rootReal, absPath);
-      if (relPath === null) {
+      const safePath = await safeRelPath(rootReal, matchedPath);
+      if (safePath === null) {
         // File outside root or unresolvable - silently skip (security)
         continue;
       }
+      const { absPath, relPath } = safePath;
+
+      if (isRecordVirtualPath(relPath)) {
+        skipped.push({ absPath, relPath, reason: "EXCLUDED" });
+        continue;
+      }
 
       // Check exclude patterns
       if (matchesCollectionExclusion(relPath, config.exclude)) {
@@ -230,7 +244,13 @@ export class FileWalker implements WalkerPort {
       }
 
       // Check include extensions
-      if (!matchesInclude(relPath, config.include)) {
+      if (
+        !matchesInclude(
+          relPath,
+          config.include,
+          config.additionalDefaultExtensions ?? []
+        )
+      ) {
         skipped.push({
           absPath,
           relPath,