@gmickel/gno 2.8.2 → 2.8.4

package/src/ingestion/prepare-convert.ts

@@ -0,0 +1,32 @@
+/**
+ * Conversion used by the file processor backends (fn-198).
+ *
+ * Native files (Markdown, plain text) skip the default registry, whose
+ * PDF/Office adapters take about half a second to load. The default
+ * registry tries these two converters first, so the output is the same.
+ *
+ * @module src/ingestion/prepare-convert
+ */
+
+import type { ConvertInput, PipelineResult } from "../converters/types";
+
+import { markdownConverter } from "../converters/native/markdown";
+import { plaintextConverter } from "../converters/native/plaintext";
+import { ConversionPipeline, getDefaultPipeline } from "../converters/pipeline";
+import { ConverterRegistry } from "../converters/registry";
+
+const nativeRegistry = new ConverterRegistry();
+nativeRegistry.register(markdownConverter);
+nativeRegistry.register(plaintextConverter);
+const nativePipeline = new ConversionPipeline(nativeRegistry);
+
+/** Convert one file, loading the heavy adapters only when needed. */
+export function convertForPreparation(
+  input: ConvertInput
+): Promise<PipelineResult> {
+  return (
+    nativeRegistry.select(input.mime, input.ext)
+      ? nativePipeline
+      : getDefaultPipeline()
+  ).convert(input);
+}

package/src/ingestion/prepare-file.ts

@@ -0,0 +1,272 @@
+/**
+ * Per-file preparation: everything CPU-bound between a file's bytes and its
+ * database write (fn-198).
+ *
+ * Conversion, metadata, tags, memory scopes, code regions, change-journal
+ * structure (previous and next), chunking and link parsing run here as one
+ * step. Sync runs it in the terminable file worker, so a file that overruns
+ * its budget is stopped mid-step; where no worker can start (a compiled
+ * executable, or injected test doubles) it runs in-process and the budget is
+ * checked between steps. The main thread keeps every database read and write.
+ *
+ * @module src/ingestion/prepare-file
+ */
+
+import type { NormalizedContentTypeRule } from "../config";
+import type {
+  ConversionArtifact,
+  ConvertInput,
+  PipelineResult,
+} from "../converters/types";
+import type { DocumentStructureSnapshot } from "../core/change-diff";
+import type { ChunkInput, DocLinkInput } from "../store/types";
+import type { DocumentMetadata } from "./document-metadata";
+import type { ChunkerPort, ChunkParams } from "./types";
+
+import { MARKDOWN_CONVERTER_ID } from "../converters/native/markdown";
+import { extractDocumentStructure } from "../core/change-diff";
+import {
+  normalizeMarkdownPath,
+  normalizeWikiName,
+  parseLinks,
+} from "../core/links";
+import { extractMemoryScopes } from "../core/memory-record";
+import { extractDocumentMetadata, extractTags } from "./document-metadata";
+import { buildLineOffsets } from "./position";
+import { type ExcludedRange, getExcludedRanges } from "./strip";
+
+/** The previously indexed revision, for the change journal. */
+export interface PreviousRevision {
+  markdown: string;
+  relPath: string;
+  dateFields: Record<string, string> | null;
+  converterId: string | null;
+}
+
+export interface PrepareFileRequest {
+  input: ConvertInput;
+  /** Extension used for path-based content-type inference. */
+  metadataExt: string;
+  contentTypeRules: NormalizedContentTypeRule[];
+  chunkParams: ChunkParams;
+  collectionLanguageHint?: string;
+  memoryManaged: boolean;
+  /**
+   * Whether this file is a Markdown source (the Markdown converter handles
+   * it). The previous revision's links are read under this rule too, so a
+   * non-Markdown source never journals link additions or removals.
+   */
+  markdownSource: boolean;
+  previous: PreviousRevision | null;
+}
+
+export interface PreparedFile {
+  artifact: ConversionArtifact;
+  metadata: DocumentMetadata;
+  nextStructure: DocumentStructureSnapshot;
+  chunkInputs: ChunkInput[];
+  linkInputs: DocLinkInput[];
+  tags: string[];
+  /** Managed-memory scopes; null outside memory-managed collections. */
+  memoryScopes: string[] | null;
+}
+
+/** Why a file was not prepared: a conversion error or a budget overrun. */
+export interface PrepareFailure {
+  code: string;
+  message: string;
+  details?: Record<string, unknown>;
+}
+
+export type PrepareOutcome =
+  | { ok: true; value: PreparedFile }
+  | { ok: false; error: PrepareFailure };
+
+/** Named steps, in order; reported as progress and in overrun messages. */
+export const PREPARE_PHASES = [
+  "previous revision",
+  "conversion",
+  "metadata extraction",
+  "code-region detection",
+  "change-journal structure",
+  "chunking",
+  "link extraction",
+] as const;
+
+export type PreparePhase = (typeof PREPARE_PHASES)[number];
+
+export interface PrepareHooks {
+  convert: (input: ConvertInput) => Promise<PipelineResult>;
+  chunker: ChunkerPort;
+  /** Called once the previous revision's structure is known (if any). */
+  onPrevious: (structure: DocumentStructureSnapshot) => void;
+  /** Called as each step starts. */
+  onPhase: (phase: PreparePhase) => void;
+  /** In-process budget checkpoint after a step; null to continue. */
+  check: (phase: PreparePhase) => PrepareFailure | null;
+}
+
+const toLinkInputs = (
+  markdown: string,
+  relPath: string,
+  excludedRanges: ExcludedRange[] | null
+): DocLinkInput[] => {
+  // Only Markdown sources carry links, as in Obsidian: link-shaped text in
+  // code, plain text or converted documents stays searchable prose.
+  if (!excludedRanges) return [];
+  const linkInputs: DocLinkInput[] = [];
+  for (const link of parseLinks(
+    markdown,
+    buildLineOffsets(markdown),
+    excludedRanges
+  )) {
+    let targetRefNorm: string;
+    if (link.kind === "wiki") {
+      targetRefNorm = normalizeWikiName(link.targetRef);
+    } else {
+      // Markdown links with a collection prefix are not supported, and a
+      // link that escapes the collection root is skipped.
+      if (link.targetCollection) continue;
+      const resolved = normalizeMarkdownPath(link.targetRef, relPath);
+      if (!resolved) continue;
+      targetRefNorm = resolved;
+    }
+    linkInputs.push({
+      targetRef: link.targetRef,
+      targetRefNorm,
+      targetAnchor: link.targetAnchor,
+      targetCollection: link.targetCollection,
+      linkType: link.kind,
+      linkText: link.displayText,
+      startLine: link.startLine,
+      startCol: link.startCol,
+      endLine: link.endLine,
+      endCol: link.endCol,
+    });
+  }
+  return linkInputs;
+};
+
+/** Prepare one file for persistence; see the module comment. */
+export async function prepareFile(
+  request: PrepareFileRequest,
+  hooks: PrepareHooks
+): Promise<PrepareOutcome> {
+  const { input, previous } = request;
+  const relPath = input.relativePath;
+  const step = (phase: PreparePhase): PrepareFailure | null =>
+    hooks.check(phase);
+
+  // The previous revision's structure comes first, so a file stopped later
+  // still journals the evidence it had.
+  let previousRanges: ExcludedRange[] | null = null;
+  if (previous) {
+    hooks.onPhase("previous revision");
+    previousRanges = request.markdownSource
+      ? getExcludedRanges(previous.markdown)
+      : null;
+    hooks.onPrevious(
+      extractDocumentStructure(
+        previous.markdown,
+        previous.relPath,
+        previous.dateFields,
+        {
+          markdownSource: request.markdownSource,
+          excludedRanges: previousRanges ?? undefined,
+        }
+      )
+    );
+    const stop = step("previous revision");
+    if (stop) return { ok: false, error: stop };
+  }
+
+  hooks.onPhase("conversion");
+  const converted = await hooks.convert(input);
+  if (!converted.ok) {
+    return {
+      ok: false,
+      error: {
+        code: converted.error.code,
+        message: converted.error.message,
+        details: converted.error.details,
+      },
+    };
+  }
+  const artifact = converted.value;
+  const markdown = artifact.markdown;
+  const stopAfterConversion = step("conversion");
+  if (stopAfterConversion) return { ok: false, error: stopAfterConversion };
+
+  hooks.onPhase("metadata extraction");
+  const metadata = extractDocumentMetadata(
+    markdown,
+    relPath,
+    request.metadataExt,
+    request.contentTypeRules
+  );
+  const tags = extractTags(markdown);
+  const memoryScopes = request.memoryManaged
+    ? extractMemoryScopes(markdown)
+    : null;
+  const stopAfterMetadata = step("metadata extraction");
+  if (stopAfterMetadata) return { ok: false, error: stopAfterMetadata };
+
+  hooks.onPhase("code-region detection");
+  const markdownSource = artifact.meta.converterId === MARKDOWN_CONVERTER_ID;
+  // Unchanged content (a re-ingest) reuses the previous revision's parse.
+  const excludedRanges = !markdownSource
+    ? null
+    : previous && previousRanges && previous.markdown === markdown
+      ? previousRanges
+      : getExcludedRanges(markdown);
+  const stopAfterRanges = step("code-region detection");
+  if (stopAfterRanges) return { ok: false, error: stopAfterRanges };
+
+  hooks.onPhase("change-journal structure");
+  const nextStructure = extractDocumentStructure(
+    markdown,
+    relPath,
+    metadata.dateFields,
+    { markdownSource, excludedRanges: excludedRanges ?? undefined }
+  );
+  const stopAfterStructure = step("change-journal structure");
+  if (stopAfterStructure) return { ok: false, error: stopAfterStructure };
+
+  hooks.onPhase("chunking");
+  const chunkInputs: ChunkInput[] = hooks.chunker
+    .chunk(
+      markdown,
+      request.chunkParams,
+      artifact.languageHint ?? request.collectionLanguageHint,
+      relPath
+    )
+    .map((c) => ({
+      seq: c.seq,
+      pos: c.pos,
+      text: c.text,
+      startLine: c.startLine,
+      endLine: c.endLine,
+      language: c.language ?? undefined,
+      tokenCount: c.tokenCount ?? undefined,
+    }));
+  const stopAfterChunking = step("chunking");
+  if (stopAfterChunking) return { ok: false, error: stopAfterChunking };
+
+  hooks.onPhase("link extraction");
+  const linkInputs = toLinkInputs(markdown, relPath, excludedRanges);
+  const stopAfterLinks = step("link extraction");
+  if (stopAfterLinks) return { ok: false, error: stopAfterLinks };
+
+  return {
+    ok: true,
+    value: {
+      artifact,
+      metadata,
+      nextStructure,
+      chunkInputs,
+      linkInputs,
+      tags,
+      memoryScopes,
+    },
+  };
+}

package/src/ingestion/prepare-worker.ts

@@ -0,0 +1,43 @@
+/**
+ * Worker entry for per-file preparation (fn-198). See `file-processor.ts`.
+ *
+ * @module src/ingestion/prepare-worker
+ */
+
+import type {
+  FileWorkerMessage,
+  FileWorkerRequest,
+  ReadyMessage,
+} from "./file-processor";
+
+import { defaultChunker } from "./chunker";
+import { convertForPreparation } from "./prepare-convert";
+import { PREPARE_PHASES, prepareFile } from "./prepare-file";
+
+declare const self: Worker;
+
+const post = (message: FileWorkerMessage | ReadyMessage): void => {
+  self.postMessage(message);
+};
+
+self.onmessage = async (event: MessageEvent<FileWorkerRequest>) => {
+  // Only the parent that created this worker may drive it; a Bun Worker sees
+  // its parent's messages with an empty origin.
+  if (event.origin !== "") return;
+  const { request, phaseSlot } = event.data;
+  const outcome = await prepareFile(request, {
+    convert: convertForPreparation,
+    chunker: defaultChunker,
+    onPrevious: (structure) => post({ type: "previous", structure }),
+    // Progress goes through shared memory, not messages: the parent only
+    // reads it when it stops a file.
+    onPhase: (phase) => {
+      Atomics.store(phaseSlot, 0, PREPARE_PHASES.indexOf(phase));
+    },
+    // The parent enforces the budget and terminates this worker.
+    check: () => null,
+  });
+  post({ type: "done", outcome });
+};
+
+post({ type: "ready" });

package/src/ingestion/record-container.ts

@@ -320,7 +320,10 @@ const loadPreviousStructure = async (
   return extractDocumentStructure(
     content,
     existing.relPath,
-    existing.dateFields
+    existing.dateFields,
+    {
+      markdownSource: false,
+    }
   );
 };
 
@@ -347,10 +350,12 @@ const persistRecord = async (
     contentType
   );
   const previousStructure = await loadPreviousStructure(input.store, existing);
+  // Records are not Markdown sources: they store and journal no links.
   const nextStructure = extractDocumentStructure(
     record.markdown,
     virtualPath,
-    dateFields
+    dateFields,
+    { markdownSource: false }
   );
   const structureDelta = diffDocumentStructure(
     previousStructure,

package/src/ingestion/strip.ts

@@ -11,12 +11,28 @@
  * Returns EXCLUDED RANGES on the original string - does NOT modify content.
  * This preserves position information for accurate line/column tracking.
  *
+ * Code regions come from the CommonMark + GFM parser. GFM table parsing is
+ * quadratic in the cells of one table (a 20,000-row table took minutes), so
+ * text with a table over the cell budget is parsed without the table
+ * extension and without inline links and inline HTML, which rescan a
+ * table-sized paragraph once per row: code spans, fences, indented code,
+ * blockquotes and lists keep their CommonMark meaning; table-cell
+ * boundaries, link destinations and HTML tags no longer hide a backtick.
+ *
  * @module src/ingestion/strip
  */
 
 import { fromMarkdown } from "mdast-util-from-markdown";
 import { gfmFromMarkdown } from "mdast-util-gfm";
+import { gfmAutolinkLiteralFromMarkdown } from "mdast-util-gfm-autolink-literal";
+import { gfmFootnoteFromMarkdown } from "mdast-util-gfm-footnote";
+import { gfmStrikethroughFromMarkdown } from "mdast-util-gfm-strikethrough";
+import { gfmTaskListItemFromMarkdown } from "mdast-util-gfm-task-list-item";
 import { gfm } from "micromark-extension-gfm";
+import { gfmAutolinkLiteral } from "micromark-extension-gfm-autolink-literal";
+import { gfmFootnote } from "micromark-extension-gfm-footnote";
+import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough";
+import { gfmTaskListItem } from "micromark-extension-gfm-task-list-item";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -45,8 +61,13 @@ export interface ExcludedRange {
 /** Frontmatter at start of file (YAML between --- delimiters) */
 const FRONTMATTER_REGEX = /^---\r?\n[\s\S]*?(?:\r?\n)?---(?:\r?\n|$)/;
 
-/** HTML comments */
-const HTML_COMMENT_REGEX = /<!--[\s\S]*?-->/g;
+/**
+ * Most `|` characters one blank-line-delimited block may hold before the
+ * GFM table extension is left out of the parse. A GFM table only ends at a
+ * blank line (or another block), and its parse cost grows with the square
+ * of its cells: 5,000 cells take about 0.3 s, 26,000 about 8 s.
+ */
+export const MAX_PARSED_TABLE_CELLS = 5000;
 
 /** Backtick (code span or fence) or tilde fence anywhere in the text. */
 const INLINE_OR_FENCE_TRIGGER_REGEX = /`|~~~/;
@@ -87,6 +108,65 @@ const mayContainCode = (markdown: string): boolean => {
   return false;
 };
 
+/**
+ * Whether any blank-line-delimited block holds more table-cell pipes than
+ * the table budget. One linear pass, stopping as soon as the budget trips.
+ */
+const hasOversizedTable = (markdown: string): boolean => {
+  let blockPipes = 0;
+  let lineHasContent = false;
+  for (let index = 0; index < markdown.length; index += 1) {
+    const code = markdown.charCodeAt(index);
+    if (code === 10 /* \n */) {
+      if (!lineHasContent) blockPipes = 0;
+      lineHasContent = false;
+    } else if (code === 124 /* | */) {
+      blockPipes += 1;
+      lineHasContent = true;
+      if (blockPipes > MAX_PARSED_TABLE_CELLS) return true;
+    } else if (code !== 32 && code !== 9 && code !== 13) {
+      lineHasContent = true;
+    }
+  }
+  return false;
+};
+
+/**
+ * Inline constructs left out with the table extension. Without table rules a
+ * large table is one paragraph, and these rescan it once per row: on every
+ * `]` link and image ends walk back through all the paragraph's earlier
+ * events looking for an opening `[` (an unmatched `]` per row made 8,000
+ * rows take 66 s), and inline HTML reads forward from every unclosed `<!--`
+ * to the paragraph's end. They only matter to code detection through a
+ * backtick inside a link destination or title or an inline HTML tag, which
+ * then reads as a code-span delimiter.
+ */
+const PARAGRAPH_RESCANNING_CONSTRUCTS = [
+  "labelEnd",
+  "gfmPotentialFootnoteCall",
+  "htmlText",
+];
+
+/**
+ * GFM without tables: every GFM construct except table rows and cells, and
+ * without the constructs in `PARAGRAPH_RESCANNING_CONSTRUCTS`.
+ */
+const GFM_WITHOUT_TABLES = {
+  extensions: [
+    gfmAutolinkLiteral(),
+    gfmFootnote(),
+    gfmStrikethrough(),
+    gfmTaskListItem(),
+    { disable: { null: PARAGRAPH_RESCANNING_CONSTRUCTS } },
+  ],
+  mdastExtensions: [
+    gfmAutolinkLiteralFromMarkdown(),
+    gfmFootnoteFromMarkdown(),
+    gfmStrikethroughFromMarkdown(),
+    gfmTaskListItemFromMarkdown(),
+  ],
+};
+
 /** Opening fence at the start of a code node: backtick or tilde fence. */
 const FENCE_START_REGEX = /^[\t ]*(?:`{3,}|~{3,})/;
 
@@ -106,16 +186,17 @@ const collectCodeRanges = (
   markdown: string,
   frontmatterEnd: number
 ): ExcludedRange[] => {
-  if (!mayContainCode(markdown.slice(frontmatterEnd))) return [];
   const source =
     frontmatterEnd > 0
       ? markdown.slice(0, frontmatterEnd).replace(/[^\r\n]/gu, " ") +
         markdown.slice(frontmatterEnd)
       : markdown;
-  const root = fromMarkdown(source, {
-    extensions: [gfm()],
-    mdastExtensions: [gfmFromMarkdown()],
-  }) as CodeNode;
+  const root = fromMarkdown(
+    source,
+    hasOversizedTable(source)
+      ? GFM_WITHOUT_TABLES
+      : { extensions: [gfm()], mdastExtensions: [gfmFromMarkdown()] }
+  ) as CodeNode;
   const ranges: ExcludedRange[] = [];
   const visit = (node: CodeNode): void => {
     const start = node.position?.start.offset;
@@ -164,21 +245,22 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
     });
   }
 
-  // 2. HTML comments
-  HTML_COMMENT_REGEX.lastIndex = 0;
-  let match: RegExpExecArray | null;
-  while ((match = HTML_COMMENT_REGEX.exec(markdown)) !== null) {
-    ranges.push({
-      start: match.index,
-      end: match.index + match[0].length,
-      kind: "html_comment",
-    });
+  // 2. HTML comments: each `<!--` to the first `-->` after it. Once no
+  // `-->` follows, no later comment can close either, so the scan stops;
+  // the regex /<!--[\s\S]*?-->/g rescanned to the end from every opener.
+  let open = markdown.indexOf("<!--");
+  while (open !== -1) {
+    const close = markdown.indexOf("-->", open + 4);
+    if (close === -1) break;
+    ranges.push({ start: open, end: close + 3, kind: "html_comment" });
+    open = markdown.indexOf("<!--", close + 3);
   }
 
   // 3. Code spans and fenced or indented code blocks, from the parser.
-  ranges.push(
-    ...collectCodeRanges(markdown, frontmatterMatch?.[0].length ?? 0)
-  );
+  const frontmatterEnd = frontmatterMatch?.[0].length ?? 0;
+  if (mayContainCode(markdown.slice(frontmatterEnd))) {
+    ranges.push(...collectCodeRanges(markdown, frontmatterEnd));
+  }
 
   // Sort by start position for efficient lookup
   ranges.sort((a, b) => a.start - b.start);
@@ -218,24 +300,47 @@ export function isExcluded(
   return false;
 }
 
+/**
+ * Largest `end` among each prefix of a start-sorted range list, built once
+ * per list. Ranges can overlap (a comment around a code span), so a range
+ * list's ends are not sorted and the prefix maximum stands in for them.
+ */
+const prefixMaxEnds = new WeakMap<readonly ExcludedRange[], Int32Array>();
+
+const getPrefixMaxEnds = (ranges: readonly ExcludedRange[]): Int32Array => {
+  let maxEnds = prefixMaxEnds.get(ranges);
+  if (maxEnds?.length !== ranges.length) {
+    maxEnds = new Int32Array(ranges.length);
+    let maxEnd = -1;
+    for (const [index, range] of ranges.entries()) {
+      maxEnd = Math.max(maxEnd, range.end);
+      maxEnds[index] = maxEnd;
+    }
+    prefixMaxEnds.set(ranges, maxEnds);
+  }
+  return maxEnds;
+};
+
 /**
  * Check if a range [start, end) intersects any excluded range.
- * More precise than isExcluded for multi-character matches.
+ * More precise than isExcluded for multi-character matches. Ranges must be
+ * sorted by start, as getExcludedRanges returns them; one lookup is a binary
+ * search, so checking every link of a large note stays linear.
  */
 export function rangeIntersectsExcluded(
   start: number,
   end: number,
-  excludedRanges: ExcludedRange[]
+  excludedRanges: readonly ExcludedRange[]
 ): boolean {
-  for (const range of excludedRanges) {
-    // Two ranges [a, b) and [c, d) intersect if a < d && c < b
-    if (start < range.end && range.start < end) {
-      return true;
-    }
-    // Early exit if we've passed the range (ranges are sorted)
-    if (range.start >= end) {
-      break;
-    }
+  // Ranges starting before `end` form a prefix; one of them intersects
+  // [start, end) exactly when the largest end among them passes `start`.
+  let low = 0;
+  let high = excludedRanges.length;
+  while (low < high) {
+    const mid = (low + high) >>> 1;
+    if ((excludedRanges[mid]?.start ?? end) < end) low = mid + 1;
+    else high = mid;
   }
-  return false;
+  if (low === 0) return false;
+  return (getPrefixMaxEnds(excludedRanges)[low - 1] ?? -1) > start;
 }

package/src/ingestion/sync-options.ts

@@ -20,13 +20,19 @@ export function resolveContentTypeRules(
 
 export function withContentTypeRules(
   options: SyncOptions = {},
-  config?: Pick<Config, "contentTypes" | "chunking">
+  config?: Pick<Config, "contentTypes" | "chunking" | "conversion">
 ): SyncOptions {
   const rules = options.contentTypeRules ?? resolveContentTypeRules(config);
   const chunking = options.chunking ?? config?.chunking;
+  const conversion = config?.conversion;
+  // Explicit per-call limits win over the configured conversion budget.
+  const limits = conversion
+    ? { ...conversion, ...options.limits }
+    : options.limits;
   return {
     ...options,
     ...(chunking ? { chunking } : {}),
+    ...(limits ? { limits } : {}),
     contentTypeRules: rules,
     contentTypeRulesFingerprint:
       options.contentTypeRulesFingerprint ??