@gmickel/gno 2.8.2 → 2.8.3

package/src/ingestion/prepare-file.ts

@@ -0,0 +1,272 @@
+/**
+ * Per-file preparation: everything CPU-bound between a file's bytes and its
+ * database write (fn-198).
+ *
+ * Conversion, metadata, tags, memory scopes, code regions, change-journal
+ * structure (previous and next), chunking and link parsing run here as one
+ * step. Sync runs it in the terminable file worker, so a file that overruns
+ * its budget is stopped mid-step; where no worker can start (a compiled
+ * executable, or injected test doubles) it runs in-process and the budget is
+ * checked between steps. The main thread keeps every database read and write.
+ *
+ * @module src/ingestion/prepare-file
+ */
+
+import type { NormalizedContentTypeRule } from "../config";
+import type {
+  ConversionArtifact,
+  ConvertInput,
+  PipelineResult,
+} from "../converters/types";
+import type { DocumentStructureSnapshot } from "../core/change-diff";
+import type { ChunkInput, DocLinkInput } from "../store/types";
+import type { DocumentMetadata } from "./document-metadata";
+import type { ChunkerPort, ChunkParams } from "./types";
+
+import { MARKDOWN_CONVERTER_ID } from "../converters/native/markdown";
+import { extractDocumentStructure } from "../core/change-diff";
+import {
+  normalizeMarkdownPath,
+  normalizeWikiName,
+  parseLinks,
+} from "../core/links";
+import { extractMemoryScopes } from "../core/memory-record";
+import { extractDocumentMetadata, extractTags } from "./document-metadata";
+import { buildLineOffsets } from "./position";
+import { type ExcludedRange, getExcludedRanges } from "./strip";
+
+/** The previously indexed revision, for the change journal. */
+export interface PreviousRevision {
+  markdown: string;
+  relPath: string;
+  dateFields: Record<string, string> | null;
+  converterId: string | null;
+}
+
+export interface PrepareFileRequest {
+  input: ConvertInput;
+  /** Extension used for path-based content-type inference. */
+  metadataExt: string;
+  contentTypeRules: NormalizedContentTypeRule[];
+  chunkParams: ChunkParams;
+  collectionLanguageHint?: string;
+  memoryManaged: boolean;
+  /**
+   * Whether this file is a Markdown source (the Markdown converter handles
+   * it). The previous revision's links are read under this rule too, so a
+   * non-Markdown source never journals link additions or removals.
+   */
+  markdownSource: boolean;
+  previous: PreviousRevision | null;
+}
+
+export interface PreparedFile {
+  artifact: ConversionArtifact;
+  metadata: DocumentMetadata;
+  nextStructure: DocumentStructureSnapshot;
+  chunkInputs: ChunkInput[];
+  linkInputs: DocLinkInput[];
+  tags: string[];
+  /** Managed-memory scopes; null outside memory-managed collections. */
+  memoryScopes: string[] | null;
+}
+
+/** Why a file was not prepared: a conversion error or a budget overrun. */
+export interface PrepareFailure {
+  code: string;
+  message: string;
+  details?: Record<string, unknown>;
+}
+
+export type PrepareOutcome =
+  | { ok: true; value: PreparedFile }
+  | { ok: false; error: PrepareFailure };
+
+/** Named steps, in order; reported as progress and in overrun messages. */
+export const PREPARE_PHASES = [
+  "previous revision",
+  "conversion",
+  "metadata extraction",
+  "code-region detection",
+  "change-journal structure",
+  "chunking",
+  "link extraction",
+] as const;
+
+export type PreparePhase = (typeof PREPARE_PHASES)[number];
+
+export interface PrepareHooks {
+  convert: (input: ConvertInput) => Promise<PipelineResult>;
+  chunker: ChunkerPort;
+  /** Called once the previous revision's structure is known (if any). */
+  onPrevious: (structure: DocumentStructureSnapshot) => void;
+  /** Called as each step starts. */
+  onPhase: (phase: PreparePhase) => void;
+  /** In-process budget checkpoint after a step; null to continue. */
+  check: (phase: PreparePhase) => PrepareFailure | null;
+}
+
+const toLinkInputs = (
+  markdown: string,
+  relPath: string,
+  excludedRanges: ExcludedRange[] | null
+): DocLinkInput[] => {
+  // Only Markdown sources carry links, as in Obsidian: link-shaped text in
+  // code, plain text or converted documents stays searchable prose.
+  if (!excludedRanges) return [];
+  const linkInputs: DocLinkInput[] = [];
+  for (const link of parseLinks(
+    markdown,
+    buildLineOffsets(markdown),
+    excludedRanges
+  )) {
+    let targetRefNorm: string;
+    if (link.kind === "wiki") {
+      targetRefNorm = normalizeWikiName(link.targetRef);
+    } else {
+      // Markdown links with a collection prefix are not supported, and a
+      // link that escapes the collection root is skipped.
+      if (link.targetCollection) continue;
+      const resolved = normalizeMarkdownPath(link.targetRef, relPath);
+      if (!resolved) continue;
+      targetRefNorm = resolved;
+    }
+    linkInputs.push({
+      targetRef: link.targetRef,
+      targetRefNorm,
+      targetAnchor: link.targetAnchor,
+      targetCollection: link.targetCollection,
+      linkType: link.kind,
+      linkText: link.displayText,
+      startLine: link.startLine,
+      startCol: link.startCol,
+      endLine: link.endLine,
+      endCol: link.endCol,
+    });
+  }
+  return linkInputs;
+};
+
+/** Prepare one file for persistence; see the module comment. */
+export async function prepareFile(
+  request: PrepareFileRequest,
+  hooks: PrepareHooks
+): Promise<PrepareOutcome> {
+  const { input, previous } = request;
+  const relPath = input.relativePath;
+  const step = (phase: PreparePhase): PrepareFailure | null =>
+    hooks.check(phase);
+
+  // The previous revision's structure comes first, so a file stopped later
+  // still journals the evidence it had.
+  let previousRanges: ExcludedRange[] | null = null;
+  if (previous) {
+    hooks.onPhase("previous revision");
+    previousRanges = request.markdownSource
+      ? getExcludedRanges(previous.markdown)
+      : null;
+    hooks.onPrevious(
+      extractDocumentStructure(
+        previous.markdown,
+        previous.relPath,
+        previous.dateFields,
+        {
+          markdownSource: request.markdownSource,
+          excludedRanges: previousRanges ?? undefined,
+        }
+      )
+    );
+    const stop = step("previous revision");
+    if (stop) return { ok: false, error: stop };
+  }
+
+  hooks.onPhase("conversion");
+  const converted = await hooks.convert(input);
+  if (!converted.ok) {
+    return {
+      ok: false,
+      error: {
+        code: converted.error.code,
+        message: converted.error.message,
+        details: converted.error.details,
+      },
+    };
+  }
+  const artifact = converted.value;
+  const markdown = artifact.markdown;
+  const stopAfterConversion = step("conversion");
+  if (stopAfterConversion) return { ok: false, error: stopAfterConversion };
+
+  hooks.onPhase("metadata extraction");
+  const metadata = extractDocumentMetadata(
+    markdown,
+    relPath,
+    request.metadataExt,
+    request.contentTypeRules
+  );
+  const tags = extractTags(markdown);
+  const memoryScopes = request.memoryManaged
+    ? extractMemoryScopes(markdown)
+    : null;
+  const stopAfterMetadata = step("metadata extraction");
+  if (stopAfterMetadata) return { ok: false, error: stopAfterMetadata };
+
+  hooks.onPhase("code-region detection");
+  const markdownSource = artifact.meta.converterId === MARKDOWN_CONVERTER_ID;
+  // Unchanged content (a re-ingest) reuses the previous revision's parse.
+  const excludedRanges = !markdownSource
+    ? null
+    : previous && previousRanges && previous.markdown === markdown
+      ? previousRanges
+      : getExcludedRanges(markdown);
+  const stopAfterRanges = step("code-region detection");
+  if (stopAfterRanges) return { ok: false, error: stopAfterRanges };
+
+  hooks.onPhase("change-journal structure");
+  const nextStructure = extractDocumentStructure(
+    markdown,
+    relPath,
+    metadata.dateFields,
+    { markdownSource, excludedRanges: excludedRanges ?? undefined }
+  );
+  const stopAfterStructure = step("change-journal structure");
+  if (stopAfterStructure) return { ok: false, error: stopAfterStructure };
+
+  hooks.onPhase("chunking");
+  const chunkInputs: ChunkInput[] = hooks.chunker
+    .chunk(
+      markdown,
+      request.chunkParams,
+      artifact.languageHint ?? request.collectionLanguageHint,
+      relPath
+    )
+    .map((c) => ({
+      seq: c.seq,
+      pos: c.pos,
+      text: c.text,
+      startLine: c.startLine,
+      endLine: c.endLine,
+      language: c.language ?? undefined,
+      tokenCount: c.tokenCount ?? undefined,
+    }));
+  const stopAfterChunking = step("chunking");
+  if (stopAfterChunking) return { ok: false, error: stopAfterChunking };
+
+  hooks.onPhase("link extraction");
+  const linkInputs = toLinkInputs(markdown, relPath, excludedRanges);
+  const stopAfterLinks = step("link extraction");
+  if (stopAfterLinks) return { ok: false, error: stopAfterLinks };
+
+  return {
+    ok: true,
+    value: {
+      artifact,
+      metadata,
+      nextStructure,
+      chunkInputs,
+      linkInputs,
+      tags,
+      memoryScopes,
+    },
+  };
+}

package/src/ingestion/prepare-worker.ts

@@ -0,0 +1,43 @@
+/**
+ * Worker entry for per-file preparation (fn-198). See `file-processor.ts`.
+ *
+ * @module src/ingestion/prepare-worker
+ */
+
+import type {
+  FileWorkerMessage,
+  FileWorkerRequest,
+  ReadyMessage,
+} from "./file-processor";
+
+import { defaultChunker } from "./chunker";
+import { convertForPreparation } from "./prepare-convert";
+import { PREPARE_PHASES, prepareFile } from "./prepare-file";
+
+declare const self: Worker;
+
+const post = (message: FileWorkerMessage | ReadyMessage): void => {
+  self.postMessage(message);
+};
+
+self.onmessage = async (event: MessageEvent<FileWorkerRequest>) => {
+  // Only the parent that created this worker may drive it; a Bun Worker sees
+  // its parent's messages with an empty origin.
+  if (event.origin !== "") return;
+  const { request, phaseSlot } = event.data;
+  const outcome = await prepareFile(request, {
+    convert: convertForPreparation,
+    chunker: defaultChunker,
+    onPrevious: (structure) => post({ type: "previous", structure }),
+    // Progress goes through shared memory, not messages: the parent only
+    // reads it when it stops a file.
+    onPhase: (phase) => {
+      Atomics.store(phaseSlot, 0, PREPARE_PHASES.indexOf(phase));
+    },
+    // The parent enforces the budget and terminates this worker.
+    check: () => null,
+  });
+  post({ type: "done", outcome });
+};
+
+post({ type: "ready" });

package/src/ingestion/record-container.ts

@@ -320,7 +320,10 @@ const loadPreviousStructure = async (
   return extractDocumentStructure(
     content,
     existing.relPath,
-    existing.dateFields
+    existing.dateFields,
+    {
+      markdownSource: false,
+    }
   );
 };
 
@@ -347,10 +350,12 @@ const persistRecord = async (
     contentType
   );
   const previousStructure = await loadPreviousStructure(input.store, existing);
+  // Records are not Markdown sources: they store and journal no links.
   const nextStructure = extractDocumentStructure(
     record.markdown,
     virtualPath,
-    dateFields
+    dateFields,
+    { markdownSource: false }
   );
   const structureDelta = diffDocumentStructure(
     previousStructure,

package/src/ingestion/strip.ts

@@ -11,12 +11,26 @@
  * Returns EXCLUDED RANGES on the original string - does NOT modify content.
  * This preserves position information for accurate line/column tracking.
  *
+ * Code regions come from the CommonMark + GFM parser. GFM table parsing is
+ * quadratic in the cells of one table (a 20,000-row table took minutes), so
+ * text with a table over the cell budget is parsed without the table
+ * extension: code spans, fences, indented code, blockquotes and lists keep
+ * their CommonMark meaning, and only table-cell boundaries are lost.
+ *
  * @module src/ingestion/strip
  */
 
 import { fromMarkdown } from "mdast-util-from-markdown";
 import { gfmFromMarkdown } from "mdast-util-gfm";
+import { gfmAutolinkLiteralFromMarkdown } from "mdast-util-gfm-autolink-literal";
+import { gfmFootnoteFromMarkdown } from "mdast-util-gfm-footnote";
+import { gfmStrikethroughFromMarkdown } from "mdast-util-gfm-strikethrough";
+import { gfmTaskListItemFromMarkdown } from "mdast-util-gfm-task-list-item";
 import { gfm } from "micromark-extension-gfm";
+import { gfmAutolinkLiteral } from "micromark-extension-gfm-autolink-literal";
+import { gfmFootnote } from "micromark-extension-gfm-footnote";
+import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough";
+import { gfmTaskListItem } from "micromark-extension-gfm-task-list-item";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -48,6 +62,14 @@ const FRONTMATTER_REGEX = /^---\r?\n[\s\S]*?(?:\r?\n)?---(?:\r?\n|$)/;
 /** HTML comments */
 const HTML_COMMENT_REGEX = /<!--[\s\S]*?-->/g;
 
+/**
+ * Most `|` characters one blank-line-delimited block may hold before the
+ * GFM table extension is left out of the parse. A GFM table only ends at a
+ * blank line (or another block), and its parse cost grows with the square
+ * of its cells: 5,000 cells take about 0.3 s, 26,000 about 8 s.
+ */
+export const MAX_PARSED_TABLE_CELLS = 5000;
+
 /** Backtick (code span or fence) or tilde fence anywhere in the text. */
 const INLINE_OR_FENCE_TRIGGER_REGEX = /`|~~~/;
 /** Blockquote markers and indentation before a line's content. */
@@ -87,6 +109,45 @@ const mayContainCode = (markdown: string): boolean => {
   return false;
 };
 
+/**
+ * Whether any blank-line-delimited block holds more table-cell pipes than
+ * the table budget. One linear pass, stopping as soon as the budget trips.
+ */
+const hasOversizedTable = (markdown: string): boolean => {
+  let blockPipes = 0;
+  let lineHasContent = false;
+  for (let index = 0; index < markdown.length; index += 1) {
+    const code = markdown.charCodeAt(index);
+    if (code === 10 /* \n */) {
+      if (!lineHasContent) blockPipes = 0;
+      lineHasContent = false;
+    } else if (code === 124 /* | */) {
+      blockPipes += 1;
+      lineHasContent = true;
+      if (blockPipes > MAX_PARSED_TABLE_CELLS) return true;
+    } else if (code !== 32 && code !== 9 && code !== 13) {
+      lineHasContent = true;
+    }
+  }
+  return false;
+};
+
+/** GFM without tables: every GFM construct except table rows and cells. */
+const GFM_WITHOUT_TABLES = {
+  extensions: [
+    gfmAutolinkLiteral(),
+    gfmFootnote(),
+    gfmStrikethrough(),
+    gfmTaskListItem(),
+  ],
+  mdastExtensions: [
+    gfmAutolinkLiteralFromMarkdown(),
+    gfmFootnoteFromMarkdown(),
+    gfmStrikethroughFromMarkdown(),
+    gfmTaskListItemFromMarkdown(),
+  ],
+};
+
 /** Opening fence at the start of a code node: backtick or tilde fence. */
 const FENCE_START_REGEX = /^[\t ]*(?:`{3,}|~{3,})/;
 
@@ -106,16 +167,17 @@ const collectCodeRanges = (
   markdown: string,
   frontmatterEnd: number
 ): ExcludedRange[] => {
-  if (!mayContainCode(markdown.slice(frontmatterEnd))) return [];
   const source =
     frontmatterEnd > 0
       ? markdown.slice(0, frontmatterEnd).replace(/[^\r\n]/gu, " ") +
         markdown.slice(frontmatterEnd)
       : markdown;
-  const root = fromMarkdown(source, {
-    extensions: [gfm()],
-    mdastExtensions: [gfmFromMarkdown()],
-  }) as CodeNode;
+  const root = fromMarkdown(
+    source,
+    hasOversizedTable(source)
+      ? GFM_WITHOUT_TABLES
+      : { extensions: [gfm()], mdastExtensions: [gfmFromMarkdown()] }
+  ) as CodeNode;
   const ranges: ExcludedRange[] = [];
   const visit = (node: CodeNode): void => {
     const start = node.position?.start.offset;
@@ -176,9 +238,10 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
   }
 
   // 3. Code spans and fenced or indented code blocks, from the parser.
-  ranges.push(
-    ...collectCodeRanges(markdown, frontmatterMatch?.[0].length ?? 0)
-  );
+  const frontmatterEnd = frontmatterMatch?.[0].length ?? 0;
+  if (mayContainCode(markdown.slice(frontmatterEnd))) {
+    ranges.push(...collectCodeRanges(markdown, frontmatterEnd));
+  }
 
   // Sort by start position for efficient lookup
   ranges.sort((a, b) => a.start - b.start);

package/src/ingestion/sync-options.ts

@@ -20,13 +20,19 @@ export function resolveContentTypeRules(
 
 export function withContentTypeRules(
   options: SyncOptions = {},
-  config?: Pick<Config, "contentTypes" | "chunking">
+  config?: Pick<Config, "contentTypes" | "chunking" | "conversion">
 ): SyncOptions {
   const rules = options.contentTypeRules ?? resolveContentTypeRules(config);
   const chunking = options.chunking ?? config?.chunking;
+  const conversion = config?.conversion;
+  // Explicit per-call limits win over the configured conversion budget.
+  const limits = conversion
+    ? { ...conversion, ...options.limits }
+    : options.limits;
   return {
     ...options,
     ...(chunking ? { chunking } : {}),
+    ...(limits ? { limits } : {}),
     contentTypeRules: rules,
     contentTypeRulesFingerprint:
       options.contentTypeRulesFingerprint ??