@gmickel/gno 2.8.2 → 2.8.4

package/src/core/link-inventory-markdown.ts

@@ -11,6 +11,7 @@ import { offsetToPosition } from "../ingestion/position";
 import { rangeIntersectsExcluded } from "../ingestion/strip";
 import {
   detectEncodingStyle,
+  isBackslashEscaped,
   parseParenthesizedDestination,
   splitDestinationPath,
   stripAngleBracketDestination,
@@ -298,12 +299,31 @@ export function inventoryInlineMarkdown(
   while (searchFrom < markdown.length) {
     const labelOpen = markdown.indexOf("[", searchFrom);
     if (labelOpen < 0) break;
-    if (labelOpen > 0 && markdown[labelOpen - 1] === "!") {
+    if (
+      (labelOpen > 0 && markdown[labelOpen - 1] === "!") ||
+      isBackslashEscaped(markdown, labelOpen)
+    ) {
+      searchFrom = labelOpen + 1;
+      continue;
+    }
+    // Match link extraction: escaped brackets are literal label text, and
+    // nested brackets must balance (`[see [1]](note.md)` is one link).
+    let labelClose = -1;
+    let depth = 0;
+    for (let i = labelOpen + 1; i < markdown.length; i += 1) {
+      const ch = markdown[i];
+      if ((ch !== "]" && ch !== "[") || isBackslashEscaped(markdown, i))
+        continue;
+      if (ch === "[") depth += 1;
+      else if (depth === 0) {
+        labelClose = i;
+        break;
+      } else depth -= 1;
+    }
+    if (labelClose < 0) {
       searchFrom = labelOpen + 1;
       continue;
     }
-    const labelClose = markdown.indexOf("]", labelOpen + 1);
-    if (labelClose < 0) break;
     if (markdown[labelClose + 1] !== "(") {
       searchFrom = labelOpen + 1;
       continue;

package/src/core/links.ts

@@ -14,6 +14,12 @@ import type { ExcludedRange } from "../ingestion/strip";
 
 import { buildLineOffsets, offsetToPosition } from "../ingestion/position";
 import { rangeIntersectsExcluded } from "../ingestion/strip";
+import {
+  isBackslashEscaped,
+  parseParenthesizedDestination,
+  stripAngleBracketDestination,
+  unescapeCommonMarkDestination,
+} from "./link-destination-parse";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -74,30 +80,77 @@ const UNSAFE_PERCENT_CODES = new Set(["%2F", "%5C", "%00", "%2f", "%5c"]);
 // Regex Patterns
 // ─────────────────────────────────────────────────────────────────────────────
 
+interface WikiLinkMatch {
+  index: number;
+  raw: string;
+  content: string;
+}
+
 /**
- * Wiki link: [[target]] or [[target|alias]] or [[target#anchor]] or [[collection:target]]
- * Captures: 1=content inside brackets
+ * Wiki links: [[target]], [[target|alias]], [[target#anchor]] or
+ * [[collection:target]]. Matches exactly what
+ * `/\[\[([^\]|]+(?:\|[^\]]+)?)\]\]/g` matches, in one pass: the content runs
+ * to the first `]`, which must be doubled; it must not start with `|`, and a
+ * `|` in it needs text after it. The regex rescanned to that `]` from every
+ * unclosed `[[`, so a note with thousands of them took minutes.
  */
-const WIKI_LINK_REGEX = /\[\[([^\]|]+(?:\|[^\]]+)?)\]\]/g;
+export function* findWikiLinks(
+  markdown: string
+): Generator<WikiLinkMatch, void, undefined> {
+  let nextClose = -2;
+  let nextPipe = -2;
+  let start = markdown.indexOf("[[");
+  while (start !== -1) {
+    const contentStart = start + 2;
+    if (nextClose < contentStart) {
+      nextClose = markdown.indexOf("]", contentStart);
+      if (nextClose === -1) return;
+    }
+    if (nextPipe !== -1 && nextPipe < contentStart) {
+      nextPipe = markdown.indexOf("|", contentStart);
+    }
+    const pipeInContent = nextPipe !== -1 && nextPipe < nextClose;
+    const matched =
+      markdown.charCodeAt(nextClose + 1) === 93 /* ] */ &&
+      nextClose > contentStart &&
+      nextPipe !== contentStart &&
+      (!pipeInContent || nextClose > nextPipe + 1);
+    if (matched) {
+      yield {
+        index: start,
+        raw: markdown.slice(start, nextClose + 2),
+        content: markdown.slice(contentStart, nextClose),
+      };
+      start = markdown.indexOf("[[", nextClose + 2);
+    } else {
+      start = markdown.indexOf("[[", start + 1);
+    }
+  }
+}
 
 /** Logseq embed: {{embed [[Page]]}} or {{embed ((block-id))}} */
 const LOGSEQ_EMBED_REGEX =
   /\{\{\s*embed\s+(\[\[[^\]]+\]\]|\(\([^)]+\)\))\s*\}\}/gi;
 
 /**
- * Markdown inline link: [text](url)
- * Captures: 1=text, 2=url (path and optional anchor)
+ * Markdown inline link opener: `[text](`, up to and including the `(`.
+ * Captures: 1=text. The destination after `(` is read by
+ * parseParenthesizedDestination, as CommonMark reads it.
  * Negative lookbehind to avoid image links ![]()
  * Link text may contain balanced square brackets one level deep
- * (`[see [1]](note.md)`), as CommonMark allows.
+ * (`[see [1]](note.md)`), as CommonMark allows; a backslash-escaped bracket
+ * (`\]`, `\[`) in the text is literal and never opens or closes it.
  *
  * SCOPE LIMITATIONS:
  * - Only matches simple inline links [text](url)
  * - Does NOT match reference-style links [text][ref] or [text]
  * - Does NOT match autolinks <url> or bare URLs
- * - Parens in URLs not supported (use %28 %29 encoding)
  */
-const MARKDOWN_LINK_REGEX = /(?<!!)\[((?:[^[\]]|\[[^[\]]*\])*)\]\(([^)]+)\)/g;
+const MARKDOWN_LINK_REGEX =
+  /(?<!!)\[((?:\\[\s\S]|[^[\]\\]|\[(?:\\[\s\S]|[^[\]\\])*\])*)\]\(/g;
+
+/** Logseq alias destination at the `(`: `([[Target]])`. */
+const LOGSEQ_ALIAS_DESTINATION_REGEX = /\((\[\[[^)]*\]\])\)/y;
 
 /** Square brackets in a destination mean the text was split, not a path. */
 const BRACKET_IN_DESTINATION_REGEX = /[[\]]/;
@@ -287,6 +340,40 @@ export function parseTargetParts(target: string): TargetParts {
 // Link Parsing
 // ─────────────────────────────────────────────────────────────────────────────
 
+/**
+ * Read an inline link destination at the `(` after `[text]`, the way
+ * CommonMark reads it: optional title, balanced parentheses, and no
+ * unescaped whitespace unless wrapped in `<...>`. Returns the unescaped
+ * destination (angle brackets removed) and the offset after the closing `)`,
+ * or null when the text is not a link. A Logseq alias `([[Target]])` is
+ * returned verbatim.
+ */
+function readInlineDestination(
+  markdown: string,
+  parenOffset: number
+): { url: string; endOffset: number; logseqAlias: boolean } | null {
+  LOGSEQ_ALIAS_DESTINATION_REGEX.lastIndex = parenOffset;
+  const logseq = LOGSEQ_ALIAS_DESTINATION_REGEX.exec(markdown);
+  if (logseq) {
+    return {
+      url: logseq[1] ?? "",
+      endOffset: parenOffset + logseq[0].length,
+      logseqAlias: true,
+    };
+  }
+  const parsed = parseParenthesizedDestination(markdown, parenOffset);
+  if (!parsed || parsed.destinationRaw.includes("\n")) {
+    return null;
+  }
+  return {
+    url: unescapeCommonMarkDestination(
+      stripAngleBracketDestination(parsed.destinationRaw).path
+    ),
+    endOffset: parsed.closeParenOffset + 1,
+    logseqAlias: false,
+  };
+}
+
 /**
  * Parse all links from markdown content.
  * Skips links inside excluded ranges (code blocks, frontmatter, etc.).
@@ -340,12 +427,9 @@ export function parseLinks(
   };
 
   // Parse wiki links
-  WIKI_LINK_REGEX.lastIndex = 0;
-  let match: RegExpExecArray | null;
-
-  while ((match = WIKI_LINK_REGEX.exec(markdown)) !== null) {
-    const startOffset = match.index;
-    const endOffset = startOffset + match[0].length;
+  for (const wiki of findWikiLinks(markdown)) {
+    const startOffset = wiki.index;
+    const endOffset = startOffset + wiki.raw.length;
 
     // Skip [[target]] nested in Logseq alias syntax: [Display]([[target]])
     if (
@@ -360,12 +444,10 @@ export function parseLinks(
       continue;
     }
 
-    const content = match[1];
-    if (!content) continue;
-
     // Parse [[target|alias]] (and table-escaped [[target\|alias]]) format
-    const { target: targetPart, alias: aliasText } =
-      splitWikiLinkContent(content);
+    const { target: targetPart, alias: aliasText } = splitWikiLinkContent(
+      wiki.content
+    );
     // Only set displayText if different from target
     const displayText =
       aliasText !== undefined && aliasText !== targetPart
@@ -376,11 +458,12 @@ export function parseLinks(
     if (!trimmedTarget) {
       continue;
     }
-    pushWikiLink(match[0], trimmedTarget, startOffset, endOffset, displayText);
+    pushWikiLink(wiki.raw, trimmedTarget, startOffset, endOffset, displayText);
   }
 
   // Parse Logseq embeds as links
   LOGSEQ_EMBED_REGEX.lastIndex = 0;
+  let match: RegExpExecArray | null;
 
   while ((match = LOGSEQ_EMBED_REGEX.exec(markdown)) !== null) {
     const startOffset = match.index;
@@ -409,32 +492,45 @@ export function parseLinks(
 
   while ((match = MARKDOWN_LINK_REGEX.exec(markdown)) !== null) {
     const startOffset = match.index;
-    const endOffset = startOffset + match[0].length;
+    const parenOffset = startOffset + match[0].length - 1;
+
+    // `\[` is literal text in CommonMark: it cannot open a link. A link
+    // may still start at a later `[` inside the text.
+    if (isBackslashEscaped(markdown, startOffset)) {
+      MARKDOWN_LINK_REGEX.lastIndex = startOffset + 1;
+      continue;
+    }
+
+    const destination = readInlineDestination(markdown, parenOffset);
+    // Not a link: a destination with an unescaped space outside `<...>`
+    // (`[x](my note.md)`) or no closing parenthesis. It is plain text; a link
+    // may still start at a later `[` inside the text.
+    if (!destination) {
+      MARKDOWN_LINK_REGEX.lastIndex = startOffset + 1;
+      continue;
+    }
+    const endOffset = destination.endOffset;
+    MARKDOWN_LINK_REGEX.lastIndex = endOffset;
 
     // Skip if inside excluded range
     if (rangeIntersectsExcluded(startOffset, endOffset, excludedRanges)) {
       continue;
     }
 
+    const raw = markdown.slice(startOffset, endOffset);
     const linkText = match[1] ?? "";
-    const url = match[2];
+    const { url } = destination;
     if (!url) continue;
 
     // Logseq alias syntax: [Display]([[Target]])
-    if (url.startsWith("[[") && url.endsWith("]]")) {
+    if (destination.logseqAlias) {
       const innerTarget = url.slice(2, -2).trim();
       if (innerTarget.length > 0) {
         const displayText =
           linkText && linkText !== innerTarget
             ? truncateText(linkText, MAX_DISPLAY_TEXT_GRAPHEMES)
             : undefined;
-        pushWikiLink(
-          match[0],
-          innerTarget,
-          startOffset,
-          endOffset,
-          displayText
-        );
+        pushWikiLink(raw, innerTarget, startOffset, endOffset, displayText);
       }
       continue;
     }
@@ -491,7 +587,7 @@ export function parseLinks(
 
     links.push({
       kind: "markdown",
-      raw: match[0],
+      raw,
       targetRef: parts.ref,
       targetAnchor: anchor ?? parts.anchor,
       targetCollection: parts.collection,

package/src/core/network-boundary-inventory.ts

@@ -305,6 +305,24 @@ export const NETWORK_BOUNDARY_INVENTORY = [
     action: null,
     enforcement: "local_process_only",
   },
+  {
+    // `ps` for the file processor child's resident memory (non-Linux Unix).
+    id: "file-processor-rss-probe",
+    key: "src/ingestion/file-processor.ts::child_process#1",
+    path: "src/ingestion/file-processor.ts",
+    primitive: "child_process",
+    action: null,
+    enforcement: "local_process_only",
+  },
+  {
+    // The same executable re-run as the file processor child (compiled builds).
+    id: "file-processor-child",
+    key: "src/ingestion/file-processor.ts::child_process#2",
+    path: "src/ingestion/file-processor.ts",
+    primitive: "child_process",
+    action: null,
+    enforcement: "local_process_only",
+  },
   {
     id: "http-mcp-tools",
     key: "logical::http-mcp-tools",

package/src/index.ts

@@ -7,6 +7,7 @@
  */
 
 import { runCli } from "./cli/run";
+import { FILE_PROCESSOR_CHILD_ENV } from "./ingestion/file-child-env";
 import { resetModelManager } from "./llm/nodeLlamaCpp/lifecycle";
 import { IMPORT_CHILD_ENV } from "./sessions/import-child-env";
 
@@ -78,6 +79,15 @@ if (process.env[IMPORT_CHILD_ENV] === "1") {
   await cleanupAndExit(0);
 }
 
+// A compiled executable re-run as the file processor child (see
+// src/ingestion/file-processor.ts) prepares files for its parent until the
+// parent disconnects.
+if (process.env[FILE_PROCESSOR_CHILD_ENV] === "1") {
+  const { runFileProcessorChild } = await import("./ingestion/file-child");
+  await runFileProcessorChild();
+  await cleanupAndExit(0);
+}
+
 // Await module completion so pending piped stdin keeps Windows Bun alive.
 await runCli(process.argv)
   .then((code) => cleanupAndExit(interruptExitCode || code))

package/src/ingestion/document-metadata.ts

@@ -0,0 +1,256 @@
+/**
+ * Document metadata and tag extraction from canonical Markdown.
+ *
+ * Pure functions shared by sync and the per-file worker (fn-198), kept out
+ * of sync.ts so the worker does not load the store.
+ *
+ * @module src/ingestion/document-metadata
+ */
+
+import type { NormalizedContentTypeRule } from "../config";
+import type { TypedMetadata } from "../core/typed-metadata";
+import type { ContentTypeSource } from "./types";
+
+import { resolveContentTypeRule } from "../config";
+import { normalizeTag, validateTag } from "../core/tags";
+import {
+  extractHashtags,
+  parseFrontmatter,
+  stripFrontmatter,
+} from "./frontmatter";
+import { extractTypedMetadata } from "./typed-metadata";
+
+/**
+ * Extract tags from markdown content.
+ * Combines frontmatter tags and inline hashtags, normalized and validated.
+ */
+export function extractTags(markdown: string): string[] {
+  const tags = new Set<string>();
+
+  // 1. Extract from frontmatter
+  const frontmatter = parseFrontmatter(markdown);
+  for (const tag of frontmatter.tags) {
+    const normalized = normalizeTag(tag);
+    if (validateTag(normalized)) {
+      tags.add(normalized);
+    }
+  }
+
+  // 2. Extract hashtags from body (after stripping frontmatter)
+  const body = stripFrontmatter(markdown);
+  const hashtags = extractHashtags(body);
+  for (const tag of hashtags) {
+    const normalized = normalizeTag(tag);
+    if (validateTag(normalized)) {
+      tags.add(normalized);
+    }
+  }
+
+  return [...tags];
+}
+
+export interface DocumentMetadata {
+  typedMetadata?: TypedMetadata;
+  metadataError?: string;
+  contentType?: string;
+  contentTypeSource: ContentTypeSource;
+  categories?: string[];
+  author?: string;
+  frontmatterDate?: string;
+  dateFields?: Record<string, string>;
+}
+
+const CODE_EXTENSIONS = new Set([
+  ".c",
+  ".cc",
+  ".cpp",
+  ".cs",
+  ".go",
+  ".java",
+  ".js",
+  ".jsx",
+  ".m",
+  ".mm",
+  ".php",
+  ".py",
+  ".rb",
+  ".rs",
+  ".swift",
+  ".ts",
+  ".tsx",
+]);
+
+const AUTHOR_KEYS = ["author", "by", "owner", "creator"] as const;
+const DATE_KEYS = [
+  "date",
+  "published",
+  "published_at",
+  "created",
+  "created_at",
+  "updated",
+  "updated_at",
+] as const;
+const DATE_FIELD_KEY_REGEX =
+  /(^|_)(date|time|created|updated|published|modified|deadline|expires|expiry|start|end)(_|$)/;
+
+function normalizeMetadataKey(rawKey: string): string {
+  return rawKey
+    .trim()
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .replace(/_+/g, "_");
+}
+
+function normalizeDate(value: unknown): string | undefined {
+  if (value instanceof Date) {
+    return Number.isNaN(value.getTime()) ? undefined : value.toISOString();
+  }
+  if (typeof value !== "string" && typeof value !== "number") {
+    return undefined;
+  }
+  const normalizedValue =
+    typeof value === "string"
+      ? value.trim().replace(/^["'](.*)["']$/, "$1")
+      : value;
+  const parsed = new Date(normalizedValue);
+  if (Number.isNaN(parsed.getTime())) {
+    return undefined;
+  }
+  return parsed.toISOString();
+}
+
+function inferPathContentType(
+  relPath: string,
+  ext: string
+): {
+  contentType: string;
+  source: ContentTypeSource;
+} {
+  const lowerPath = relPath.toLowerCase();
+  if (CODE_EXTENSIONS.has(ext.toLowerCase())) {
+    return { contentType: "code", source: "path-ext" };
+  }
+  if (/(meeting|standup|retro|minutes)/.test(lowerPath)) {
+    return { contentType: "meeting", source: "path-ext" };
+  }
+  if (/(spec|rfc|adr|design)/.test(lowerPath)) {
+    return { contentType: "spec", source: "path-ext" };
+  }
+  if (/(notes|journal|log)/.test(lowerPath)) {
+    return { contentType: "notes", source: "path-ext" };
+  }
+  return { contentType: "prose", source: "fallback" };
+}
+
+function normalizeFrontmatterScalar(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed.length < 2) {
+    return trimmed;
+  }
+  const first = trimmed[0];
+  const last = trimmed.at(-1);
+  if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+    return trimmed.slice(1, -1).trim();
+  }
+  return trimmed;
+}
+
+function parseCategories(input: unknown): string[] {
+  if (Array.isArray(input)) {
+    return input
+      .filter((v): v is string => typeof v === "string")
+      .map((v) => normalizeFrontmatterScalar(v).toLowerCase())
+      .filter((v) => v.length > 0);
+  }
+  if (typeof input === "string") {
+    return input
+      .split(",")
+      .map((v) => normalizeFrontmatterScalar(v).toLowerCase())
+      .filter((v) => v.length > 0);
+  }
+  return [];
+}
+
+export function extractDocumentMetadata(
+  markdown: string,
+  relPath: string,
+  ext: string,
+  contentTypeRules: NormalizedContentTypeRule[] = []
+): DocumentMetadata {
+  const parsed = parseFrontmatter(markdown);
+  const metadata = parsed.metadata;
+  const rawFrontmatterType =
+    typeof metadata.type === "string"
+      ? normalizeFrontmatterScalar(metadata.type)
+      : "";
+  const configuredRule = resolveContentTypeRule(
+    rawFrontmatterType,
+    relPath,
+    contentTypeRules
+  );
+  const inferred = inferPathContentType(relPath, ext);
+  const contentType = configuredRule?.rule.id ?? inferred.contentType;
+  const contentTypeSource: ContentTypeSource =
+    configuredRule?.source === "configured-id"
+      ? "frontmatter-type"
+      : configuredRule?.source === "prefix"
+        ? "prefix"
+        : inferred.source;
+  const categories = new Set<string>([contentType]);
+
+  const fmCategories = parseCategories(
+    metadata.category ?? metadata.categories ?? metadata.type
+  );
+  for (const category of fmCategories) {
+    categories.add(category);
+  }
+
+  let author: string | undefined;
+  for (const key of AUTHOR_KEYS) {
+    const value = metadata[key];
+    if (typeof value === "string" && value.trim().length > 0) {
+      author = value.trim();
+      break;
+    }
+  }
+
+  const normalizedMetadata = new Map<string, unknown>();
+  for (const [rawKey, value] of Object.entries(metadata)) {
+    const key = normalizeMetadataKey(rawKey);
+    if (key.length > 0 && !normalizedMetadata.has(key)) {
+      normalizedMetadata.set(key, value);
+    }
+  }
+
+  let frontmatterDate: string | undefined;
+  for (const key of DATE_KEYS) {
+    const normalized = normalizeDate(normalizedMetadata.get(key));
+    if (normalized) {
+      frontmatterDate = normalized;
+      break;
+    }
+  }
+
+  const dateFields: Record<string, string> = {};
+  for (const [key, value] of normalizedMetadata.entries()) {
+    if (!DATE_FIELD_KEY_REGEX.test(key)) {
+      continue;
+    }
+    const normalized = normalizeDate(value);
+    if (normalized) {
+      dateFields[key] = normalized;
+    }
+  }
+
+  return {
+    ...extractTypedMetadata(markdown),
+    contentType,
+    contentTypeSource,
+    categories: [...categories],
+    author,
+    frontmatterDate,
+    dateFields: Object.keys(dateFields).length > 0 ? dateFields : undefined,
+  };
+}

package/src/ingestion/file-child-env.ts

@@ -0,0 +1,9 @@
+/**
+ * Environment flag that makes a compiled GNO executable serve as the file
+ * processor child instead of the CLI (fn-198). Kept in its own module so the
+ * CLI entry can check it without loading the ingestion stack.
+ *
+ * @module src/ingestion/file-child-env
+ */
+
+export const FILE_PROCESSOR_CHILD_ENV = "GNO_INTERNAL_FILE_PROCESSOR_CHILD";