@gmickel/gno 2.8.1 → 2.8.3

package/src/core/link-destination-parse.ts

@@ -42,6 +42,19 @@ const COMMONMARK_ESCAPABLE = new Set([
   " ",
 ]);
 
+/**
+ * True when the character at `index` is backslash-escaped: preceded by an odd
+ * number of backslashes. An escaped `[` is literal text in CommonMark and
+ * cannot open a link.
+ */
+export function isBackslashEscaped(text: string, index: number): boolean {
+  let backslashes = 0;
+  for (let i = index - 1; i >= 0 && text[i] === "\\"; i -= 1) {
+    backslashes += 1;
+  }
+  return backslashes % 2 === 1;
+}
+
 export interface LinkEncodingStyle {
   spaces: "percent" | "backslash" | "raw";
   parens: "percent" | "backslash" | "raw";
@@ -220,10 +233,10 @@ export function parseParenthesizedDestination(
         i += 1;
         continue;
       }
-      if (
-        depth === 0 &&
-        (ch === " " || ch === "\t" || ch === "\n" || ch === "\r")
-      ) {
+      if (ch === " " || ch === "\t" || ch === "\n" || ch === "\r") {
+        // CommonMark: a bare destination has no unescaped whitespace, inside
+        // nested parentheses too (`(foo(bar baz).md)` is not a link).
+        if (depth > 0) return null;
         destEnd = i;
         break;
       }
@@ -262,6 +275,10 @@ export function parseParenthesizedDestination(
     close = j;
   }
 
+  // The destination and optional title must end at a real `)`:
+  // `[x](<note.md>` or `[x](<note.md>.` is not a link.
+  if (markdown[close] !== ")") return null;
+
   const destinationRaw = markdown.slice(destStart, destEnd);
   if (!destinationRaw) return null;
   return {

package/src/core/link-inventory-markdown.ts

@@ -11,6 +11,7 @@ import { offsetToPosition } from "../ingestion/position";
 import { rangeIntersectsExcluded } from "../ingestion/strip";
 import {
   detectEncodingStyle,
+  isBackslashEscaped,
   parseParenthesizedDestination,
   splitDestinationPath,
   stripAngleBracketDestination,
@@ -298,12 +299,31 @@ export function inventoryInlineMarkdown(
   while (searchFrom < markdown.length) {
     const labelOpen = markdown.indexOf("[", searchFrom);
     if (labelOpen < 0) break;
-    if (labelOpen > 0 && markdown[labelOpen - 1] === "!") {
+    if (
+      (labelOpen > 0 && markdown[labelOpen - 1] === "!") ||
+      isBackslashEscaped(markdown, labelOpen)
+    ) {
+      searchFrom = labelOpen + 1;
+      continue;
+    }
+    // Match link extraction: escaped brackets are literal label text, and
+    // nested brackets must balance (`[see [1]](note.md)` is one link).
+    let labelClose = -1;
+    let depth = 0;
+    for (let i = labelOpen + 1; i < markdown.length; i += 1) {
+      const ch = markdown[i];
+      if ((ch !== "]" && ch !== "[") || isBackslashEscaped(markdown, i))
+        continue;
+      if (ch === "[") depth += 1;
+      else if (depth === 0) {
+        labelClose = i;
+        break;
+      } else depth -= 1;
+    }
+    if (labelClose < 0) {
       searchFrom = labelOpen + 1;
       continue;
     }
-    const labelClose = markdown.indexOf("]", labelOpen + 1);
-    if (labelClose < 0) break;
     if (markdown[labelClose + 1] !== "(") {
       searchFrom = labelOpen + 1;
       continue;

package/src/core/link-inventory-opaque.ts

@@ -81,7 +81,9 @@ export function findExcludedKind(
 export function codeContextReason(
   kind: ExcludedRange["kind"] | null
 ): "code_fence_context" | "inline_code_context" | null {
-  if (kind === "fenced_code") return "code_fence_context";
+  if (kind === "fenced_code" || kind === "indented_code") {
+    return "code_fence_context";
+  }
   if (kind === "inline_code") return "inline_code_context";
   return null;
 }

package/src/core/links.ts

@@ -14,6 +14,12 @@ import type { ExcludedRange } from "../ingestion/strip";
 
 import { buildLineOffsets, offsetToPosition } from "../ingestion/position";
 import { rangeIntersectsExcluded } from "../ingestion/strip";
+import {
+  isBackslashEscaped,
+  parseParenthesizedDestination,
+  stripAngleBracketDestination,
+  unescapeCommonMarkDestination,
+} from "./link-destination-parse";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -85,19 +91,24 @@ const LOGSEQ_EMBED_REGEX =
   /\{\{\s*embed\s+(\[\[[^\]]+\]\]|\(\([^)]+\)\))\s*\}\}/gi;
 
 /**
- * Markdown inline link: [text](url)
- * Captures: 1=text, 2=url (path and optional anchor)
+ * Markdown inline link opener: `[text](`, up to and including the `(`.
+ * Captures: 1=text. The destination after `(` is read by
+ * parseParenthesizedDestination, as CommonMark reads it.
  * Negative lookbehind to avoid image links ![]()
  * Link text may contain balanced square brackets one level deep
- * (`[see [1]](note.md)`), as CommonMark allows.
+ * (`[see [1]](note.md)`), as CommonMark allows; a backslash-escaped bracket
+ * (`\]`, `\[`) in the text is literal and never opens or closes it.
  *
  * SCOPE LIMITATIONS:
  * - Only matches simple inline links [text](url)
  * - Does NOT match reference-style links [text][ref] or [text]
  * - Does NOT match autolinks <url> or bare URLs
- * - Parens in URLs not supported (use %28 %29 encoding)
  */
-const MARKDOWN_LINK_REGEX = /(?<!!)\[((?:[^[\]]|\[[^[\]]*\])*)\]\(([^)]+)\)/g;
+const MARKDOWN_LINK_REGEX =
+  /(?<!!)\[((?:\\[\s\S]|[^[\]\\]|\[(?:\\[\s\S]|[^[\]\\])*\])*)\]\(/g;
+
+/** Logseq alias destination at the `(`: `([[Target]])`. */
+const LOGSEQ_ALIAS_DESTINATION_REGEX = /\((\[\[[^)]*\]\])\)/y;
 
 /** Square brackets in a destination mean the text was split, not a path. */
 const BRACKET_IN_DESTINATION_REGEX = /[[\]]/;
@@ -287,6 +298,40 @@ export function parseTargetParts(target: string): TargetParts {
 // Link Parsing
 // ─────────────────────────────────────────────────────────────────────────────
 
+/**
+ * Read an inline link destination at the `(` after `[text]`, the way
+ * CommonMark reads it: optional title, balanced parentheses, and no
+ * unescaped whitespace unless wrapped in `<...>`. Returns the unescaped
+ * destination (angle brackets removed) and the offset after the closing `)`,
+ * or null when the text is not a link. A Logseq alias `([[Target]])` is
+ * returned verbatim.
+ */
+function readInlineDestination(
+  markdown: string,
+  parenOffset: number
+): { url: string; endOffset: number; logseqAlias: boolean } | null {
+  LOGSEQ_ALIAS_DESTINATION_REGEX.lastIndex = parenOffset;
+  const logseq = LOGSEQ_ALIAS_DESTINATION_REGEX.exec(markdown);
+  if (logseq) {
+    return {
+      url: logseq[1] ?? "",
+      endOffset: parenOffset + logseq[0].length,
+      logseqAlias: true,
+    };
+  }
+  const parsed = parseParenthesizedDestination(markdown, parenOffset);
+  if (!parsed || parsed.destinationRaw.includes("\n")) {
+    return null;
+  }
+  return {
+    url: unescapeCommonMarkDestination(
+      stripAngleBracketDestination(parsed.destinationRaw).path
+    ),
+    endOffset: parsed.closeParenOffset + 1,
+    logseqAlias: false,
+  };
+}
+
 /**
  * Parse all links from markdown content.
  * Skips links inside excluded ranges (code blocks, frontmatter, etc.).
@@ -409,32 +454,45 @@ export function parseLinks(
 
   while ((match = MARKDOWN_LINK_REGEX.exec(markdown)) !== null) {
     const startOffset = match.index;
-    const endOffset = startOffset + match[0].length;
+    const parenOffset = startOffset + match[0].length - 1;
+
+    // `\[` is literal text in CommonMark: it cannot open a link. A link
+    // may still start at a later `[` inside the text.
+    if (isBackslashEscaped(markdown, startOffset)) {
+      MARKDOWN_LINK_REGEX.lastIndex = startOffset + 1;
+      continue;
+    }
+
+    const destination = readInlineDestination(markdown, parenOffset);
+    // Not a link: a destination with an unescaped space outside `<...>`
+    // (`[x](my note.md)`) or no closing parenthesis. It is plain text; a link
+    // may still start at a later `[` inside the text.
+    if (!destination) {
+      MARKDOWN_LINK_REGEX.lastIndex = startOffset + 1;
+      continue;
+    }
+    const endOffset = destination.endOffset;
+    MARKDOWN_LINK_REGEX.lastIndex = endOffset;
 
     // Skip if inside excluded range
     if (rangeIntersectsExcluded(startOffset, endOffset, excludedRanges)) {
       continue;
     }
 
+    const raw = markdown.slice(startOffset, endOffset);
     const linkText = match[1] ?? "";
-    const url = match[2];
+    const { url } = destination;
     if (!url) continue;
 
     // Logseq alias syntax: [Display]([[Target]])
-    if (url.startsWith("[[") && url.endsWith("]]")) {
+    if (destination.logseqAlias) {
       const innerTarget = url.slice(2, -2).trim();
       if (innerTarget.length > 0) {
         const displayText =
           linkText && linkText !== innerTarget
             ? truncateText(linkText, MAX_DISPLAY_TEXT_GRAPHEMES)
             : undefined;
-        pushWikiLink(
-          match[0],
-          innerTarget,
-          startOffset,
-          endOffset,
-          displayText
-        );
+        pushWikiLink(raw, innerTarget, startOffset, endOffset, displayText);
       }
       continue;
     }
@@ -491,7 +549,7 @@ export function parseLinks(
 
     links.push({
       kind: "markdown",
-      raw: match[0],
+      raw,
       targetRef: parts.ref,
       targetAnchor: anchor ?? parts.anchor,
       targetCollection: parts.collection,

package/src/core/network-boundary-inventory.ts

@@ -305,6 +305,24 @@ export const NETWORK_BOUNDARY_INVENTORY = [
     action: null,
     enforcement: "local_process_only",
   },
+  {
+    // `ps` for the file processor child's resident memory (non-Linux Unix).
+    id: "file-processor-rss-probe",
+    key: "src/ingestion/file-processor.ts::child_process#1",
+    path: "src/ingestion/file-processor.ts",
+    primitive: "child_process",
+    action: null,
+    enforcement: "local_process_only",
+  },
+  {
+    // The same executable re-run as the file processor child (compiled builds).
+    id: "file-processor-child",
+    key: "src/ingestion/file-processor.ts::child_process#2",
+    path: "src/ingestion/file-processor.ts",
+    primitive: "child_process",
+    action: null,
+    enforcement: "local_process_only",
+  },
   {
     id: "http-mcp-tools",
     key: "logical::http-mcp-tools",

package/src/index.ts

@@ -7,6 +7,7 @@
  */
 
 import { runCli } from "./cli/run";
+import { FILE_PROCESSOR_CHILD_ENV } from "./ingestion/file-child-env";
 import { resetModelManager } from "./llm/nodeLlamaCpp/lifecycle";
 import { IMPORT_CHILD_ENV } from "./sessions/import-child-env";
 
@@ -78,6 +79,15 @@ if (process.env[IMPORT_CHILD_ENV] === "1") {
   await cleanupAndExit(0);
 }
 
+// A compiled executable re-run as the file processor child (see
+// src/ingestion/file-processor.ts) prepares files for its parent until the
+// parent disconnects.
+if (process.env[FILE_PROCESSOR_CHILD_ENV] === "1") {
+  const { runFileProcessorChild } = await import("./ingestion/file-child");
+  await runFileProcessorChild();
+  await cleanupAndExit(0);
+}
+
 // Await module completion so pending piped stdin keeps Windows Bun alive.
 await runCli(process.argv)
   .then((code) => cleanupAndExit(interruptExitCode || code))

package/src/ingestion/document-metadata.ts

@@ -0,0 +1,256 @@
+/**
+ * Document metadata and tag extraction from canonical Markdown.
+ *
+ * Pure functions shared by sync and the per-file worker (fn-198), kept out
+ * of sync.ts so the worker does not load the store.
+ *
+ * @module src/ingestion/document-metadata
+ */
+
+import type { NormalizedContentTypeRule } from "../config";
+import type { TypedMetadata } from "../core/typed-metadata";
+import type { ContentTypeSource } from "./types";
+
+import { resolveContentTypeRule } from "../config";
+import { normalizeTag, validateTag } from "../core/tags";
+import {
+  extractHashtags,
+  parseFrontmatter,
+  stripFrontmatter,
+} from "./frontmatter";
+import { extractTypedMetadata } from "./typed-metadata";
+
+/**
+ * Extract tags from markdown content.
+ * Combines frontmatter tags and inline hashtags, normalized and validated.
+ */
+export function extractTags(markdown: string): string[] {
+  const tags = new Set<string>();
+
+  // 1. Extract from frontmatter
+  const frontmatter = parseFrontmatter(markdown);
+  for (const tag of frontmatter.tags) {
+    const normalized = normalizeTag(tag);
+    if (validateTag(normalized)) {
+      tags.add(normalized);
+    }
+  }
+
+  // 2. Extract hashtags from body (after stripping frontmatter)
+  const body = stripFrontmatter(markdown);
+  const hashtags = extractHashtags(body);
+  for (const tag of hashtags) {
+    const normalized = normalizeTag(tag);
+    if (validateTag(normalized)) {
+      tags.add(normalized);
+    }
+  }
+
+  return [...tags];
+}
+
+export interface DocumentMetadata {
+  typedMetadata?: TypedMetadata;
+  metadataError?: string;
+  contentType?: string;
+  contentTypeSource: ContentTypeSource;
+  categories?: string[];
+  author?: string;
+  frontmatterDate?: string;
+  dateFields?: Record<string, string>;
+}
+
+const CODE_EXTENSIONS = new Set([
+  ".c",
+  ".cc",
+  ".cpp",
+  ".cs",
+  ".go",
+  ".java",
+  ".js",
+  ".jsx",
+  ".m",
+  ".mm",
+  ".php",
+  ".py",
+  ".rb",
+  ".rs",
+  ".swift",
+  ".ts",
+  ".tsx",
+]);
+
+const AUTHOR_KEYS = ["author", "by", "owner", "creator"] as const;
+const DATE_KEYS = [
+  "date",
+  "published",
+  "published_at",
+  "created",
+  "created_at",
+  "updated",
+  "updated_at",
+] as const;
+const DATE_FIELD_KEY_REGEX =
+  /(^|_)(date|time|created|updated|published|modified|deadline|expires|expiry|start|end)(_|$)/;
+
+function normalizeMetadataKey(rawKey: string): string {
+  return rawKey
+    .trim()
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .replace(/_+/g, "_");
+}
+
+function normalizeDate(value: unknown): string | undefined {
+  if (value instanceof Date) {
+    return Number.isNaN(value.getTime()) ? undefined : value.toISOString();
+  }
+  if (typeof value !== "string" && typeof value !== "number") {
+    return undefined;
+  }
+  const normalizedValue =
+    typeof value === "string"
+      ? value.trim().replace(/^["'](.*)["']$/, "$1")
+      : value;
+  const parsed = new Date(normalizedValue);
+  if (Number.isNaN(parsed.getTime())) {
+    return undefined;
+  }
+  return parsed.toISOString();
+}
+
+function inferPathContentType(
+  relPath: string,
+  ext: string
+): {
+  contentType: string;
+  source: ContentTypeSource;
+} {
+  const lowerPath = relPath.toLowerCase();
+  if (CODE_EXTENSIONS.has(ext.toLowerCase())) {
+    return { contentType: "code", source: "path-ext" };
+  }
+  if (/(meeting|standup|retro|minutes)/.test(lowerPath)) {
+    return { contentType: "meeting", source: "path-ext" };
+  }
+  if (/(spec|rfc|adr|design)/.test(lowerPath)) {
+    return { contentType: "spec", source: "path-ext" };
+  }
+  if (/(notes|journal|log)/.test(lowerPath)) {
+    return { contentType: "notes", source: "path-ext" };
+  }
+  return { contentType: "prose", source: "fallback" };
+}
+
+function normalizeFrontmatterScalar(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed.length < 2) {
+    return trimmed;
+  }
+  const first = trimmed[0];
+  const last = trimmed.at(-1);
+  if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+    return trimmed.slice(1, -1).trim();
+  }
+  return trimmed;
+}
+
+function parseCategories(input: unknown): string[] {
+  if (Array.isArray(input)) {
+    return input
+      .filter((v): v is string => typeof v === "string")
+      .map((v) => normalizeFrontmatterScalar(v).toLowerCase())
+      .filter((v) => v.length > 0);
+  }
+  if (typeof input === "string") {
+    return input
+      .split(",")
+      .map((v) => normalizeFrontmatterScalar(v).toLowerCase())
+      .filter((v) => v.length > 0);
+  }
+  return [];
+}
+
+export function extractDocumentMetadata(
+  markdown: string,
+  relPath: string,
+  ext: string,
+  contentTypeRules: NormalizedContentTypeRule[] = []
+): DocumentMetadata {
+  const parsed = parseFrontmatter(markdown);
+  const metadata = parsed.metadata;
+  const rawFrontmatterType =
+    typeof metadata.type === "string"
+      ? normalizeFrontmatterScalar(metadata.type)
+      : "";
+  const configuredRule = resolveContentTypeRule(
+    rawFrontmatterType,
+    relPath,
+    contentTypeRules
+  );
+  const inferred = inferPathContentType(relPath, ext);
+  const contentType = configuredRule?.rule.id ?? inferred.contentType;
+  const contentTypeSource: ContentTypeSource =
+    configuredRule?.source === "configured-id"
+      ? "frontmatter-type"
+      : configuredRule?.source === "prefix"
+        ? "prefix"
+        : inferred.source;
+  const categories = new Set<string>([contentType]);
+
+  const fmCategories = parseCategories(
+    metadata.category ?? metadata.categories ?? metadata.type
+  );
+  for (const category of fmCategories) {
+    categories.add(category);
+  }
+
+  let author: string | undefined;
+  for (const key of AUTHOR_KEYS) {
+    const value = metadata[key];
+    if (typeof value === "string" && value.trim().length > 0) {
+      author = value.trim();
+      break;
+    }
+  }
+
+  const normalizedMetadata = new Map<string, unknown>();
+  for (const [rawKey, value] of Object.entries(metadata)) {
+    const key = normalizeMetadataKey(rawKey);
+    if (key.length > 0 && !normalizedMetadata.has(key)) {
+      normalizedMetadata.set(key, value);
+    }
+  }
+
+  let frontmatterDate: string | undefined;
+  for (const key of DATE_KEYS) {
+    const normalized = normalizeDate(normalizedMetadata.get(key));
+    if (normalized) {
+      frontmatterDate = normalized;
+      break;
+    }
+  }
+
+  const dateFields: Record<string, string> = {};
+  for (const [key, value] of normalizedMetadata.entries()) {
+    if (!DATE_FIELD_KEY_REGEX.test(key)) {
+      continue;
+    }
+    const normalized = normalizeDate(value);
+    if (normalized) {
+      dateFields[key] = normalized;
+    }
+  }
+
+  return {
+    ...extractTypedMetadata(markdown),
+    contentType,
+    contentTypeSource,
+    categories: [...categories],
+    author,
+    frontmatterDate,
+    dateFields: Object.keys(dateFields).length > 0 ? dateFields : undefined,
+  };
+}

package/src/ingestion/file-child-env.ts

@@ -0,0 +1,9 @@
+/**
+ * Environment flag that makes a compiled GNO executable serve as the file
+ * processor child instead of the CLI (fn-198). Kept in its own module so the
+ * CLI entry can check it without loading the ingestion stack.
+ *
+ * @module src/ingestion/file-child-env
+ */
+
+export const FILE_PROCESSOR_CHILD_ENV = "GNO_INTERNAL_FILE_PROCESSOR_CHILD";

package/src/ingestion/file-child.ts

@@ -0,0 +1,106 @@
+/**
+ * Child-process backend of the file processor (fn-198).
+ *
+ * Where a TypeScript worker cannot start (a standalone compiled executable),
+ * sync prepares each file in a child process instead: the same executable
+ * re-run with `FILE_PROCESSOR_CHILD_ENV` set (see src/index.ts), or this
+ * module under a source runtime. It serves `prepareFile` requests over IPC,
+ * one at a time, until its parent disconnects; the parent kills it when a
+ * file overruns its budget, exactly as it terminates the worker, and on
+ * every shutdown path it controls.
+ *
+ * A child busy in a synchronous step cannot notice its parent dying. On
+ * Linux it asks the kernel to SIGKILL it when the parent dies
+ * (PR_SET_PDEATHSIG, through Bun's built-in FFI), which holds even if the
+ * parent is SIGKILLed. Elsewhere it checks between steps that its parent is
+ * still there and exits if not; a parent killed with SIGKILL on macOS or
+ * Windows can leave the child running until its current step ends.
+ *
+ * @module src/ingestion/file-child
+ */
+
+import type {
+  FileChildMessage,
+  FileWorkerRequest,
+  ReadyMessage,
+} from "./file-processor";
+
+import { defaultChunker } from "./chunker";
+import { FILE_PROCESSOR_CHILD_ENV } from "./file-child-env";
+import { convertForPreparation } from "./prepare-convert";
+import { prepareFile } from "./prepare-file";
+
+const send = (message: FileChildMessage | ReadyMessage): void => {
+  process.send?.(message);
+};
+
+const PR_SET_PDEATHSIG = 1;
+const SIGKILL = 9;
+const LINUX_LIBC_NAMES = ["libc.so.6", "libc.so"];
+
+/** Linux: have the kernel SIGKILL this process when its parent dies. */
+async function killWithParent(): Promise<void> {
+  if (process.platform !== "linux") return;
+  try {
+    // bun:ffi is built into Bun (no native dependency); loaded only here.
+    const { dlopen, FFIType } = await import("bun:ffi");
+    for (const name of LINUX_LIBC_NAMES) {
+      try {
+        const libc = dlopen(name, {
+          prctl: {
+            args: [
+              FFIType.i32,
+              FFIType.u64,
+              FFIType.u64,
+              FFIType.u64,
+              FFIType.u64,
+            ],
+            returns: FFIType.i32,
+          },
+        });
+        libc.symbols.prctl(PR_SET_PDEATHSIG, BigInt(SIGKILL), 0n, 0n, 0n);
+        libc.close();
+        return;
+      } catch {
+        // Try the next libc name; the between-step check still applies.
+      }
+    }
+  } catch {
+    // No FFI in this runtime: the between-step check still applies.
+  }
+}
+
+/** Serve prepare requests from the parent until it disconnects. */
+export async function runFileProcessorChild(): Promise<void> {
+  // Nothing this child starts may re-enter child mode.
+  delete process.env[FILE_PROCESSOR_CHILD_ENV];
+  const parentPid = process.ppid;
+  await killWithParent();
+  // The parent may have died before the death signal was armed.
+  const parentGone = (): boolean => process.ppid !== parentPid;
+  if (parentGone()) process.exit(1);
+  return new Promise((resolve) => {
+    process.on("message", async (job: Pick<FileWorkerRequest, "request">) => {
+      const outcome = await prepareFile(job.request, {
+        convert: convertForPreparation,
+        chunker: defaultChunker,
+        onPrevious: (structure) => send({ type: "previous", structure }),
+        onPhase: (phase) => send({ type: "phase", phase }),
+        // The parent enforces the budget and kills this process; between
+        // steps the child only checks that its parent is still alive.
+        check: () => {
+          if (parentGone()) process.exit(1);
+          return null;
+        },
+      });
+      send({ type: "done", outcome });
+    });
+    process.on("disconnect", () => resolve());
+    send({ type: "ready" });
+  });
+}
+
+if (import.meta.main) {
+  await runFileProcessorChild();
+  process.exit(0);
+}