@gmickel/gno 2.8.2 → 2.8.3

package/src/ingestion/document-metadata.ts

@@ -0,0 +1,256 @@
+/**
+ * Document metadata and tag extraction from canonical Markdown.
+ *
+ * Pure functions shared by sync and the per-file worker (fn-198), kept out
+ * of sync.ts so the worker does not load the store.
+ *
+ * @module src/ingestion/document-metadata
+ */
+
+import type { NormalizedContentTypeRule } from "../config";
+import type { TypedMetadata } from "../core/typed-metadata";
+import type { ContentTypeSource } from "./types";
+
+import { resolveContentTypeRule } from "../config";
+import { normalizeTag, validateTag } from "../core/tags";
+import {
+  extractHashtags,
+  parseFrontmatter,
+  stripFrontmatter,
+} from "./frontmatter";
+import { extractTypedMetadata } from "./typed-metadata";
+
+/**
+ * Extract tags from markdown content.
+ * Combines frontmatter tags and inline hashtags, normalized and validated.
+ */
+export function extractTags(markdown: string): string[] {
+  const tags = new Set<string>();
+
+  // 1. Extract from frontmatter
+  const frontmatter = parseFrontmatter(markdown);
+  for (const tag of frontmatter.tags) {
+    const normalized = normalizeTag(tag);
+    if (validateTag(normalized)) {
+      tags.add(normalized);
+    }
+  }
+
+  // 2. Extract hashtags from body (after stripping frontmatter)
+  const body = stripFrontmatter(markdown);
+  const hashtags = extractHashtags(body);
+  for (const tag of hashtags) {
+    const normalized = normalizeTag(tag);
+    if (validateTag(normalized)) {
+      tags.add(normalized);
+    }
+  }
+
+  return [...tags];
+}
+
+export interface DocumentMetadata {
+  typedMetadata?: TypedMetadata;
+  metadataError?: string;
+  contentType?: string;
+  contentTypeSource: ContentTypeSource;
+  categories?: string[];
+  author?: string;
+  frontmatterDate?: string;
+  dateFields?: Record<string, string>;
+}
+
+const CODE_EXTENSIONS = new Set([
+  ".c",
+  ".cc",
+  ".cpp",
+  ".cs",
+  ".go",
+  ".java",
+  ".js",
+  ".jsx",
+  ".m",
+  ".mm",
+  ".php",
+  ".py",
+  ".rb",
+  ".rs",
+  ".swift",
+  ".ts",
+  ".tsx",
+]);
+
+const AUTHOR_KEYS = ["author", "by", "owner", "creator"] as const;
+const DATE_KEYS = [
+  "date",
+  "published",
+  "published_at",
+  "created",
+  "created_at",
+  "updated",
+  "updated_at",
+] as const;
+const DATE_FIELD_KEY_REGEX =
+  /(^|_)(date|time|created|updated|published|modified|deadline|expires|expiry|start|end)(_|$)/;
+
+function normalizeMetadataKey(rawKey: string): string {
+  return rawKey
+    .trim()
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .replace(/_+/g, "_");
+}
+
+function normalizeDate(value: unknown): string | undefined {
+  if (value instanceof Date) {
+    return Number.isNaN(value.getTime()) ? undefined : value.toISOString();
+  }
+  if (typeof value !== "string" && typeof value !== "number") {
+    return undefined;
+  }
+  const normalizedValue =
+    typeof value === "string"
+      ? value.trim().replace(/^["'](.*)["']$/, "$1")
+      : value;
+  const parsed = new Date(normalizedValue);
+  if (Number.isNaN(parsed.getTime())) {
+    return undefined;
+  }
+  return parsed.toISOString();
+}
+
+function inferPathContentType(
+  relPath: string,
+  ext: string
+): {
+  contentType: string;
+  source: ContentTypeSource;
+} {
+  const lowerPath = relPath.toLowerCase();
+  if (CODE_EXTENSIONS.has(ext.toLowerCase())) {
+    return { contentType: "code", source: "path-ext" };
+  }
+  if (/(meeting|standup|retro|minutes)/.test(lowerPath)) {
+    return { contentType: "meeting", source: "path-ext" };
+  }
+  if (/(spec|rfc|adr|design)/.test(lowerPath)) {
+    return { contentType: "spec", source: "path-ext" };
+  }
+  if (/(notes|journal|log)/.test(lowerPath)) {
+    return { contentType: "notes", source: "path-ext" };
+  }
+  return { contentType: "prose", source: "fallback" };
+}
+
+function normalizeFrontmatterScalar(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed.length < 2) {
+    return trimmed;
+  }
+  const first = trimmed[0];
+  const last = trimmed.at(-1);
+  if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+    return trimmed.slice(1, -1).trim();
+  }
+  return trimmed;
+}
+
+function parseCategories(input: unknown): string[] {
+  if (Array.isArray(input)) {
+    return input
+      .filter((v): v is string => typeof v === "string")
+      .map((v) => normalizeFrontmatterScalar(v).toLowerCase())
+      .filter((v) => v.length > 0);
+  }
+  if (typeof input === "string") {
+    return input
+      .split(",")
+      .map((v) => normalizeFrontmatterScalar(v).toLowerCase())
+      .filter((v) => v.length > 0);
+  }
+  return [];
+}
+
+export function extractDocumentMetadata(
+  markdown: string,
+  relPath: string,
+  ext: string,
+  contentTypeRules: NormalizedContentTypeRule[] = []
+): DocumentMetadata {
+  const parsed = parseFrontmatter(markdown);
+  const metadata = parsed.metadata;
+  const rawFrontmatterType =
+    typeof metadata.type === "string"
+      ? normalizeFrontmatterScalar(metadata.type)
+      : "";
+  const configuredRule = resolveContentTypeRule(
+    rawFrontmatterType,
+    relPath,
+    contentTypeRules
+  );
+  const inferred = inferPathContentType(relPath, ext);
+  const contentType = configuredRule?.rule.id ?? inferred.contentType;
+  const contentTypeSource: ContentTypeSource =
+    configuredRule?.source === "configured-id"
+      ? "frontmatter-type"
+      : configuredRule?.source === "prefix"
+        ? "prefix"
+        : inferred.source;
+  const categories = new Set<string>([contentType]);
+
+  const fmCategories = parseCategories(
+    metadata.category ?? metadata.categories ?? metadata.type
+  );
+  for (const category of fmCategories) {
+    categories.add(category);
+  }
+
+  let author: string | undefined;
+  for (const key of AUTHOR_KEYS) {
+    const value = metadata[key];
+    if (typeof value === "string" && value.trim().length > 0) {
+      author = value.trim();
+      break;
+    }
+  }
+
+  const normalizedMetadata = new Map<string, unknown>();
+  for (const [rawKey, value] of Object.entries(metadata)) {
+    const key = normalizeMetadataKey(rawKey);
+    if (key.length > 0 && !normalizedMetadata.has(key)) {
+      normalizedMetadata.set(key, value);
+    }
+  }
+
+  let frontmatterDate: string | undefined;
+  for (const key of DATE_KEYS) {
+    const normalized = normalizeDate(normalizedMetadata.get(key));
+    if (normalized) {
+      frontmatterDate = normalized;
+      break;
+    }
+  }
+
+  const dateFields: Record<string, string> = {};
+  for (const [key, value] of normalizedMetadata.entries()) {
+    if (!DATE_FIELD_KEY_REGEX.test(key)) {
+      continue;
+    }
+    const normalized = normalizeDate(value);
+    if (normalized) {
+      dateFields[key] = normalized;
+    }
+  }
+
+  return {
+    ...extractTypedMetadata(markdown),
+    contentType,
+    contentTypeSource,
+    categories: [...categories],
+    author,
+    frontmatterDate,
+    dateFields: Object.keys(dateFields).length > 0 ? dateFields : undefined,
+  };
+}

package/src/ingestion/file-child-env.ts

@@ -0,0 +1,9 @@
+/**
+ * Environment flag that makes a compiled GNO executable serve as the file
+ * processor child instead of the CLI (fn-198). Kept in its own module so the
+ * CLI entry can check it without loading the ingestion stack.
+ *
+ * @module src/ingestion/file-child-env
+ */
+
+export const FILE_PROCESSOR_CHILD_ENV = "GNO_INTERNAL_FILE_PROCESSOR_CHILD";

package/src/ingestion/file-child.ts

@@ -0,0 +1,106 @@
+/**
+ * Child-process backend of the file processor (fn-198).
+ *
+ * Where a TypeScript worker cannot start (a standalone compiled executable),
+ * sync prepares each file in a child process instead: the same executable
+ * re-run with `FILE_PROCESSOR_CHILD_ENV` set (see src/index.ts), or this
+ * module under a source runtime. It serves `prepareFile` requests over IPC,
+ * one at a time, until its parent disconnects; the parent kills it when a
+ * file overruns its budget, exactly as it terminates the worker, and on
+ * every shutdown path it controls.
+ *
+ * A child busy in a synchronous step cannot notice its parent dying. On
+ * Linux it asks the kernel to SIGKILL it when the parent dies
+ * (PR_SET_PDEATHSIG, through Bun's built-in FFI), which holds even if the
+ * parent is SIGKILLed. Elsewhere it checks between steps that its parent is
+ * still there and exits if not; a parent killed with SIGKILL on macOS or
+ * Windows can leave the child running until its current step ends.
+ *
+ * @module src/ingestion/file-child
+ */
+
+import type {
+  FileChildMessage,
+  FileWorkerRequest,
+  ReadyMessage,
+} from "./file-processor";
+
+import { defaultChunker } from "./chunker";
+import { FILE_PROCESSOR_CHILD_ENV } from "./file-child-env";
+import { convertForPreparation } from "./prepare-convert";
+import { prepareFile } from "./prepare-file";
+
+const send = (message: FileChildMessage | ReadyMessage): void => {
+  process.send?.(message);
+};
+
+const PR_SET_PDEATHSIG = 1;
+const SIGKILL = 9;
+const LINUX_LIBC_NAMES = ["libc.so.6", "libc.so"];
+
+/** Linux: have the kernel SIGKILL this process when its parent dies. */
+async function killWithParent(): Promise<void> {
+  if (process.platform !== "linux") return;
+  try {
+    // bun:ffi is built into Bun (no native dependency); loaded only here.
+    const { dlopen, FFIType } = await import("bun:ffi");
+    for (const name of LINUX_LIBC_NAMES) {
+      try {
+        const libc = dlopen(name, {
+          prctl: {
+            args: [
+              FFIType.i32,
+              FFIType.u64,
+              FFIType.u64,
+              FFIType.u64,
+              FFIType.u64,
+            ],
+            returns: FFIType.i32,
+          },
+        });
+        libc.symbols.prctl(PR_SET_PDEATHSIG, BigInt(SIGKILL), 0n, 0n, 0n);
+        libc.close();
+        return;
+      } catch {
+        // Try the next libc name; the between-step check still applies.
+      }
+    }
+  } catch {
+    // No FFI in this runtime: the between-step check still applies.
+  }
+}
+
+/** Serve prepare requests from the parent until it disconnects. */
+export async function runFileProcessorChild(): Promise<void> {
+  // Nothing this child starts may re-enter child mode.
+  delete process.env[FILE_PROCESSOR_CHILD_ENV];
+  const parentPid = process.ppid;
+  await killWithParent();
+  // The parent may have died before the death signal was armed.
+  const parentGone = (): boolean => process.ppid !== parentPid;
+  if (parentGone()) process.exit(1);
+  return new Promise((resolve) => {
+    process.on("message", async (job: Pick<FileWorkerRequest, "request">) => {
+      const outcome = await prepareFile(job.request, {
+        convert: convertForPreparation,
+        chunker: defaultChunker,
+        onPrevious: (structure) => send({ type: "previous", structure }),
+        onPhase: (phase) => send({ type: "phase", phase }),
+        // The parent enforces the budget and kills this process; between
+        // steps the child only checks that its parent is still alive.
+        check: () => {
+          if (parentGone()) process.exit(1);
+          return null;
+        },
+      });
+      send({ type: "done", outcome });
+    });
+    process.on("disconnect", () => resolve());
+    send({ type: "ready" });
+  });
+}
+
+if (import.meta.main) {
+  await runFileProcessorChild();
+  process.exit(0);
+}