@rubytech/create-maxy-code 0.1.138 → 0.1.142

package/payload/platform/lib/obsidian-parser/src/index.ts

@@ -0,0 +1,572 @@
+/**
+ * obsidian-parser — pure functions for parsing an Obsidian vault on disk.
+ *
+ * No DB access, no MCP, no I/O side effects other than fs reads. Composed by
+ * the `obsidian-vault-import` MCP tool in the memory plugin.
+ *
+ * Doctrine: every regex is linear-time (no catastrophic backtracking), every
+ * function is deterministic. The agent never calls this code; the server does.
+ */
+
+import { createHash } from "node:crypto";
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ParsedPage {
+  /** Path relative to the vault root, with forward slashes. The MERGE natural key. */
+  obsidianPath: string;
+  /** Filename without `.md` — the default wikilink target form. */
+  title: string;
+  /** Aliases declared in frontmatter (`aliases:` array). */
+  aliases: string[];
+  /** Tag tokens (without `#`), unioned from frontmatter `tags:` and inline `#tag` body matches. */
+  tags: string[];
+  /** Body text (frontmatter stripped). Stored verbatim. */
+  body: string;
+  /** SHA-256 of the on-disk file bytes — idempotency key. */
+  contentHash: string;
+  /** Detected daily-note date (YYYY-MM-DD) or null. */
+  noteDate: string | null;
+  /** Wikilinks emitted by this page. */
+  wikilinks: WikilinkOccurrence[];
+  /** Embedded attachments declared via `![[file.png]]` or `![alt](file.png)`. */
+  attachments: AttachmentOccurrence[];
+  /** Last-modified timestamp from fs stat — used for modified-since filtering. */
+  mtimeMs: number;
+}
+
+export interface WikilinkOccurrence {
+  /** The exact text inside `[[…]]` before any alias `|`. May contain `#anchor`. */
+  rawTarget: string;
+  /** Target after stripping `#anchor`. */
+  target: string;
+  /** Anchor portion (after `#`) or null. */
+  anchor: string | null;
+  /** Alias text after `|`, or null when absent. The display text. */
+  alias: string | null;
+}
+
+export interface AttachmentOccurrence {
+  /** Path as written in the markdown. May be relative to the page or to the vault root. */
+  rawPath: string;
+  /** Resolved path relative to vault root, normalised. May not exist on disk yet. */
+  resolvedPath: string;
+  /** Display alt text (from `![alt](path)` form) or null for embed-syntax `![[file]]`. */
+  alt: string | null;
+}
+
+export interface VaultWalkOptions {
+  /** Filter: only pages whose path starts with one of these folder prefixes. */
+  folderPrefixes?: string[];
+  /** Filter: only pages whose tags intersect with this set. */
+  tagFilter?: string[];
+  /** Filter: only pages modified at or after this epoch-ms. */
+  modifiedSinceMs?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Vault walk
+// ---------------------------------------------------------------------------
+
+/**
+ * Walk the vault directory and return one ParsedPage per `.md` file that
+ * survives the optional filter set.
+ *
+ * Path-traversal defence: paths containing `..` segments are rejected at
+ * entry. The `vaultRoot` passed in is assumed to have already been verified
+ * by the caller as living under the account's archive directory.
+ */
+export async function walkVault(
+  vaultRoot: string,
+  opts: VaultWalkOptions = {},
+): Promise<ParsedPage[]> {
+  const pages: ParsedPage[] = [];
+  const queue: string[] = [vaultRoot];
+
+  while (queue.length > 0) {
+    const dir = queue.shift()!;
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch (err) {
+      throw new Error(
+        `obsidian-parser: cannot read directory ${dir}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      const abs = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        queue.push(abs);
+        continue;
+      }
+      if (!entry.isFile()) continue;
+      if (!entry.name.toLowerCase().endsWith(".md")) continue;
+
+      const rel = toPosixRelative(vaultRoot, abs);
+      if (rel.split("/").includes("..")) continue;
+
+      if (opts.folderPrefixes && opts.folderPrefixes.length > 0) {
+        const matchesPrefix = opts.folderPrefixes.some((p) =>
+          rel === p || rel.startsWith(p.replace(/\/?$/, "/")),
+        );
+        if (!matchesPrefix) continue;
+      }
+
+      let stat;
+      let bytes;
+      try {
+        stat = await fs.stat(abs);
+        bytes = await fs.readFile(abs);
+      } catch (err) {
+        // Per-file isolation — log and skip, don't abort the walk.
+        process.stderr.write(
+          `[obsidian-parser] event=read-failed file=${rel} reason=${
+            err instanceof Error ? err.message : String(err)
+          }\n`,
+        );
+        continue;
+      }
+
+      if (opts.modifiedSinceMs && stat.mtimeMs < opts.modifiedSinceMs) continue;
+
+      let parsed: ParsedPage;
+      try {
+        parsed = parsePage(rel, bytes, stat.mtimeMs);
+      } catch (err) {
+        process.stderr.write(
+          `[obsidian-parser] event=parse-failed file=${rel} reason=${
+            err instanceof Error ? err.message : String(err)
+          }\n`,
+        );
+        continue;
+      }
+
+      if (opts.tagFilter && opts.tagFilter.length > 0) {
+        const wanted = new Set(opts.tagFilter.map(normalizeTag));
+        const hit = parsed.tags.some((t) => wanted.has(normalizeTag(t)));
+        if (!hit) continue;
+      }
+
+      pages.push(parsed);
+    }
+  }
+
+  pages.sort((a, b) => a.obsidianPath.localeCompare(b.obsidianPath));
+  return pages;
+}
+
+function toPosixRelative(root: string, abs: string): string {
+  const rel = path.relative(root, abs);
+  return rel.split(path.sep).join("/");
+}
+
+// ---------------------------------------------------------------------------
+// Page parser
+// ---------------------------------------------------------------------------
+
+const DAILY_NOTE_FILENAME_RE = /^(\d{4})-(\d{2})-(\d{2})(?:\.md)?$/;
+const DAILY_NOTE_FOLDER_NAMES = new Set([
+  "daily",
+  "daily-notes",
+  "daily notes",
+  "journal",
+  "journals",
+]);
+
+export function parsePage(
+  obsidianPath: string,
+  bytes: Buffer,
+  mtimeMs: number,
+): ParsedPage {
+  const contentHash = createHash("sha256").update(bytes).digest("hex");
+  const text = bytes.toString("utf8");
+
+  const { frontmatter, body } = splitFrontmatter(text);
+
+  const title = path
+    .basename(obsidianPath)
+    .replace(/\.md$/i, "")
+    .normalize("NFC");
+
+  const aliases = extractAliases(frontmatter);
+  const frontmatterTags = extractFrontmatterTags(frontmatter);
+  const { bodyTags, wikilinks, attachments } = scanBody(body, obsidianPath);
+  const tags = dedupeNormalized([...frontmatterTags, ...bodyTags]);
+  const noteDate = detectDailyNote(obsidianPath, frontmatter);
+
+  return {
+    obsidianPath,
+    title,
+    aliases: aliases.map((a) => a.normalize("NFC")),
+    tags,
+    body,
+    contentHash,
+    noteDate,
+    wikilinks,
+    attachments,
+    mtimeMs,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Frontmatter
+// ---------------------------------------------------------------------------
+
+interface ParsedFrontmatter {
+  raw: string;
+  /** Map of top-level key → raw value string. Arrays are returned as the
+   *  block following the key (caller parses per-key). */
+  fields: Map<string, string>;
+}
+
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/;
+
+export function splitFrontmatter(
+  text: string,
+): { frontmatter: ParsedFrontmatter; body: string } {
+  const match = text.match(FRONTMATTER_RE);
+  if (!match) {
+    return { frontmatter: { raw: "", fields: new Map() }, body: text };
+  }
+  const raw = match[1];
+  const body = text.slice(match[0].length);
+
+  // Minimal YAML-subset parser: top-level `key: value` and `key:` followed by
+  // `  - item` lines. Not a full YAML — we explicitly limit the surface so a
+  // malformed frontmatter degrades to "no fields found" rather than throwing.
+  const fields = new Map<string, string>();
+  const lines = raw.split(/\r?\n/);
+  let currentKey: string | null = null;
+  let currentBlock: string[] = [];
+
+  for (const line of lines) {
+    if (/^[#]/.test(line.trim())) continue; // comment
+    const flush = () => {
+      if (currentKey !== null) {
+        const existing = fields.get(currentKey);
+        const next = currentBlock.join("\n");
+        fields.set(currentKey, existing ? existing + "\n" + next : next);
+      }
+      currentKey = null;
+      currentBlock = [];
+    };
+
+    const topMatch = line.match(/^([A-Za-z_][\w-]*)\s*:\s*(.*)$/);
+    if (topMatch && !line.startsWith(" ") && !line.startsWith("\t")) {
+      flush();
+      currentKey = topMatch[1];
+      const value = topMatch[2];
+      if (value.trim() === "") {
+        currentBlock = [];
+      } else {
+        currentBlock = [value];
+        flush();
+      }
+      continue;
+    }
+    if (currentKey !== null) {
+      currentBlock.push(line);
+    }
+  }
+  if (currentKey !== null) {
+    const existing = fields.get(currentKey);
+    const next = currentBlock.join("\n");
+    fields.set(currentKey, existing ? existing + "\n" + next : next);
+  }
+
+  return { frontmatter: { raw, fields }, body };
+}
+
+function extractAliases(fm: ParsedFrontmatter): string[] {
+  return parseYamlList(fm.fields.get("aliases"));
+}
+
+function extractFrontmatterTags(fm: ParsedFrontmatter): string[] {
+  return parseYamlList(fm.fields.get("tags")).map(normalizeTag);
+}
+
+function parseYamlList(value: string | undefined): string[] {
+  if (!value) return [];
+  const trimmed = value.trim();
+  if (!trimmed) return [];
+  // Flow-style: [a, b, c]
+  if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
+    return trimmed
+      .slice(1, -1)
+      .split(",")
+      .map((s) => s.trim().replace(/^["']|["']$/g, ""))
+      .filter(Boolean);
+  }
+  // Block-style: - item per line, or single scalar
+  const lines = trimmed.split(/\r?\n/);
+  if (lines.length === 1 && !lines[0].startsWith("-")) {
+    return [lines[0].replace(/^["']|["']$/g, "").trim()].filter(Boolean);
+  }
+  const out: string[] = [];
+  for (const line of lines) {
+    const m = line.match(/^\s*-\s*(.*?)\s*$/);
+    if (m) {
+      const v = m[1].replace(/^["']|["']$/g, "").trim();
+      if (v) out.push(v);
+    }
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Body scan: wikilinks, tags, attachments — all code-fence-aware
+// ---------------------------------------------------------------------------
+
+const WIKILINK_RE = /\[\[([^\]\r\n]+)\]\]/g;
+const EMBED_RE = /!\[\[([^\]\r\n]+)\]\]/g;
+const IMAGE_RE = /!\[([^\]\r\n]*)\]\(([^)\r\n]+)\)/g;
+const INLINE_TAG_RE = /(^|[\s(])#([A-Za-z][\w/-]*)/g;
+const ATTACHMENT_EXT_RE = /\.(png|jpe?g|gif|webp|svg|pdf|mp3|mp4|mov|wav|m4a|webm)$/i;
+
+interface BodyScan {
+  bodyTags: string[];
+  wikilinks: WikilinkOccurrence[];
+  attachments: AttachmentOccurrence[];
+}
+
+function scanBody(body: string, obsidianPath: string): BodyScan {
+  const bodyTags: string[] = [];
+  const wikilinks: WikilinkOccurrence[] = [];
+  const attachments: AttachmentOccurrence[] = [];
+
+  for (const segment of splitOnCodeFences(body)) {
+    if (segment.isCode) continue;
+    const text = segment.text;
+
+    // Embeds first — they share `[[…]]` with wikilinks but the `!` prefix
+    // routes them to attachments.
+    const embedSpans: Array<[number, number]> = [];
+    for (const m of text.matchAll(EMBED_RE)) {
+      const inner = m[1].trim();
+      const resolved = resolveAttachmentPath(inner, obsidianPath);
+      if (resolved && ATTACHMENT_EXT_RE.test(resolved)) {
+        attachments.push({ rawPath: inner, resolvedPath: resolved, alt: null });
+      } else {
+        // Non-attachment embed (e.g. another note transclusion). Treat as a
+        // wikilink so the cross-reference is preserved.
+        wikilinks.push(parseWikilinkInner(inner));
+      }
+      embedSpans.push([m.index ?? 0, (m.index ?? 0) + m[0].length]);
+    }
+
+    for (const m of text.matchAll(WIKILINK_RE)) {
+      const start = m.index ?? 0;
+      if (embedSpans.some(([s, e]) => start >= s - 1 && start < e)) continue;
+      wikilinks.push(parseWikilinkInner(m[1]));
+    }
+
+    for (const m of text.matchAll(IMAGE_RE)) {
+      const rawPath = m[2].trim();
+      const alt = m[1].trim() || null;
+      if (/^https?:\/\//i.test(rawPath)) continue; // remote
+      const resolved = resolveAttachmentPath(rawPath, obsidianPath);
+      if (resolved && ATTACHMENT_EXT_RE.test(resolved)) {
+        attachments.push({ rawPath, resolvedPath: resolved, alt });
+      }
+    }
+
+    for (const m of text.matchAll(INLINE_TAG_RE)) {
+      bodyTags.push(normalizeTag(m[2]));
+    }
+  }
+
+  return {
+    bodyTags: dedupeNormalized(bodyTags),
+    wikilinks,
+    attachments,
+  };
+}
+
+function parseWikilinkInner(inner: string): WikilinkOccurrence {
+  const trimmed = inner.trim();
+  const aliasIdx = trimmed.indexOf("|");
+  let raw = trimmed;
+  let alias: string | null = null;
+  if (aliasIdx >= 0) {
+    raw = trimmed.slice(0, aliasIdx).trim();
+    alias = trimmed.slice(aliasIdx + 1).trim() || null;
+  }
+  const anchorIdx = raw.indexOf("#");
+  let target = raw;
+  let anchor: string | null = null;
+  if (anchorIdx >= 0) {
+    target = raw.slice(0, anchorIdx).trim();
+    anchor = raw.slice(anchorIdx + 1).trim() || null;
+  }
+  return { rawTarget: raw, target: target.normalize("NFC"), anchor, alias };
+}
+
+function resolveAttachmentPath(
+  rawPath: string,
+  obsidianPath: string,
+): string | null {
+  if (!rawPath) return null;
+  if (rawPath.startsWith("/") || /^[A-Za-z]:[\\/]/.test(rawPath)) return null;
+  if (rawPath.split("/").includes("..")) return null;
+  // Obsidian's default: vault-root relative. We also accept page-relative.
+  // For the resolved form we prefer vault-root-relative (more deterministic
+  // for MERGE), normalising path separators.
+  const norm = rawPath.replace(/\\/g, "/");
+  if (norm.includes("/")) return norm;
+  // Bare filename — could be in any attachments folder. Caller's index does
+  // the lookup; we record the bare form.
+  return norm;
+}
+
+function splitOnCodeFences(text: string): Array<{ text: string; isCode: boolean }> {
+  const out: Array<{ text: string; isCode: boolean }> = [];
+  const fenceRe = /(^|\n)(```|~~~)[^\n]*\n([\s\S]*?)\n(```|~~~)/g;
+  let lastIdx = 0;
+  for (const m of text.matchAll(fenceRe)) {
+    const start = (m.index ?? 0) + (m[1] ? m[1].length : 0);
+    out.push({ text: text.slice(lastIdx, start), isCode: false });
+    out.push({ text: m[0], isCode: true });
+    lastIdx = start + m[0].length - (m[1] ? m[1].length : 0);
+  }
+  out.push({ text: text.slice(lastIdx), isCode: false });
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Tag normalization
+// ---------------------------------------------------------------------------
+
+export function normalizeTag(tag: string): string {
+  return tag
+    .replace(/^#/, "")
+    .normalize("NFC")
+    .trim()
+    .toLowerCase();
+}
+
+function dedupeNormalized(values: string[]): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const v of values) {
+    const k = v.normalize("NFC").trim().toLowerCase();
+    if (!k) continue;
+    if (seen.has(k)) continue;
+    seen.add(k);
+    out.push(v);
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Daily-notes detection
+// ---------------------------------------------------------------------------
+
+export function detectDailyNote(
+  obsidianPath: string,
+  fm: ParsedFrontmatter,
+): string | null {
+  // 1. Frontmatter `date:` or `created:` wins if it's an ISO date.
+  for (const key of ["date", "created"]) {
+    const v = fm.fields.get(key);
+    if (v) {
+      const m = v.trim().match(/^(\d{4}-\d{2}-\d{2})/);
+      if (m) return m[1];
+    }
+  }
+  // 2. Filename matches YYYY-MM-DD.md.
+  const basename = path.basename(obsidianPath);
+  const fileMatch = basename.match(DAILY_NOTE_FILENAME_RE);
+  if (fileMatch) return `${fileMatch[1]}-${fileMatch[2]}-${fileMatch[3]}`;
+
+  // 3. Filename is YYYY-MM-DD inside a known daily-notes folder.
+  const parts = obsidianPath.split("/");
+  const inDailyFolder = parts
+    .slice(0, -1)
+    .some((p) => DAILY_NOTE_FOLDER_NAMES.has(p.toLowerCase()));
+  if (inDailyFolder) {
+    const stem = basename.replace(/\.md$/i, "");
+    const m = stem.match(/^(\d{4})-(\d{2})-(\d{2})$/);
+    if (m) return `${m[1]}-${m[2]}-${m[3]}`;
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Wikilink resolution helpers
+// ---------------------------------------------------------------------------
+
+export interface VaultIndex {
+  /** Lowercased title (without `.md`) → set of `obsidianPath` candidates. */
+  byTitle: Map<string, string[]>;
+  /** Lowercased alias → set of `obsidianPath` candidates. */
+  byAlias: Map<string, string[]>;
+}
+
+export function buildVaultIndex(pages: ParsedPage[]): VaultIndex {
+  const byTitle = new Map<string, string[]>();
+  const byAlias = new Map<string, string[]>();
+  for (const p of pages) {
+    pushKey(byTitle, p.title.toLowerCase(), p.obsidianPath);
+    for (const a of p.aliases) {
+      pushKey(byAlias, a.toLowerCase(), p.obsidianPath);
+    }
+  }
+  return { byTitle, byAlias };
+}
+
+function pushKey(m: Map<string, string[]>, k: string, v: string): void {
+  const arr = m.get(k);
+  if (arr) {
+    if (!arr.includes(v)) arr.push(v);
+  } else {
+    m.set(k, [v]);
+  }
+}
+
+/**
+ * Look up a wikilink target inside the vault, returning candidate page paths.
+ *
+ * The target is matched against (a) page titles and (b) declared aliases.
+ * Path-form targets (`Folder/Page`) match directly. Returns an empty array
+ * when no intra-vault candidate exists — the caller falls through to entity
+ * resolution.
+ */
+export function resolveWikilinkInVault(
+  target: string,
+  index: VaultIndex,
+  knownPaths: Set<string>,
+): string[] {
+  const t = target.normalize("NFC").trim();
+  if (!t) return [];
+  // Path-form: contains a slash or ends with `.md`.
+  if (t.includes("/") || t.toLowerCase().endsWith(".md")) {
+    const norm = t.replace(/\\/g, "/").replace(/\.md$/i, "") + ".md";
+    if (knownPaths.has(norm)) return [norm];
+    return [];
+  }
+  const titleHits = index.byTitle.get(t.toLowerCase()) ?? [];
+  const aliasHits = index.byAlias.get(t.toLowerCase()) ?? [];
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const p of [...titleHits, ...aliasHits]) {
+    if (seen.has(p)) continue;
+    seen.add(p);
+    out.push(p);
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Import-id helper
+// ---------------------------------------------------------------------------
+
+export function makeImportId(): string {
+  return `obs-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+}

package/payload/platform/lib/obsidian-parser/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src"]
+}