@theglitchking/semantic-pages 0.1.0

package/dist/chunk-KF45H64M.js

@@ -0,0 +1,622 @@
+// src/core/embedder.ts
+import { pipeline } from "@huggingface/transformers";
+import { readFile, writeFile, mkdir } from "fs/promises";
+import { join } from "path";
+import { existsSync } from "fs";
+import { homedir } from "os";
+var DEFAULT_MODEL = "nomic-ai/nomic-embed-text-v1.5";
+var CACHE_DIR = join(homedir(), ".semantic-pages", "models");
+var Embedder = class {
+  model;
+  extractor = null;
+  dimensions = 0;
+  constructor(model = DEFAULT_MODEL) {
+    this.model = model;
+  }
+  async init() {
+    if (this.extractor) return;
+    await mkdir(CACHE_DIR, { recursive: true });
+    this.extractor = await pipeline("feature-extraction", this.model, {
+      cache_dir: CACHE_DIR,
+      dtype: "fp32"
+    });
+    const test = await this.embed("test");
+    this.dimensions = test.length;
+  }
+  async embed(text) {
+    if (!this.extractor) throw new Error("Embedder not initialized. Call init() first.");
+    const output = await this.extractor(text, { pooling: "mean", normalize: true });
+    return new Float32Array(output.data);
+  }
+  async embedBatch(texts) {
+    return Promise.all(texts.map((t) => this.embed(t)));
+  }
+  getDimensions() {
+    return this.dimensions;
+  }
+  getModel() {
+    return this.model;
+  }
+  async saveEmbeddings(embeddings, indexPath) {
+    const entries = [];
+    for (const [key, vec] of embeddings) {
+      entries.push({ key, data: Array.from(vec) });
+    }
+    await writeFile(join(indexPath, "embeddings.json"), JSON.stringify(entries));
+  }
+  async loadEmbeddings(indexPath) {
+    const filePath = join(indexPath, "embeddings.json");
+    if (!existsSync(filePath)) return /* @__PURE__ */ new Map();
+    const raw = await readFile(filePath, "utf-8");
+    const entries = JSON.parse(raw);
+    const map = /* @__PURE__ */ new Map();
+    for (const entry of entries) {
+      map.set(entry.key, new Float32Array(entry.data));
+    }
+    return map;
+  }
+};
+
+// src/core/graph.ts
+import Graph from "graphology";
+import { bfsFromNode } from "graphology-traversal";
+import { bidirectional } from "graphology-shortest-path";
+import { readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
+import { join as join2 } from "path";
+import { existsSync as existsSync2 } from "fs";
+var GraphBuilder = class {
+  graph;
+  constructor() {
+    this.graph = new Graph({ type: "directed", multi: false });
+  }
+  buildFromDocuments(documents) {
+    this.graph.clear();
+    for (const doc of documents) {
+      this.graph.addNode(doc.path, {
+        title: doc.title,
+        tags: doc.tags
+      });
+    }
+    const pathLookup = /* @__PURE__ */ new Map();
+    for (const doc of documents) {
+      const nameNoExt = doc.path.replace(/\.md$/, "");
+      const basename = nameNoExt.split("/").pop();
+      pathLookup.set(basename.toLowerCase(), doc.path);
+      pathLookup.set(nameNoExt.toLowerCase(), doc.path);
+    }
+    for (const doc of documents) {
+      for (const link of doc.wikilinks) {
+        const target = pathLookup.get(link.toLowerCase());
+        if (target && target !== doc.path && !this.graph.hasEdge(doc.path, target)) {
+          this.graph.addEdge(doc.path, target, { type: "wikilink", weight: 1 });
+        }
+      }
+    }
+    const tagToNotes = /* @__PURE__ */ new Map();
+    for (const doc of documents) {
+      for (const tag of doc.tags) {
+        const notes = tagToNotes.get(tag) || [];
+        notes.push(doc.path);
+        tagToNotes.set(tag, notes);
+      }
+    }
+    for (const [, notes] of tagToNotes) {
+      for (let i = 0; i < notes.length; i++) {
+        for (let j = i + 1; j < notes.length; j++) {
+          if (!this.graph.hasEdge(notes[i], notes[j])) {
+            this.graph.addEdge(notes[i], notes[j], { type: "tag", weight: 0.5 });
+          }
+          if (!this.graph.hasEdge(notes[j], notes[i])) {
+            this.graph.addEdge(notes[j], notes[i], { type: "tag", weight: 0.5 });
+          }
+        }
+      }
+    }
+  }
+  backlinks(notePath) {
+    if (!this.graph.hasNode(notePath)) return [];
+    return this.graph.inNeighbors(notePath).map((n) => this.nodeToGraphNode(n));
+  }
+  forwardlinks(notePath) {
+    if (!this.graph.hasNode(notePath)) return [];
+    return this.graph.outNeighbors(notePath).map((n) => this.nodeToGraphNode(n));
+  }
+  findPath(from, to) {
+    if (!this.graph.hasNode(from) || !this.graph.hasNode(to)) return null;
+    const path = bidirectional(this.graph, from, to);
+    return path;
+  }
+  searchGraph(concept, maxDepth = 2) {
+    const startNodes = this.graph.nodes().filter((n) => {
+      const attrs = this.graph.getNodeAttributes(n);
+      return n.toLowerCase().includes(concept.toLowerCase()) || attrs.title?.toLowerCase().includes(concept.toLowerCase()) || attrs.tags?.some((t) => t.toLowerCase().includes(concept.toLowerCase()));
+    });
+    const visited = /* @__PURE__ */ new Set();
+    for (const start of startNodes) {
+      let depth = 0;
+      bfsFromNode(this.graph, start, (node) => {
+        visited.add(node);
+        depth++;
+        return depth > maxDepth;
+      });
+    }
+    return [...visited].map((n) => this.nodeToGraphNode(n));
+  }
+  statistics() {
+    const nodes = this.graph.order;
+    const edges = this.graph.size;
+    const orphans = this.graph.nodes().filter(
+      (n) => this.graph.degree(n) === 0
+    );
+    const connections = this.graph.nodes().map((n) => ({
+      path: n,
+      connections: this.graph.degree(n)
+    }));
+    connections.sort((a, b) => b.connections - a.connections);
+    const maxPossibleEdges = nodes * (nodes - 1);
+    const density = maxPossibleEdges > 0 ? edges / maxPossibleEdges : 0;
+    return {
+      totalNodes: nodes,
+      totalEdges: edges,
+      orphanCount: orphans.length,
+      mostConnected: connections.slice(0, 10),
+      density
+    };
+  }
+  nodeToGraphNode(nodePath) {
+    const attrs = this.graph.getNodeAttributes(nodePath);
+    return {
+      path: nodePath,
+      title: attrs.title || nodePath,
+      tags: attrs.tags || [],
+      linkCount: this.graph.outDegree(nodePath),
+      backlinkCount: this.graph.inDegree(nodePath)
+    };
+  }
+  async save(indexPath) {
+    const data = this.graph.export();
+    await writeFile2(join2(indexPath, "graph.json"), JSON.stringify(data));
+  }
+  async load(indexPath) {
+    const filePath = join2(indexPath, "graph.json");
+    if (!existsSync2(filePath)) return false;
+    const raw = await readFile2(filePath, "utf-8");
+    const data = JSON.parse(raw);
+    this.graph.import(data);
+    return true;
+  }
+  getGraph() {
+    return this.graph;
+  }
+};
+
+// src/core/vector.ts
+import hnswlib from "hnswlib-node";
+import { readFile as readFile3, writeFile as writeFile3 } from "fs/promises";
+import { join as join3 } from "path";
+import { existsSync as existsSync3 } from "fs";
+var { HierarchicalNSW } = hnswlib;
+var VectorIndex = class {
+  index = null;
+  dimensions;
+  chunkMeta = [];
+  constructor(dimensions) {
+    this.dimensions = dimensions;
+  }
+  build(embeddings, meta) {
+    if (embeddings.length === 0) {
+      this.index = null;
+      this.chunkMeta = [];
+      return;
+    }
+    this.index = new HierarchicalNSW("cosine", this.dimensions);
+    this.index.initIndex(embeddings.length);
+    for (let i = 0; i < embeddings.length; i++) {
+      this.index.addPoint(Array.from(embeddings[i]), i);
+    }
+    this.chunkMeta = meta;
+  }
+  search(queryEmbedding, k = 10) {
+    if (!this.index || this.chunkMeta.length === 0) return [];
+    const numResults = Math.min(k, this.chunkMeta.length);
+    const result = this.index.searchKnn(Array.from(queryEmbedding), numResults);
+    const seen = /* @__PURE__ */ new Set();
+    const results = [];
+    for (let i = 0; i < result.neighbors.length; i++) {
+      const idx = result.neighbors[i];
+      const meta = this.chunkMeta[idx];
+      if (!meta || seen.has(meta.docPath)) continue;
+      seen.add(meta.docPath);
+      results.push({
+        path: meta.docPath,
+        title: meta.docPath,
+        score: 1 - result.distances[i],
+        snippet: meta.text.slice(0, 200),
+        matchedChunk: meta.text
+      });
+    }
+    return results;
+  }
+  async save(indexPath) {
+    if (!this.index) return;
+    this.index.writeIndexSync(join3(indexPath, "hnsw.bin"));
+    await writeFile3(
+      join3(indexPath, "hnsw-meta.json"),
+      JSON.stringify(this.chunkMeta)
+    );
+  }
+  async load(indexPath) {
+    const hnswPath = join3(indexPath, "hnsw.bin");
+    const metaPath = join3(indexPath, "hnsw-meta.json");
+    if (!existsSync3(hnswPath) || !existsSync3(metaPath)) return false;
+    const raw = await readFile3(metaPath, "utf-8");
+    this.chunkMeta = JSON.parse(raw);
+    this.index = new HierarchicalNSW("cosine", this.dimensions);
+    this.index.initIndex(this.chunkMeta.length);
+    this.index.readIndexSync(hnswPath);
+    return true;
+  }
+  getChunkMeta() {
+    return this.chunkMeta;
+  }
+};
+
+// src/core/search-text.ts
+import { minimatch } from "minimatch";
+var TextSearch = class {
+  documents = [];
+  setDocuments(documents) {
+    this.documents = documents;
+  }
+  search(options) {
+    const { pattern, regex, caseSensitive, pathGlob, tagFilter, limit = 20 } = options;
+    let matcher;
+    if (regex) {
+      const flags = caseSensitive ? "g" : "gi";
+      const re = new RegExp(pattern, flags);
+      matcher = (text) => {
+        re.lastIndex = 0;
+        const m = re.exec(text);
+        return { matched: !!m, index: m?.index ?? -1 };
+      };
+    } else {
+      const needle = caseSensitive ? pattern : pattern.toLowerCase();
+      matcher = (text) => {
+        const haystack = caseSensitive ? text : text.toLowerCase();
+        const idx = haystack.indexOf(needle);
+        return { matched: idx >= 0, index: idx };
+      };
+    }
+    const results = [];
+    for (const doc of this.documents) {
+      if (pathGlob && !minimatch(doc.path, pathGlob)) continue;
+      if (tagFilter?.length) {
+        const hasTag = tagFilter.some((t) => doc.tags.includes(t));
+        if (!hasTag) continue;
+      }
+      const { matched, index } = matcher(doc.content);
+      if (!matched) continue;
+      const snippetStart = Math.max(0, index - 80);
+      const snippetEnd = Math.min(doc.content.length, index + 120);
+      const snippet = doc.content.slice(snippetStart, snippetEnd).trim();
+      results.push({
+        path: doc.path,
+        title: doc.title,
+        score: 1,
+        snippet: (snippetStart > 0 ? "..." : "") + snippet + (snippetEnd < doc.content.length ? "..." : "")
+      });
+      if (results.length >= limit) break;
+    }
+    return results;
+  }
+};
+
+// src/core/crud.ts
+import { readFile as readFile4, writeFile as writeFile4, unlink, rename, mkdir as mkdir2 } from "fs/promises";
+import { dirname, join as join4 } from "path";
+import { existsSync as existsSync4 } from "fs";
+import { glob } from "glob";
+import matter from "gray-matter";
+var NoteCrud = class {
+  notesPath;
+  constructor(notesPath) {
+    this.notesPath = notesPath;
+  }
+  async create(relativePath, content, frontmatter) {
+    const absPath = join4(this.notesPath, relativePath);
+    if (existsSync4(absPath)) {
+      throw new Error(`Note already exists: ${relativePath}`);
+    }
+    await mkdir2(dirname(absPath), { recursive: true });
+    let fileContent;
+    if (frontmatter && Object.keys(frontmatter).length > 0) {
+      fileContent = matter.stringify(content, frontmatter);
+    } else {
+      fileContent = content;
+    }
+    await writeFile4(absPath, fileContent, "utf-8");
+    return relativePath;
+  }
+  async read(relativePath) {
+    const absPath = join4(this.notesPath, relativePath);
+    return readFile4(absPath, "utf-8");
+  }
+  async readMultiple(paths) {
+    const results = /* @__PURE__ */ new Map();
+    await Promise.all(
+      paths.map(async (p) => {
+        try {
+          const content = await this.read(p);
+          results.set(p, content);
+        } catch {
+          results.set(p, `[Error: could not read ${p}]`);
+        }
+      })
+    );
+    return results;
+  }
+  async update(relativePath, content, options) {
+    const absPath = join4(this.notesPath, relativePath);
+    if (!existsSync4(absPath)) {
+      throw new Error(`Note does not exist: ${relativePath}`);
+    }
+    const existing = await readFile4(absPath, "utf-8");
+    let updated;
+    switch (options.mode) {
+      case "overwrite":
+        updated = content;
+        break;
+      case "append":
+        updated = existing + "\n" + content;
+        break;
+      case "prepend": {
+        const { data, content: body } = matter(existing);
+        const newBody = content + "\n" + body;
+        updated = Object.keys(data).length > 0 ? matter.stringify(newBody, data) : newBody;
+        break;
+      }
+      case "patch-by-heading": {
+        if (!options.heading) throw new Error("patch-by-heading requires a heading");
+        updated = this.patchByHeading(existing, options.heading, content);
+        break;
+      }
+    }
+    await writeFile4(absPath, updated, "utf-8");
+  }
+  async delete(relativePath) {
+    const absPath = join4(this.notesPath, relativePath);
+    if (!existsSync4(absPath)) {
+      throw new Error(`Note does not exist: ${relativePath}`);
+    }
+    await unlink(absPath);
+  }
+  async move(fromPath, toPath) {
+    const absFrom = join4(this.notesPath, fromPath);
+    const absTo = join4(this.notesPath, toPath);
+    if (!existsSync4(absFrom)) {
+      throw new Error(`Note does not exist: ${fromPath}`);
+    }
+    if (existsSync4(absTo)) {
+      throw new Error(`Destination already exists: ${toPath}`);
+    }
+    await mkdir2(dirname(absTo), { recursive: true });
+    await rename(absFrom, absTo);
+    await this.updateWikilinksAfterMove(fromPath, toPath);
+  }
+  async updateWikilinksAfterMove(oldPath, newPath) {
+    const oldName = oldPath.replace(/\.md$/, "").split("/").pop();
+    const newName = newPath.replace(/\.md$/, "").split("/").pop();
+    if (oldName === newName) return;
+    const files = await glob("**/*.md", { cwd: this.notesPath });
+    for (const file of files) {
+      const absPath = join4(this.notesPath, file);
+      const content = await readFile4(absPath, "utf-8");
+      const pattern = new RegExp(`\\[\\[${escapeRegex(oldName)}(\\|[^\\]]*)?\\]\\]`, "g");
+      if (!pattern.test(content)) continue;
+      const updated = content.replace(pattern, `[[${newName}$1]]`);
+      await writeFile4(absPath, updated, "utf-8");
+    }
+  }
+  patchByHeading(content, heading, newContent) {
+    const lines = content.split("\n");
+    const headingPattern = new RegExp(`^#{1,6}\\s+${escapeRegex(heading)}\\s*$`, "i");
+    let headingIndex = -1;
+    let headingLevel = 0;
+    for (let i = 0; i < lines.length; i++) {
+      if (headingPattern.test(lines[i])) {
+        headingIndex = i;
+        const match = lines[i].match(/^(#{1,6})\s+/);
+        headingLevel = match ? match[1].length : 1;
+        break;
+      }
+    }
+    if (headingIndex === -1) {
+      throw new Error(`Heading not found: ${heading}`);
+    }
+    let endIndex = lines.length;
+    for (let i = headingIndex + 1; i < lines.length; i++) {
+      const match = lines[i].match(/^(#{1,6})\s+/);
+      if (match && match[1].length <= headingLevel) {
+        endIndex = i;
+        break;
+      }
+    }
+    const before = lines.slice(0, headingIndex + 1);
+    const after = lines.slice(endIndex);
+    return [...before, "", newContent, "", ...after].join("\n");
+  }
+};
+function escapeRegex(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// src/core/frontmatter.ts
+import { readFile as readFile5, writeFile as writeFile5 } from "fs/promises";
+import { join as join5 } from "path";
+import { glob as glob2 } from "glob";
+import matter2 from "gray-matter";
+var FrontmatterManager = class {
+  notesPath;
+  constructor(notesPath) {
+    this.notesPath = notesPath;
+  }
+  async get(relativePath) {
+    const absPath = join5(this.notesPath, relativePath);
+    const raw = await readFile5(absPath, "utf-8");
+    const { data } = matter2(raw);
+    return data;
+  }
+  async update(relativePath, fields) {
+    const absPath = join5(this.notesPath, relativePath);
+    const raw = await readFile5(absPath, "utf-8");
+    const { data, content } = matter2(raw);
+    for (const [key, value] of Object.entries(fields)) {
+      if (value === null || value === void 0) {
+        delete data[key];
+      } else {
+        data[key] = value;
+      }
+    }
+    const updated = matter2.stringify(content, data);
+    await writeFile5(absPath, updated, "utf-8");
+  }
+};
+var TagManager = class {
+  notesPath;
+  constructor(notesPath) {
+    this.notesPath = notesPath;
+  }
+  async list(relativePath) {
+    const absPath = join5(this.notesPath, relativePath);
+    const raw = await readFile5(absPath, "utf-8");
+    const { data, content } = matter2(raw);
+    const fmTags = Array.isArray(data.tags) ? data.tags : [];
+    const inlineTags = [...content.matchAll(/(?:^|\s)#([a-zA-Z][\w-/]*)/g)].map(
+      (m) => m[1]
+    );
+    return [.../* @__PURE__ */ new Set([...fmTags, ...inlineTags])];
+  }
+  async add(relativePath, tags) {
+    const absPath = join5(this.notesPath, relativePath);
+    const raw = await readFile5(absPath, "utf-8");
+    const { data, content } = matter2(raw);
+    const existing = Array.isArray(data.tags) ? data.tags : [];
+    const merged = [.../* @__PURE__ */ new Set([...existing, ...tags])];
+    data.tags = merged;
+    const updated = matter2.stringify(content, data);
+    await writeFile5(absPath, updated, "utf-8");
+  }
+  async remove(relativePath, tags) {
+    const absPath = join5(this.notesPath, relativePath);
+    const raw = await readFile5(absPath, "utf-8");
+    const { data, content } = matter2(raw);
+    if (Array.isArray(data.tags)) {
+      data.tags = data.tags.filter((t) => !tags.includes(t));
+    }
+    let updatedContent = content;
+    for (const tag of tags) {
+      const pattern = new RegExp(`(^|\\s)#${escapeRegex2(tag)}(?=\\s|$)`, "g");
+      updatedContent = updatedContent.replace(pattern, "$1");
+    }
+    const updated = matter2.stringify(updatedContent, data);
+    await writeFile5(absPath, updated, "utf-8");
+  }
+  async renameVaultWide(oldTag, newTag) {
+    const files = await glob2("**/*.md", { cwd: this.notesPath });
+    let count = 0;
+    for (const file of files) {
+      const absPath = join5(this.notesPath, file);
+      const raw = await readFile5(absPath, "utf-8");
+      const { data, content } = matter2(raw);
+      let changed = false;
+      if (Array.isArray(data.tags)) {
+        const idx = data.tags.indexOf(oldTag);
+        if (idx >= 0) {
+          data.tags[idx] = newTag;
+          changed = true;
+        }
+      }
+      const pattern = new RegExp(`(^|\\s)#${escapeRegex2(oldTag)}(?=\\s|$)`, "g");
+      const updatedContent = content.replace(pattern, `$1#${newTag}`);
+      if (updatedContent !== content) changed = true;
+      if (changed) {
+        const updated = matter2.stringify(updatedContent, data);
+        await writeFile5(absPath, updated, "utf-8");
+        count++;
+      }
+    }
+    return count;
+  }
+};
+function escapeRegex2(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// src/core/watcher.ts
+import { watch } from "chokidar";
+import { EventEmitter } from "events";
+var Watcher = class extends EventEmitter {
+  notesPath;
+  fsWatcher = null;
+  debounceMs;
+  pendingChanges = /* @__PURE__ */ new Set();
+  debounceTimer = null;
+  readyPromise = null;
+  usePolling;
+  constructor(notesPath, debounceMs = 500, usePolling = false) {
+    super();
+    this.notesPath = notesPath;
+    this.debounceMs = debounceMs;
+    this.usePolling = usePolling;
+  }
+  start() {
+    if (this.fsWatcher) return;
+    this.fsWatcher = watch("**/*.md", {
+      cwd: this.notesPath,
+      ignoreInitial: true,
+      ignored: [
+        "**/node_modules/**",
+        "**/.semantic-pages-index/**",
+        "**/.git/**"
+      ],
+      ...this.usePolling ? { usePolling: true, interval: 100 } : {}
+    });
+    this.readyPromise = new Promise((resolve) => {
+      this.fsWatcher.on("ready", resolve);
+    });
+    this.fsWatcher.on("add", (path) => this.enqueue(path));
+    this.fsWatcher.on("change", (path) => this.enqueue(path));
+    this.fsWatcher.on("unlink", (path) => this.enqueue(path));
+    this.fsWatcher.on("error", (err) => this.emit("error", err));
+  }
+  async ready() {
+    if (this.readyPromise) await this.readyPromise;
+  }
+  stop() {
+    if (this.debounceTimer) clearTimeout(this.debounceTimer);
+    this.fsWatcher?.close();
+    this.fsWatcher = null;
+    this.pendingChanges.clear();
+  }
+  enqueue(path) {
+    this.pendingChanges.add(path);
+    if (this.debounceTimer) clearTimeout(this.debounceTimer);
+    this.debounceTimer = setTimeout(() => {
+      const paths = [...this.pendingChanges];
+      this.pendingChanges.clear();
+      this.emit("changed", paths);
+    }, this.debounceMs);
+  }
+};
+
+export {
+  Embedder,
+  GraphBuilder,
+  VectorIndex,
+  TextSearch,
+  NoteCrud,
+  FrontmatterManager,
+  TagManager,
+  Watcher
+};
+//# sourceMappingURL=chunk-KF45H64M.js.map