@sunub/obsidian-mcp-server 0.3.25 → 0.3.27

package/build/utils/worker/WorkerPool.js

@@ -0,0 +1,89 @@
+import { cpus } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { Worker } from "node:worker_threads";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export class WorkerPool {
+    workers = [];
+    freeWorkers = [];
+    queue = [];
+    poolSize;
+    workerPath;
+    constructor(poolSize) {
+        this.poolSize = poolSize || Math.max(2, cpus().length - 1);
+        this.workerPath = path.resolve(__dirname, "./chunkingWorker.js");
+    }
+    init() {
+        if (this.workers.length > 0)
+            return;
+        for (let i = 0; i < this.poolSize; i++) {
+            const worker = new Worker(this.workerPath);
+            this.workers.push(worker);
+            this.freeWorkers.push(worker);
+        }
+    }
+    runTask(data) {
+        this.init();
+        return new Promise((resolve, reject) => {
+            this.queue.push({ data, resolve, reject });
+            this.executeNext();
+        });
+    }
+    executeNext() {
+        if (this.queue.length === 0 || this.freeWorkers.length === 0) {
+            return;
+        }
+        const worker = this.freeWorkers.pop();
+        if (!worker) {
+            console.error("Unexpected error: No free worker available after check.");
+            return;
+        }
+        const nextItem = this.queue.shift();
+        if (!nextItem) {
+            this.freeWorkers.push(worker);
+            return;
+        }
+        const { data, resolve, reject } = nextItem;
+        const messageHandler = (res) => {
+            cleanup();
+            this.freeWorkers.push(worker);
+            this.executeNext();
+            if (res.success && res.records) {
+                resolve(res.records);
+            }
+            else {
+                reject(new Error(res.error || "Unknown worker error"));
+            }
+        };
+        const errorHandler = (err) => {
+            cleanup();
+            worker.terminate();
+            const idx = this.workers.indexOf(worker);
+            if (idx !== -1)
+                this.workers.splice(idx, 1);
+            const newWorker = new Worker(this.workerPath);
+            this.workers.push(newWorker);
+            this.freeWorkers.push(newWorker);
+            this.executeNext();
+            reject(err);
+        };
+        const cleanup = () => {
+            worker.off("message", messageHandler);
+            worker.off("error", errorHandler);
+        };
+        worker.on("message", messageHandler);
+        worker.on("error", errorHandler);
+        worker.postMessage(data);
+    }
+    terminateAll() {
+        for (const worker of this.workers) {
+            worker.terminate();
+        }
+        this.workers = [];
+        this.freeWorkers = [];
+        this.queue = [];
+    }
+}
+export const chunkingWorkerPool = new WorkerPool();
+//# sourceMappingURL=WorkerPool.js.map

package/build/utils/worker/WorkerPool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WorkerPool.js","sourceRoot":"","sources":["../../../src/utils/worker/WorkerPool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAQ3C,MAAM,OAAO,UAAU;IACd,OAAO,GAAa,EAAE,CAAC;IACvB,WAAW,GAAa,EAAE,CAAC;IAC3B,KAAK,GAAgB,EAAE,CAAC;IACxB,QAAQ,CAAS;IACjB,UAAU,CAAS;IAE3B,YAAY,QAAiB;QAC5B,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;IAEM,IAAI;QACV,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IAEM,OAAO,CAAC,IAAyB;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3C,IAAI,CAAC,WAAW,EAAE,CAAC;QACpB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,WAAW;QAClB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO;QACR,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;YACzE,OAAO;QACR,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,OAAO;QACR,CAAC;QAED,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAE3C,MAAM,cAAc,GAAG,CAAC,GAAyB,EAAE,EAAE;YACpD,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,WAAW,EAAE,CAAC;YAEnB,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,sBAAsB,CAAC,CAAC,CAAC;YACxD,CAAC;QACF,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,CAAC,GAAU,EAAE,EAAE;YACnC,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,GAAG,KAAK,CAAC,CAAC;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAE5C,MAAM,SAAS,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAEjC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACpB,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACnC,CAAC,CAAC;QAEF,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAEjC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAEM,YAAY;QAClB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,CAAC,SAAS,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;IACjB,CAAC;CACD;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,UAAU,EAAE,CAAC"}

package/build/utils/worker/chunkingWorker.d.ts

@@ -0,0 +1,11 @@
+import type { ChunkMetadata } from "../VectorDB.js";
+export interface ChunkingWorkerInput {
+    filePath: string;
+    fileContent: string;
+    birthTime: string;
+}
+export interface ChunkingWorkerOutput {
+    success: boolean;
+    records?: ChunkMetadata[];
+    error?: string;
+}

package/build/utils/worker/chunkingWorker.js

@@ -0,0 +1,92 @@
+import path from "node:path";
+import { parentPort } from "node:worker_threads";
+import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
+import { encodingForModel } from "js-tiktoken";
+import { parse as parseMatter } from "../processor/MatterParser.js";
+const enc = encodingForModel("gpt-3.5-turbo");
+const splitter = new RecursiveCharacterTextSplitter({
+    chunkSize: 500,
+    chunkOverlap: 50,
+    lengthFunction: (text) => enc.encode(text).length,
+});
+function extractHeadingsWithPositions(body) {
+    const headingRegex = /^(#{1,3})\s+(.+)$/gm;
+    const results = [];
+    const matches = body.matchAll(headingRegex);
+    for (const match of matches) {
+        results.push({
+            heading: match[2].trim(),
+            pos: match.index ?? 0,
+            depth: match[1].length,
+        });
+    }
+    return results;
+}
+function findSectionForChunk(body, chunk, headings) {
+    if (headings.length === 0)
+        return null;
+    const pos = body.indexOf(chunk.slice(0, 60));
+    if (pos === -1)
+        return null;
+    let section = null;
+    for (const h of headings) {
+        if (h.pos <= pos)
+            section = h.heading;
+    }
+    return section;
+}
+parentPort?.on("message", async (data) => {
+    const { filePath, fileContent, birthTime } = data;
+    const fileName = path.basename(filePath);
+    try {
+        const { frontmatter, content: body } = parseMatter(filePath, birthTime, fileContent);
+        const headings = extractHeadingsWithPositions(body);
+        const titleFromBody = headings.find((h) => h.depth === 1)?.heading ?? null;
+        const docTitle = frontmatter.title || titleFromBody || fileName;
+        const docStructure = headings
+            .filter((h) => h.depth <= 2)
+            .slice(0, 4)
+            .map((h) => h.heading.slice(0, 25));
+        const chunks = await splitter.splitText(body);
+        const records = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const sectionHeading = findSectionForChunk(body, chunk, headings);
+            const context = [
+                sectionHeading ? `Section: ${sectionHeading.slice(0, 30)}` : "",
+                docStructure.length > 1
+                    ? `Outline: ${docStructure.slice(0, 3).join(" > ")}`
+                    : "",
+            ]
+                .filter(Boolean)
+                .join("\n");
+            records.push({
+                id: `${filePath}_chunk_${i}`,
+                filePath,
+                fileName,
+                chunkIndex: i,
+                content: chunk,
+                context,
+                metadata: {
+                    title: docTitle,
+                    date: frontmatter.date instanceof Date
+                        ? frontmatter.date.toISOString()
+                        : frontmatter.date || birthTime,
+                    tags: frontmatter.tags?.join(", ") ?? "",
+                    summary: frontmatter.summary || "",
+                    slug: frontmatter.slug || "",
+                    category: frontmatter.category || "any",
+                    completed: frontmatter.completed ?? false,
+                },
+            });
+        }
+        const response = { success: true, records };
+        parentPort?.postMessage(response);
+    }
+    catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        const response = { success: false, error: errorMsg };
+        parentPort?.postMessage(response);
+    }
+});
+//# sourceMappingURL=chunkingWorker.js.map

package/build/utils/worker/chunkingWorker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkingWorker.js","sourceRoot":"","sources":["../../../src/utils/worker/chunkingWorker.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAgBpE,MAAM,GAAG,GAAG,gBAAgB,CAAC,eAAe,CAAC,CAAC;AAC9C,MAAM,QAAQ,GAAG,IAAI,8BAA8B,CAAC;IACnD,SAAS,EAAE,GAAG;IACd,YAAY,EAAE,EAAE;IAChB,cAAc,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;CACzD,CAAC,CAAC;AAEH,SAAS,4BAA4B,CAAC,IAAY;IACjD,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC5C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;YACxB,GAAG,EAAE,KAAK,CAAC,KAAK,IAAI,CAAC;YACrB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;SACtB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAC3B,IAAY,EACZ,KAAa,EACb,QAAwB;IAExB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7C,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;IACvC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,UAAU,EAAE,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,IAAyB,EAAE,EAAE;IAC7D,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEzC,IAAI,CAAC;QACJ,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,WAAW,CACjD,QAAQ,EACR,SAAS,EACT,WAAW,CACX,CAAC;QAEF,MAAM,QAAQ,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC;QAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,IAAI,aAAa,IAAI,QAAQ,CAAC;QAChE,MAAM,YAAY,GAAG,QAAQ;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAErC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAoB,EAAE,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,cAAc,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClE,MAAM,OAAO,GAAG;gBACf,cAAc,CAAC,CAAC,CAAC,YAAY,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;gBAC/D,YAAY,CAAC,MAAM,GAAG,CAAC;oBACtB,CAAC,CAAC,YAAY,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBACpD,CAAC,CAAC,EAAE;aACL;iBACC,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,CAAC;YAEb,OAAO,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,GAAG,QAAQ,UAAU,CAAC,EAAE;gBAC5B,QAAQ;gBACR,QAAQ;gBACR,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,KAAK;gBACd,OAAO;gBACP,QAAQ,EAAE;oBACT,KAAK,EAAE,QAAQ;oBACf,IAAI,EACH,WAAW,CAAC,IAAI,YAAY,IAAI;wBAC/B,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE;wBAChC,CAAC,CAAC,WAAW,CAAC,IAAI,IAAI,SAAS;oBACjC,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACxC,OAAO,EAAE,WAAW,CAAC,OAAO,IAAI,EAAE;oBAClC,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,EAAE;oBAC5B,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,KAAK;oBACvC,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,KAAK;iBACzC;aACD,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAyB,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAClE,UAAU,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAyB,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QAC3E,UAAU,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;AACF,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@sunub/obsidian-mcp-server",
-	"version": "0.3.25",
+	"version": "0.3.27",
 	"description": "A server for the Obsidian Model Context Protocol.",
 	"keywords": [
 		"obsidian",
@@ -36,8 +36,6 @@
 		"check": "npx biome check --write",
 		"cli": "npx tsx src/cli/index.tsx"
 	},
-	"author": "sunub",
-	"license": "ISC",
 	"dependencies": {
 		"@huggingface/transformers": "^4.2.0",
 		"@lancedb/lancedb": "^0.27.2",