@khoinguyen2002/doc-mcp 1.0.3 → 1.0.5

package/dist/db/vector.js

@@ -1,8 +1,17 @@
-import { QdrantClient } from '@qdrant/js-client-rest';
-import { v4 as uuidv4 } from 'uuid';
-import { config } from '../config.js';
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { v4 as uuidv4, v5 as uuidv5 } from "uuid";
+import { config } from "../config.js";
 let client = null;
-const COLLECTION_NAME = 'project_memory';
+const COLLECTION_NAME = "project_memory";
+// Fixed namespace for deterministic point IDs (uuid v5)
+const POINT_NAMESPACE = "1b671a64-40d5-491e-99b0-da01ff1f3341";
+/**
+ * Deterministic Qdrant point ID: uuidv5(fileId:blockIndex, NS)
+ * Same input → same ID → upsert overwrites correctly.
+ */
+export function getBlockPointId(fileId, blockIndex) {
+    return uuidv5(`${fileId}:${blockIndex}`, POINT_NAMESPACE);
+}
 export async function initVectorDB() {
     if (!client) {
         client = new QdrantClient({
@@ -10,83 +19,215 @@ export async function initVectorDB() {
             apiKey: config.QDRANT_API_KEY,
         });
         console.error(`Connected to Qdrant at ${config.QDRANT_URL}`);
-        // Check if collection exists
         const res = await client.getCollections();
-        const exists = res.collections.some(c => c.name === COLLECTION_NAME);
+        const exists = res.collections.some((c) => c.name === COLLECTION_NAME);
         if (!exists) {
             console.error(`Creating Qdrant collection: ${COLLECTION_NAME}`);
             const dummyVector = await embedText("test");
             const dimension = dummyVector.length;
             await client.createCollection(COLLECTION_NAME, {
-                vectors: {
-                    size: dimension,
-                    distance: "Cosine",
-                },
+                vectors: { size: dimension, distance: "Cosine" },
             });
             await client.createPayloadIndex(COLLECTION_NAME, {
-                field_name: "folderId",
+                field_name: "source",
                 field_schema: "keyword",
             });
             await client.createPayloadIndex(COLLECTION_NAME, {
-                field_name: "file_id",
-                field_schema: "keyword",
+                field_name: "block_index",
+                field_schema: "integer",
             });
             await client.createPayloadIndex(COLLECTION_NAME, {
-                field_name: "source",
+                field_name: "block_hash",
                 field_schema: "keyword",
             });
+            // Full-text index on `text` payload for exact/keyword search.
+            // whitespace tokenizer keeps API paths (e.g. /v1/foo/bar) as single tokens.
+            // lowercase=true makes searches case-insensitive.
+            await client.createPayloadIndex(COLLECTION_NAME, {
+                field_name: "text",
+                field_schema: {
+                    type: "text",
+                    tokenizer: "whitespace",
+                    min_token_len: 2,
+                    max_token_len: 200,
+                    lowercase: true,
+                },
+            });
             console.error(`Collection ${COLLECTION_NAME} created with dimension ${dimension}.`);
         }
     }
 }
-export async function embedText(text) {
-    const response = await fetch("https://openrouter.ai/api/v1/embeddings", {
-        method: "POST",
-        headers: {
-            "Authorization": `Bearer ${config.OPENROUTER_API_KEY}`,
-            "Content-Type": "application/json"
-        },
-        body: JSON.stringify({
-            model: config.EMBEDDING_MODEL_ID,
-            input: text
-        })
-    });
-    if (!response.ok) {
-        const errText = await response.text();
-        throw new Error(`OpenRouter Embedding API failed: ${response.status} ${errText}`);
+export async function embedText(text, maxRetries = 5) {
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+        try {
+            const response = await fetch("https://openrouter.ai/api/v1/embeddings", {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${config.OPENROUTER_API_KEY}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    model: config.EMBEDDING_MODEL_ID,
+                    input: text,
+                }),
+            });
+            if (!response.ok) {
+                if (response.status === 429 && attempt < maxRetries - 1) {
+                    const delay = Math.pow(2, attempt) * 1000 + Math.random() * 1000;
+                    console.error(`[Rate Limit] OpenRouter 429. Retrying in ${Math.round(delay)}ms... (Attempt ${attempt + 1}/${maxRetries})`);
+                    await new Promise((res) => setTimeout(res, delay));
+                    continue;
+                }
+                const errText = await response.text();
+                throw new Error(`OpenRouter Embedding API failed: ${response.status} ${errText}`);
+            }
+            const json = await response.json();
+            if (!json.data || !json.data[0] || !json.data[0].embedding) {
+                throw new Error(`Invalid response from OpenRouter: ${JSON.stringify(json)}`);
+            }
+            return json.data[0].embedding;
+        }
+        catch (err) {
+            if (attempt >= maxRetries - 1)
+                throw err;
+            const delay = Math.pow(2, attempt) * 1000 + Math.random() * 1000;
+            console.error(`[Error] ${err.message}. Retrying in ${Math.round(delay)}ms... (Attempt ${attempt + 1}/${maxRetries})`);
+            await new Promise((res) => setTimeout(res, delay));
+        }
     }
-    const json = await response.json();
-    if (!json.data || !json.data[0] || !json.data[0].embedding) {
-        throw new Error("Invalid response from OpenRouter Embedding API");
+    throw new Error("Max retries reached for embedding");
+}
+/**
+ * Embed nhiều texts trong 1 API call (batch).
+ * OpenRouter hỗ trợ input: string[] → trả data[i].embedding.
+ */
+export async function embedBatch(texts, maxRetries = 5) {
+    if (texts.length === 0)
+        return [];
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+        try {
+            const response = await fetch("https://openrouter.ai/api/v1/embeddings", {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${config.OPENROUTER_API_KEY}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    model: config.EMBEDDING_MODEL_ID,
+                    input: texts,
+                }),
+            });
+            if (!response.ok) {
+                if (response.status === 429 && attempt < maxRetries - 1) {
+                    const delay = Math.pow(2, attempt) * 1000 + Math.random() * 1000;
+                    console.error(`[Rate Limit] OpenRouter 429 (batch). Retrying in ${Math.round(delay)}ms... (Attempt ${attempt + 1}/${maxRetries})`);
+                    await new Promise((res) => setTimeout(res, delay));
+                    continue;
+                }
+                const errText = await response.text();
+                throw new Error(`OpenRouter Batch Embedding API failed: ${response.status} ${errText}`);
+            }
+            const json = await response.json();
+            if (!json.data || !Array.isArray(json.data)) {
+                throw new Error(`Invalid batch response from OpenRouter: ${JSON.stringify(json)}`);
+            }
+            return json.data.map((item) => item.embedding);
+        }
+        catch (err) {
+            if (attempt >= maxRetries - 1)
+                throw err;
+            const delay = Math.pow(2, attempt) * 1000 + Math.random() * 1000;
+            console.error(`[Error] ${err.message}. Retrying in ${Math.round(delay)}ms... (Attempt ${attempt + 1}/${maxRetries})`);
+            await new Promise((res) => setTimeout(res, delay));
+        }
     }
-    return json.data[0].embedding;
+    throw new Error("Max retries reached for batch embedding");
 }
-export async function upsertProjectDocument(folderId, text, metadata = {}) {
+/**
+ * Bulk upsert nhiều chunks vào Qdrant trong 1 HTTP call.
+ */
+export async function upsertChunkBatch(chunks) {
     await initVectorDB();
     if (!client)
         throw new Error("Qdrant not initialized");
-    const vector = await embedText(text);
+    if (chunks.length === 0)
+        return;
     await client.upsert(COLLECTION_NAME, {
         wait: true,
-        points: [
-            {
-                id: uuidv4(),
-                vector: vector,
-                payload: {
-                    folderId,
-                    text,
-                    source: metadata.source || "user",
-                    file_id: metadata.file_id || null,
-                    modified_time: metadata.modified_time || null,
-                    metadata: JSON.stringify(metadata),
-                    createdAt: new Date().toISOString()
-                }
-            }
-        ]
+        points: chunks.map((c) => ({
+            id: c.pointId,
+            vector: c.vector,
+            payload: {
+                text: c.text,
+                title: c.title,
+                block_index: c.blockIndex,
+                block_hash: c.blockHash,
+                source: c.source,
+                offset: c.offset,
+            },
+        })),
     });
-    console.error(`Upserted document chunk for folder ${folderId}`);
+    console.error(`Upserted ${chunks.length} chunk(s) to Qdrant.`);
 }
-export async function searchProjectMemory(folderId, query, topK = 3) {
+/**
+ * Fetch block_hash AND offset for a list of point IDs.
+ * Used to diff block-level changes during re-sync (hash) and
+ * detect stale offsets in unchanged blocks (offset).
+ */
+export async function getBlockMetaByIds(pointIds) {
+    await initVectorDB();
+    if (!client || pointIds.length === 0)
+        return {};
+    const results = await client.retrieve(COLLECTION_NAME, {
+        ids: pointIds,
+        with_payload: ["block_hash", "offset"],
+        with_vector: false,
+    });
+    const metaMap = {};
+    for (const point of results) {
+        const hash = point.payload?.block_hash;
+        const offset = point.payload?.offset;
+        if (hash !== undefined) {
+            metaMap[point.id] = { hash, offset: offset ?? 0 };
+        }
+    }
+    return metaMap;
+}
+/**
+ * Update only the `offset` payload field for a set of points (no re-embedding).
+ * Called for unchanged blocks whose character position shifted due to earlier edits.
+ * Uses parallel setPayload calls (lightweight metadata-only updates).
+ */
+export async function updateBlockOffsets(updates) {
+    if (updates.length === 0)
+        return;
+    await initVectorDB();
+    if (!client)
+        throw new Error("Qdrant not initialized");
+    await Promise.all(updates.map(({ pointId, offset }) => client.setPayload(COLLECTION_NAME, {
+        payload: { offset },
+        points: [pointId],
+        wait: false, // fire-and-forget per point; all resolve before function returns
+    })));
+    console.error(`[Sync] Updated offset for ${updates.length} unchanged block(s).`);
+}
+/**
+ * Xóa Qdrant points theo danh sách IDs.
+ */
+export async function deletePointsByIds(pointIds) {
+    await initVectorDB();
+    if (!client || pointIds.length === 0)
+        return;
+    await client.delete(COLLECTION_NAME, {
+        wait: true,
+        points: pointIds,
+    });
+    console.error(`Deleted ${pointIds.length} obsolete block(s) from Qdrant.`);
+}
+/**
+ * Global semantic search — không filter theo folder hay file.
+ */
+export async function searchProjectMemory(query, topK = 3) {
     await initVectorDB();
     if (!client)
         throw new Error("Qdrant not initialized");
@@ -96,22 +237,10 @@ export async function searchProjectMemory(folderId, query, topK = 3) {
             vector: queryVector,
             limit: topK,
             with_payload: true,
-            filter: {
-                must: [
-                    {
-                        key: "folderId",
-                        match: {
-                            value: folderId
-                        }
-                    }
-                ]
-            }
         });
-        // Map to match LanceDB format expected by other tools
-        return results.map(r => ({
+        return results.map((r) => ({
             id: r.id,
-            vector: r.vector,
-            ...r.payload
+            ...r.payload,
         }));
     }
     catch (err) {
@@ -119,54 +248,65 @@ export async function searchProjectMemory(folderId, query, topK = 3) {
         return [];
     }
 }
-export async function deleteProjectDocument(folderId, fileId) {
-    await initVectorDB();
-    if (!client)
-        return;
-    await client.delete(COLLECTION_NAME, {
-        filter: {
-            must: [
-                { key: "folderId", match: { value: folderId } },
-                { key: "file_id", match: { value: fileId } }
-            ]
-        }
-    });
-    console.error(`Deleted old chunks from Qdrant for ${folderId} / ${fileId}`);
-}
-export async function checkProjectDocumentExists(folderId, fileId) {
+/**
+ * Exhaustive full-text search using Qdrant's inverted index on the `text` field.
+ * Uses whitespace tokenizer → API paths like /v1/foo/bar match as single tokens.
+ * Paginates through all results server-side (no full collection scan in JS).
+ */
+export async function exactSearchChunks(term, limit = 50) {
     await initVectorDB();
     if (!client)
-        return false;
-    const res = await client.count(COLLECTION_NAME, {
-        filter: {
-            must: [
-                { key: "folderId", match: { value: folderId } },
-                { key: "file_id", match: { value: fileId } }
-            ]
+        throw new Error("Qdrant not initialized");
+    const filter = {
+        must: [
+            {
+                key: "text",
+                match: { text: term.toLowerCase() },
+            },
+        ],
+    };
+    const results = [];
+    let offset = undefined;
+    // Paginate until all matching points are collected or limit is reached
+    do {
+        const page = await client.scroll(COLLECTION_NAME, {
+            filter,
+            with_payload: true,
+            with_vector: false,
+            limit: Math.min(100, limit - results.length),
+            ...(offset !== undefined ? { offset } : {}),
+        });
+        for (const point of page.points) {
+            results.push({ id: point.id, ...point.payload });
         }
-    });
-    return res.count > 0;
+        offset = page.next_page_offset;
+    } while (offset != null && results.length < limit);
+    return results;
 }
-export async function getProjectDocumentMetadata(folderId) {
+/**
+ * Upsert agent note với random UUID (không có fileId).
+ */
+export async function upsertAgentNote(text) {
     await initVectorDB();
     if (!client)
-        return {};
-    const res = await client.scroll(COLLECTION_NAME, {
-        filter: {
-            must: [
-                { key: "folderId", match: { value: folderId } },
-                { key: "source", match: { value: "google_drive" } }
-            ]
-        },
-        limit: 10000,
-        with_payload: ["file_id", "modified_time"],
-        with_vector: false
+        throw new Error("Qdrant not initialized");
+    const vector = await embedText(text);
+    await client.upsert(COLLECTION_NAME, {
+        wait: true,
+        points: [
+            {
+                id: uuidv4(),
+                vector,
+                payload: {
+                    text,
+                    title: "Agent Note",
+                    block_index: 0,
+                    block_hash: "",
+                    source: "agent",
+                    offset: 0,
+                },
+            },
+        ],
     });
-    const fileMap = {};
-    for (const r of res.points) {
-        if (r.payload && r.payload.file_id && r.payload.modified_time) {
-            fileMap[r.payload.file_id] = r.payload.modified_time;
-        }
-    }
-    return fileMap;
+    console.error("Upserted agent note to Qdrant.");
 }

package/dist/mcp-server.js

@@ -2,33 +2,22 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
-import { listDriveFiles, readDriveDocument, } from "./tools/driveTools.js";
-import { saveAgentNote, searchKnowledge } from "./tools/knowledgeTools.js";
-import { config } from "./config.js";
-const DRIVE_FOLDER_ID = config.DOC_MCP_DRIVE_FOLDER_ID;
-if (!DRIVE_FOLDER_ID) {
-    console.error("Missing DOC_MCP_DRIVE_FOLDER_ID environment variable. The doc-agent requires a target folder ID.");
-    process.exit(1);
-}
+import { listDriveFiles, readDriveDocument } from "./tools/driveTools.js";
+import { searchKnowledge, searchExact } from "./tools/knowledgeTools.js";
 const server = new McpServer({
     name: "doc-agent",
-    version: "1.0.0",
+    version: "1.2.0",
 });
-// Register tools
 server.registerTool("list_drive_files", {
-    description: "List and search for Google Drive documents and subfolders in a specific folder.",
+    description: "List all Google Drive documents accessible to this agent. Returns file IDs, names, and types. Use keyword to filter by title.",
     inputSchema: {
         keyword: z
             .string()
             .optional()
-            .describe("Optional keyword to search for in document titles"),
-        targetFolderId: z
-            .string()
-            .optional()
-            .describe("Optional Google Drive folder ID to list contents from. Defaults to the root knowledge folder."),
+            .describe("Optional keyword to filter documents by title"),
     },
-}, async ({ keyword, targetFolderId }) => {
-    const res = await listDriveFiles(keyword, targetFolderId);
+}, async ({ keyword }) => {
+    const res = await listDriveFiles(keyword);
     if (!res.success) {
         return {
             content: [{ type: "text", text: `Error: ${res.error}` }],
@@ -40,11 +29,17 @@ server.registerTool("list_drive_files", {
     };
 });
 server.registerTool("read_drive_document", {
-    description: "Read the content of a specific Google Drive document. The document will also be automatically ingested into vector memory for future semantic search.",
+    description: "Read the Markdown content of a specific Google Drive document. Automatically syncs the latest version. Use 'offset' (from search_knowledge results) to navigate to a specific section, and 'limit' to control how much content to return.",
     inputSchema: {
         fileId: z.string().describe("The Google Drive file ID to read"),
-        offset: z.number().optional().describe("Starting character index (default: 0)"),
-        limit: z.number().optional().describe("Maximum number of characters to return (default: 10000)"),
+        offset: z
+            .number()
+            .optional()
+            .describe("Starting character index in the Markdown content (default: 0)"),
+        limit: z
+            .number()
+            .optional()
+            .describe("Maximum characters to return (default: 10000)"),
     },
 }, async ({ fileId, offset, limit }) => {
     const res = await readDriveDocument(fileId, offset, limit);
@@ -58,13 +53,17 @@ server.registerTool("read_drive_document", {
         content: [{ type: "text", text: JSON.stringify(res.data, null, 2) }],
     };
 });
-server.registerTool("save_agent_note", {
-    description: "Save an agent note, thought, or summary directly into the vector memory.",
+server.registerTool("search_knowledge", {
+    description: "Semantic vector search across all accessible Google Drive documents. Automatically syncs latest document changes before searching. Returns relevant Markdown chunks with title and character offset.",
     inputSchema: {
-        content: z.string().describe("The note or knowledge content to store"),
+        query: z.string().describe("The search query"),
+        topK: z
+            .number()
+            .optional()
+            .describe("Number of results to return (default: 3)"),
     },
-}, async ({ content }) => {
-    const res = await saveAgentNote(content);
+}, async ({ query, topK }) => {
+    const res = await searchKnowledge(query, topK);
     if (!res.success) {
         return {
             content: [{ type: "text", text: `Error: ${res.error}` }],
@@ -72,20 +71,32 @@ server.registerTool("save_agent_note", {
         };
     }
     return {
-        content: [{ type: "text", text: res.message || "Saved successfully" }],
+        content: [
+            {
+                type: "text",
+                text: typeof res.results === "string"
+                    ? res.results
+                    : JSON.stringify(res.results),
+            },
+        ],
     };
 });
-server.registerTool("search_knowledge", {
-    description: "Search the folder's vector memory for relevant context or knowledge.",
+server.registerTool("search_exact", {
+    description: "Exhaustive keyword search across all accessible Google Drive documents using full-text index. " +
+        "Unlike search_knowledge (semantic/vector), this finds EVERY chunk containing the exact term — " +
+        "ideal for specific identifiers: API paths (/v1/foo/bar), function names, config keys, error codes. " +
+        "Case-insensitive. Automatically syncs latest document changes before searching.",
     inputSchema: {
-        query: z.string().describe("The search query"),
-        topK: z
+        term: z
+            .string()
+            .describe("Exact term to search for (e.g. '/product-orchestrator/v1/products/filter', 'ServiceCode.mkp')"),
+        limit: z
             .number()
             .optional()
-            .describe("Number of results to return (default: 3)"),
+            .describe("Max results to return (default: 50)"),
     },
-}, async ({ query, topK }) => {
-    const res = await searchKnowledge(query, topK);
+}, async ({ term, limit }) => {
+    const res = await searchExact(term, limit);
     if (!res.success) {
         return {
             content: [{ type: "text", text: `Error: ${res.error}` }],
@@ -98,16 +109,15 @@ server.registerTool("search_knowledge", {
                 type: "text",
                 text: typeof res.results === "string"
                     ? res.results
-                    : JSON.stringify(res.results),
+                    : JSON.stringify(res, null, 2),
             },
         ],
     };
 });
-// Start the server
 async function run() {
     const transport = new StdioServerTransport();
     await server.connect(transport);
-    console.error("doc-agent MCP server running on stdio");
+    console.error("doc-agent MCP server v1.2.0 running on stdio");
 }
 run().catch((error) => {
     console.error("Fatal error running server:", error);

package/dist/tools/driveTools.d.ts

@@ -1,21 +1,32 @@
-export declare function listDriveFiles(keyword?: string, targetFolderId?: string): Promise<{
+/**
+ * List all Google Docs the Service Account can read.
+ * Optional keyword filter on document title.
+ */
+export declare function listDriveFiles(keyword?: string): Promise<{
     success: boolean;
-    results: import("googleapis").drive_v3.Schema$File[];
+    results: any[];
     error?: undefined;
 } | {
     success: boolean;
     error: any;
     results?: undefined;
 }>;
-export declare function syncSingleDocument(fileId: string, folderId: string): Promise<{
-    synced: boolean;
-    content: string;
-    driveModifiedTime: string;
+/**
+ * Sync all documents the SA can see:
+ * - New/changed files → syncSingleDocument()
+ * - Files removed from Drive → delete from Qdrant + Redis
+ */
+export declare function syncAllDocuments(): Promise<{
+    success: boolean;
+    error?: undefined;
 } | {
-    synced: boolean;
-    driveModifiedTime: string;
-    content?: undefined;
+    success: boolean;
+    error: any;
 }>;
+/**
+ * Read a specific Google Drive document, triggering incremental sync first.
+ * Returns paginated Markdown content.
+ */
 export declare function readDriveDocument(fileId: string, offset?: number, limit?: number): Promise<{
     success: boolean;
     data: {
@@ -34,11 +45,4 @@ export declare function readDriveDocument(fileId: string, offset?: number, limit
     error: any;
     data?: undefined;
 }>;
-export declare function syncFolderState(folderId: string): Promise<{
-    success: boolean;
-    error?: undefined;
-} | {
-    success: boolean;
-    error: any;
-}>;
 //# sourceMappingURL=driveTools.d.ts.map

package/dist/tools/driveTools.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driveTools.d.ts","sourceRoot":"","sources":["../../src/tools/driveTools.ts"],"names":[],"mappings":"AAiCA,wBAAsB,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM;;;;;;;;GAoC7E;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;;;;;;;;GA6CxE;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,GAAE,MAAU,EAAE,KAAK,GAAE,MAAc;;;;;;;;;;;;;;;;;GAsDhG;AAED,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM;;;;;;GAqDrD"}
+{"version":3,"file":"driveTools.d.ts","sourceRoot":"","sources":["../../src/tools/driveTools.ts"],"names":[],"mappings":"AA4BA;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM;;;;;;;;GA8BpD;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB;;;;;;GAwDrC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,MAAU,EAClB,KAAK,GAAE,MAAc;;;;;;;;;;;;;;;;;GAoCtB"}