@khoinguyen2002/doc-mcp 1.0.4 → 1.0.5

package/dist/config.d.ts

@@ -2,12 +2,14 @@ export declare const config: {
     QDRANT_URL: string;
     OPENROUTER_API_KEY: string;
     EMBEDDING_MODEL_ID: string;
-    CHUNK_SIZE: number;
-    CHUNK_OVERLAP: number;
-    DOC_MCP_DRIVE_FOLDER_ID?: string | undefined;
+    MAX_CHUNK_SIZE: number;
+    EMBEDDING_MAX_TOKENS: number;
+    EMBEDDING_RPM: number;
+    UPSTASH_REDIS_REST_URL: string;
+    UPSTASH_REDIS_REST_TOKEN: string;
     DOC_MCP_GOOGLE_CLIENT_EMAIL?: string | undefined;
     DOC_MCP_GOOGLE_PRIVATE_KEY?: string | undefined;
     QDRANT_API_KEY?: string | undefined;
+    VISION_MODEL_ID?: string | undefined;
 };
-export type Config = typeof config;
 //# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AA2BA,eAAO,MAAM,MAAM;;;;;;;;;;CAAe,CAAC;AACnC,MAAM,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AA+CA,eAAO,MAAM,MAAM;;;;;;;;;;;;;CAAe,CAAC"}

package/dist/config.js

@@ -1,15 +1,30 @@
 import { z } from "zod";
 const schema = z.object({
-    DOC_MCP_DRIVE_FOLDER_ID: z.string().optional(),
     DOC_MCP_GOOGLE_CLIENT_EMAIL: z.string().email().optional(),
     DOC_MCP_GOOGLE_PRIVATE_KEY: z.string().optional(),
-    // Vector DB / Embeddings
+    // Vector DB
     QDRANT_URL: z.string().url().describe("The URL of your Qdrant instance"),
-    QDRANT_API_KEY: z.string().optional().describe("API Key for Qdrant Cloud (optional for local)"),
+    QDRANT_API_KEY: z
+        .string()
+        .optional()
+        .describe("API Key for Qdrant Cloud (optional for local)"),
+    // Embeddings
     OPENROUTER_API_KEY: z.string().min(1),
-    EMBEDDING_MODEL_ID: z.string().default("nvidia/llama-nemotron-embed-vl-1b-v2:free"),
-    CHUNK_SIZE: z.coerce.number().int().positive().default(4000),
-    CHUNK_OVERLAP: z.coerce.number().int().nonnegative().default(500),
+    EMBEDDING_MODEL_ID: z
+        .string()
+        .default("nvidia/llama-nemotron-embed-vl-1b-v2:free"),
+    // Max chunk size in Markdown chars — system may use a smaller value if
+    // the embedding model's token budget requires it (see ingestFlow.ts)
+    MAX_CHUNK_SIZE: z.coerce.number().int().positive().default(3000),
+    // Max tokens per embedding API call (for batch packing)
+    EMBEDDING_MAX_TOKENS: z.coerce.number().int().positive().default(32000),
+    // Max embedding API requests per minute
+    EMBEDDING_RPM: z.coerce.number().int().positive().default(40),
+    // Vision LLM model ID for image descriptions (optional, skip if not set)
+    VISION_MODEL_ID: z.string().optional(),
+    // Upstash Redis (for sync state)
+    UPSTASH_REDIS_REST_URL: z.string().url(),
+    UPSTASH_REDIS_REST_TOKEN: z.string().min(1),
 });
 function loadConfig() {
     const result = schema.safeParse(process.env);
@@ -17,7 +32,7 @@ function loadConfig() {
         const missing = result.error.issues
             .map((i) => `  ${i.path.join(".")}: ${i.message}`)
             .join("\n");
-        throw new Error(`Invalid environment configuration for doc-mcp:\n${missing}`);
+        throw new Error(`doc-mcp configuration error:\n${missing}\n\nPlease check your environment variables.`);
     }
     return result.data;
 }

package/dist/db/rateLimiter.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Call này trước mỗi embedBatch() để đảm bảo không vượt EMBEDDING_RPM.
+ * Sliding window đơn giản: enforce minimum gap = 60000 / RPM giữa các lần gọi.
+ */
+export declare function waitForRateLimit(): Promise<void>;
+//# sourceMappingURL=rateLimiter.d.ts.map

package/dist/db/rateLimiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rateLimiter.d.ts","sourceRoot":"","sources":["../../src/db/rateLimiter.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAWtD"}

package/dist/db/rateLimiter.js

@@ -0,0 +1,20 @@
+import { config } from "../config.js";
+const MIN_GAP_MS = Math.ceil(60000 / config.EMBEDDING_RPM);
+let lastCallTime = 0;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Call này trước mỗi embedBatch() để đảm bảo không vượt EMBEDDING_RPM.
+ * Sliding window đơn giản: enforce minimum gap = 60000 / RPM giữa các lần gọi.
+ */
+export async function waitForRateLimit() {
+    const now = Date.now();
+    const elapsed = now - lastCallTime;
+    if (elapsed < MIN_GAP_MS) {
+        const waitMs = MIN_GAP_MS - elapsed;
+        console.error(`[RateLimit] Waiting ${waitMs}ms before next embedding call...`);
+        await sleep(waitMs);
+    }
+    lastCallTime = Date.now();
+}

package/dist/db/syncState.d.ts

@@ -0,0 +1,12 @@
+export interface SyncEntry {
+    modifiedTime: string;
+    blockCount: number;
+    title: string;
+}
+export declare function getAllSyncEntries(): Promise<Record<string, SyncEntry>>;
+export declare function getSyncEntry(fileId: string): Promise<SyncEntry | null>;
+export declare function setSyncEntry(fileId: string, entry: SyncEntry): Promise<void>;
+export declare function deleteSyncEntry(fileId: string): Promise<void>;
+export declare function getImageDesc(imageHash: string): Promise<string | null>;
+export declare function setImageDesc(imageHash: string, description: string): Promise<void>;
+//# sourceMappingURL=syncState.d.ts.map

package/dist/db/syncState.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"syncState.d.ts","sourceRoot":"","sources":["../../src/db/syncState.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAkB5E;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAW5E;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC,CAGf;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGnE;AAOD,wBAAsB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAI5E;AAED,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAGf"}

package/dist/db/syncState.js

@@ -0,0 +1,69 @@
+import { Redis } from "@upstash/redis";
+import { config } from "../config.js";
+const HASH_KEY = "doc_sync_state";
+let _redis = null;
+function getRedis() {
+    if (!_redis) {
+        _redis = new Redis({
+            url: config.UPSTASH_REDIS_REST_URL,
+            token: config.UPSTASH_REDIS_REST_TOKEN,
+        });
+    }
+    return _redis;
+}
+export async function getAllSyncEntries() {
+    const redis = getRedis();
+    const raw = await redis.hgetall(HASH_KEY);
+    if (!raw)
+        return {};
+    const result = {};
+    for (const [fileId, value] of Object.entries(raw)) {
+        if (!value)
+            continue;
+        try {
+            result[fileId] =
+                typeof value === "string"
+                    ? JSON.parse(value)
+                    : value;
+        }
+        catch {
+            // skip malformed entries
+        }
+    }
+    return result;
+}
+export async function getSyncEntry(fileId) {
+    const redis = getRedis();
+    const raw = await redis.hget(HASH_KEY, fileId);
+    if (!raw)
+        return null;
+    try {
+        return typeof raw === "string"
+            ? JSON.parse(raw)
+            : raw;
+    }
+    catch {
+        return null;
+    }
+}
+export async function setSyncEntry(fileId, entry) {
+    const redis = getRedis();
+    await redis.hset(HASH_KEY, { [fileId]: JSON.stringify(entry) });
+}
+export async function deleteSyncEntry(fileId) {
+    const redis = getRedis();
+    await redis.hdel(HASH_KEY, fileId);
+}
+// ─── Image Description Cache ──────────────────────────────────────────────────
+// Global hash: md5(imageBinary) → description text
+// Deduplicates across docs (same image used in multiple files reuses description)
+const IMG_DESC_KEY = "img_desc";
+export async function getImageDesc(imageHash) {
+    const redis = getRedis();
+    const raw = await redis.hget(IMG_DESC_KEY, imageHash);
+    return raw ? String(raw) : null;
+}
+export async function setImageDesc(imageHash, description) {
+    const redis = getRedis();
+    await redis.hset(IMG_DESC_KEY, { [imageHash]: description });
+}

package/dist/db/vector.d.ts

@@ -1,8 +1,63 @@
+/**
+ * Deterministic Qdrant point ID: uuidv5(fileId:blockIndex, NS)
+ * Same input → same ID → upsert overwrites correctly.
+ */
+export declare function getBlockPointId(fileId: string, blockIndex: number): string;
 export declare function initVectorDB(): Promise<void>;
-export declare function embedText(text: string): Promise<number[]>;
-export declare function upsertProjectDocument(folderId: string, text: string, metadata?: Record<string, any>): Promise<void>;
-export declare function searchProjectMemory(folderId: string, query: string, topK?: number): Promise<any[]>;
-export declare function deleteProjectDocument(folderId: string, fileId: string): Promise<void>;
-export declare function checkProjectDocumentExists(folderId: string, fileId: string): Promise<boolean>;
-export declare function getProjectDocumentMetadata(folderId: string): Promise<Record<string, string>>;
+export declare function embedText(text: string, maxRetries?: number): Promise<number[]>;
+/**
+ * Embed nhiều texts trong 1 API call (batch).
+ * OpenRouter hỗ trợ input: string[] → trả data[i].embedding.
+ */
+export declare function embedBatch(texts: string[], maxRetries?: number): Promise<number[][]>;
+export interface ChunkUpsert {
+    pointId: string;
+    vector: number[];
+    text: string;
+    title: string;
+    blockIndex: number;
+    blockHash: string;
+    source: string;
+    offset: number;
+}
+/**
+ * Bulk upsert nhiều chunks vào Qdrant trong 1 HTTP call.
+ */
+export declare function upsertChunkBatch(chunks: ChunkUpsert[]): Promise<void>;
+/**
+ * Fetch block_hash AND offset for a list of point IDs.
+ * Used to diff block-level changes during re-sync (hash) and
+ * detect stale offsets in unchanged blocks (offset).
+ */
+export declare function getBlockMetaByIds(pointIds: string[]): Promise<Record<string, {
+    hash: string;
+    offset: number;
+}>>;
+/**
+ * Update only the `offset` payload field for a set of points (no re-embedding).
+ * Called for unchanged blocks whose character position shifted due to earlier edits.
+ * Uses parallel setPayload calls (lightweight metadata-only updates).
+ */
+export declare function updateBlockOffsets(updates: {
+    pointId: string;
+    offset: number;
+}[]): Promise<void>;
+/**
+ * Xóa Qdrant points theo danh sách IDs.
+ */
+export declare function deletePointsByIds(pointIds: string[]): Promise<void>;
+/**
+ * Global semantic search — không filter theo folder hay file.
+ */
+export declare function searchProjectMemory(query: string, topK?: number): Promise<any[]>;
+/**
+ * Exhaustive full-text search using Qdrant's inverted index on the `text` field.
+ * Uses whitespace tokenizer → API paths like /v1/foo/bar match as single tokens.
+ * Paginates through all results server-side (no full collection scan in JS).
+ */
+export declare function exactSearchChunks(term: string, limit?: number): Promise<any[]>;
+/**
+ * Upsert agent note với random UUID (không có fileId).
+ */
+export declare function upsertAgentNote(text: string): Promise<void>;
 //# sourceMappingURL=vector.d.ts.map

package/dist/db/vector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../src/db/vector.ts"],"names":[],"mappings":"AAOA,wBAAsB,YAAY,kBAqCjC;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAwB/D;AAED,wBAAsB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B7H;AAED,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAiC3G;AAED,wBAAsB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAa3F;AAED,wBAAsB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAanG;AAED,wBAAsB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAwBlG"}
+{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../src/db/vector.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAE1E;AAED,wBAAsB,YAAY,kBAgDjC;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,EAAE,CAAC,CA+CnB;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,UAAU,SAAI,GACb,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAiDrB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB3E;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAAE,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAmB3D;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,IAAI,CAAC,CAef;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CASzE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,MAAU,GACf,OAAO,CAAC,GAAG,EAAE,CAAC,CAoBhB;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,KAAK,GAAE,MAAW,GACjB,OAAO,CAAC,GAAG,EAAE,CAAC,CAkChB;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBjE"}