mcpscraper-memory-db 1.0.0

package/dist/rag/embedder.js

@@ -0,0 +1,93 @@
+const EMBED_DIM = Number((process.env.JINA_EMBED_DIM ?? "1024").trim());
+const modelId = (process.env.JINA_EMBED_MODEL ?? "jina-embeddings-v5-omni-small").trim();
+const MAX_ATTEMPTS = 5;
+function buildKeyPool() {
+  const env = process.env;
+  const keys = Array.from({ length: 5 }, (_, index) => {
+    const keyName = index === 0 ? "JINA_API_KEY" : `JINA_API_KEY_${index + 1}`;
+    return env[keyName]?.trim();
+  });
+  const pool = [];
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    if (!key) continue;
+    const weight = i === 2 ? 5 : 1;
+    for (let w = 0; w < weight; w++) pool.push(key);
+  }
+  return pool;
+}
+let _pool = null;
+function getKeyPool() {
+  if (!_pool) _pool = buildKeyPool();
+  return _pool;
+}
+let _keyIndex = 0;
+function nextKey(pool) {
+  const key = pool[_keyIndex % pool.length];
+  _keyIndex++;
+  return key;
+}
+const jinaEmbedder = {
+  specificationVersion: "v2",
+  provider: "jina",
+  modelId,
+  maxEmbeddingsPerCall: 100,
+  supportsParallelCalls: false,
+  async doEmbed({ values, abortSignal, headers }) {
+    const pool = getKeyPool();
+    if (pool.length === 0) {
+      throw new Error("No JINA_API_KEY set (checked JINA_API_KEY through JINA_API_KEY_5)");
+    }
+    let lastError;
+    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+      if (abortSignal?.aborted) {
+        throw abortSignal.reason instanceof Error ? abortSignal.reason : new DOMException("Aborted", "AbortError");
+      }
+      const apiKey = nextKey(pool);
+      const requestHeaders = {
+        "content-type": "application/json",
+        authorization: `Bearer ${apiKey}`
+      };
+      if (headers) {
+        for (const [key, value] of Object.entries(headers)) {
+          if (value !== void 0) requestHeaders[key] = value;
+        }
+      }
+      try {
+        const response = await fetch("https://api.jina.ai/v1/embeddings", {
+          method: "POST",
+          headers: requestHeaders,
+          body: JSON.stringify({
+            model: modelId,
+            input: values,
+            dimensions: EMBED_DIM,
+            task: "retrieval.passage"
+          }),
+          signal: abortSignal
+        });
+        if ((response.status === 429 || response.status === 524) && attempt < MAX_ATTEMPTS - 1) {
+          lastError = new Error(`Jina embeddings request failed: ${response.status} ${response.statusText}`);
+          continue;
+        }
+        if (!response.ok) {
+          throw new Error(`Jina embeddings request failed: ${response.status} ${response.statusText}`);
+        }
+        const data = await response.json();
+        return {
+          embeddings: data.data.map((d) => d.embedding),
+          usage: { tokens: data.usage?.total_tokens ?? 0 }
+        };
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") throw err;
+        lastError = err;
+        if (attempt < MAX_ATTEMPTS - 1) continue;
+      }
+    }
+    throw lastError instanceof Error ? lastError : new Error("Jina embeddings request failed after retries");
+  }
+};
+export {
+  EMBED_DIM,
+  jinaEmbedder
+};
+//# sourceMappingURL=embedder.js.map

package/dist/rag/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/rag/embedder.ts"],"sourcesContent":["import type { EmbeddingModelV2 } from '@ai-sdk/provider'\n\nexport const EMBED_DIM = Number((process.env.JINA_EMBED_DIM ?? '1024').trim())\n\nconst modelId = (process.env.JINA_EMBED_MODEL ?? 'jina-embeddings-v5-omni-small').trim()\n\nconst MAX_ATTEMPTS = 5\n\ninterface JinaEmbeddingResponse {\n  data: { embedding: number[] }[]\n  usage?: { total_tokens?: number }\n}\n\nfunction buildKeyPool(): string[] {\n  const env = process.env\n  const keys = Array.from({ length: 5 }, (_, index) => {\n    const keyName = index === 0 ? 'JINA_API_KEY' : `JINA_API_KEY_${index + 1}`\n    return env[keyName]?.trim()\n  })\n  const pool: string[] = []\n  for (let i = 0; i < keys.length; i++) {\n    const key = keys[i]\n    if (!key) continue\n    const weight = i === 2 ? 5 : 1\n    for (let w = 0; w < weight; w++) pool.push(key)\n  }\n  return pool\n}\n\nlet _pool: string[] | null = null\nfunction getKeyPool(): string[] {\n  if (!_pool) _pool = buildKeyPool()\n  return _pool\n}\n\nlet _keyIndex = 0\nfunction nextKey(pool: string[]): string {\n  const key = pool[_keyIndex % pool.length]\n  _keyIndex++\n  return key\n}\n\nexport const jinaEmbedder: EmbeddingModelV2<string> = {\n  specificationVersion: 'v2',\n  provider: 'jina',\n  modelId,\n  maxEmbeddingsPerCall: 100,\n  supportsParallelCalls: false,\n  async doEmbed({ values, abortSignal, headers }) {\n    const pool = getKeyPool()\n    if (pool.length === 0) {\n      throw new Error('No JINA_API_KEY set (checked JINA_API_KEY through JINA_API_KEY_5)')\n    }\n\n    let lastError: unknown\n\n    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n      if (abortSignal?.aborted) {\n        throw abortSignal.reason instanceof Error ? abortSignal.reason : new DOMException('Aborted', 'AbortError')\n      }\n\n      const apiKey = nextKey(pool)\n      const requestHeaders: Record<string, string> = {\n        'content-type': 'application/json',\n        authorization: `Bearer ${apiKey}`,\n      }\n      if (headers) {\n        for (const [key, value] of Object.entries(headers)) {\n          if (value !== undefined) requestHeaders[key] = value\n        }\n      }\n\n      try {\n        const response = await fetch('https://api.jina.ai/v1/embeddings', {\n          method: 'POST',\n          headers: requestHeaders,\n          body: JSON.stringify({\n            model: modelId,\n            input: values,\n            dimensions: EMBED_DIM,\n            task: 'retrieval.passage',\n          }),\n          signal: abortSignal,\n        })\n\n        if ((response.status === 429 || response.status === 524) && attempt < MAX_ATTEMPTS - 1) {\n          lastError = new Error(`Jina embeddings request failed: ${response.status} ${response.statusText}`)\n          continue\n        }\n\n        if (!response.ok) {\n          throw new Error(`Jina embeddings request failed: ${response.status} ${response.statusText}`)\n        }\n\n        const data = (await response.json()) as JinaEmbeddingResponse\n\n        return {\n          embeddings: data.data.map((d) => d.embedding),\n          usage: { tokens: data.usage?.total_tokens ?? 0 },\n        }\n      } catch (err) {\n        if (err instanceof Error && err.name === 'AbortError') throw err\n        lastError = err\n        if (attempt < MAX_ATTEMPTS - 1) continue\n      }\n    }\n\n    throw lastError instanceof Error ? lastError : new Error('Jina embeddings request failed after retries')\n  },\n}\n"],"mappings":"AAEO,MAAM,YAAY,QAAQ,QAAQ,IAAI,kBAAkB,QAAQ,KAAK,CAAC;AAE7E,MAAM,WAAW,QAAQ,IAAI,oBAAoB,iCAAiC,KAAK;AAEvF,MAAM,eAAe;AAOrB,SAAS,eAAyB;AAChC,QAAM,MAAM,QAAQ;AACpB,QAAM,OAAO,MAAM,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,UAAU;AACnD,UAAM,UAAU,UAAU,IAAI,iBAAiB,gBAAgB,QAAQ,CAAC;AACxE,WAAO,IAAI,OAAO,GAAG,KAAK;AAAA,EAC5B,CAAC;AACD,QAAM,OAAiB,CAAC;AACxB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,CAAC,IAAK;AACV,UAAM,SAAS,MAAM,IAAI,IAAI;AAC7B,aAAS,IAAI,GAAG,IAAI,QAAQ,IAAK,MAAK,KAAK,GAAG;AAAA,EAChD;AACA,SAAO;AACT;AAEA,IAAI,QAAyB;AAC7B,SAAS,aAAuB;AAC9B,MAAI,CAAC,MAAO,SAAQ,aAAa;AACjC,SAAO;AACT;AAEA,IAAI,YAAY;AAChB,SAAS,QAAQ,MAAwB;AACvC,QAAM,MAAM,KAAK,YAAY,KAAK,MAAM;AACxC;AACA,SAAO;AACT;AAEO,MAAM,eAAyC;AAAA,EACpD,sBAAsB;AAAA,EACtB,UAAU;AAAA,EACV;AAAA,EACA,sBAAsB;AAAA,EACtB,uBAAuB;AAAA,EACvB,MAAM,QAAQ,EAAE,QAAQ,aAAa,QAAQ,GAAG;AAC9C,UAAM,OAAO,WAAW;AACxB,QAAI,KAAK,WAAW,GAAG;AACrB,YAAM,IAAI,MAAM,mEAAmE;AAAA,IACrF;AAEA,QAAI;AAEJ,aAAS,UAAU,GAAG,UAAU,cAAc,WAAW;AACvD,UAAI,aAAa,SAAS;AACxB,cAAM,YAAY,kBAAkB,QAAQ,YAAY,SAAS,IAAI,aAAa,WAAW,YAAY;AAAA,MAC3G;AAEA,YAAM,SAAS,QAAQ,IAAI;AAC3B,YAAM,iBAAyC;AAAA,QAC7C,gBAAgB;AAAA,QAChB,eAAe,UAAU,MAAM;AAAA,MACjC;AACA,UAAI,SAAS;AACX,mBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,cAAI,UAAU,OAAW,gBAAe,GAAG,IAAI;AAAA,QACjD;AAAA,MACF;AAEA,UAAI;AACF,cAAM,WAAW,MAAM,MAAM,qCAAqC;AAAA,UAChE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,MAAM,KAAK,UAAU;AAAA,YACnB,OAAO;AAAA,YACP,OAAO;AAAA,YACP,YAAY;AAAA,YACZ,MAAM;AAAA,UACR,CAAC;AAAA,UACD,QAAQ;AAAA,QACV,CAAC;AAED,aAAK,SAAS,WAAW,OAAO,SAAS,WAAW,QAAQ,UAAU,eAAe,GAAG;AACtF,sBAAY,IAAI,MAAM,mCAAmC,SAAS,MAAM,IAAI,SAAS,UAAU,EAAE;AACjG;AAAA,QACF;AAEA,YAAI,CAAC,SAAS,IAAI;AAChB,gBAAM,IAAI,MAAM,mCAAmC,SAAS,MAAM,IAAI,SAAS,UAAU,EAAE;AAAA,QAC7F;AAEA,cAAM,OAAQ,MAAM,SAAS,KAAK;AAElC,eAAO;AAAA,UACL,YAAY,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,SAAS;AAAA,UAC5C,OAAO,EAAE,QAAQ,KAAK,OAAO,gBAAgB,EAAE;AAAA,QACjD;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,eAAe,SAAS,IAAI,SAAS,aAAc,OAAM;AAC7D,oBAAY;AACZ,YAAI,UAAU,eAAe,EAAG;AAAA,MAClC;AAAA,IACF;AAEA,UAAM,qBAAqB,QAAQ,YAAY,IAAI,MAAM,8CAA8C;AAAA,EACzG;AACF;","names":[]}

package/dist/rag/index-pipeline.d.ts

@@ -0,0 +1,22 @@
+declare function indexDocument(args: {
+    vault: string;
+    id: string;
+    text: string;
+    source: string;
+    capturedAt?: string;
+    identity?: string;
+    op?: string;
+}): Promise<{
+    chunks: number;
+    tokens: number;
+    costUsd: number;
+}>;
+declare function removeDocument(args: {
+    vault: string;
+    id: string;
+}): Promise<void>;
+declare function removeVaultVectors(args: {
+    vault: string;
+}): Promise<void>;
+
+export { indexDocument, removeDocument, removeVaultVectors };

package/dist/rag/index-pipeline.js

@@ -0,0 +1,62 @@
+import { MDocument } from "@mastra/rag";
+import { embedMany } from "ai";
+import { createHash } from "node:crypto";
+import { jinaEmbedder } from "./embedder";
+import { memoryVectors, INDEX_NAME, ensureIndex } from "./vector-store";
+import { recordCost } from "../db/usage";
+import { embedCostUsd } from "../lib/cost-rates";
+async function indexDocument(args) {
+  await ensureIndex();
+  const doc = MDocument.fromText(args.text);
+  const chunks = await doc.chunk({ strategy: "recursive", maxSize: 512, overlap: 50 });
+  if (chunks.length === 0) {
+    return { chunks: 0, tokens: 0, costUsd: 0 };
+  }
+  const { embeddings, usage } = await embedMany({
+    model: jinaEmbedder,
+    values: chunks.map((c) => c.text)
+  });
+  const tokens = usage?.tokens ?? 0;
+  const costUsd = embedCostUsd(tokens);
+  if (args.identity) {
+    try {
+      await recordCost({ identity: args.identity, source: "embed", op: args.op ?? "index", tokens, costUsd, vault: args.vault });
+    } catch {
+    }
+  }
+  const chunkIds = chunks.map(
+    (_, i) => createHash("sha256").update(`${args.vault}::${args.id}::${i}`).digest("hex")
+  );
+  const capturedAt = args.capturedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  await memoryVectors.upsert({
+    indexName: INDEX_NAME,
+    vectors: embeddings,
+    ids: chunkIds,
+    metadata: chunks.map((c) => ({
+      text: c.text,
+      source: args.source,
+      vault: args.vault,
+      docId: args.id,
+      capturedAt
+    }))
+  });
+  return { chunks: chunks.length, tokens, costUsd };
+}
+async function removeDocument(args) {
+  await memoryVectors.deleteVectors({
+    indexName: INDEX_NAME,
+    filter: { vault: args.vault, docId: args.id }
+  });
+}
+async function removeVaultVectors(args) {
+  await memoryVectors.deleteVectors({
+    indexName: INDEX_NAME,
+    filter: { vault: args.vault }
+  });
+}
+export {
+  indexDocument,
+  removeDocument,
+  removeVaultVectors
+};
+//# sourceMappingURL=index-pipeline.js.map

package/dist/rag/index-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/rag/index-pipeline.ts"],"sourcesContent":["import { MDocument } from '@mastra/rag'\nimport { embedMany } from 'ai'\nimport { createHash } from 'node:crypto'\nimport { jinaEmbedder } from './embedder'\nimport { memoryVectors, INDEX_NAME, ensureIndex } from './vector-store'\nimport { recordCost } from '../db/usage'\nimport { embedCostUsd } from '../lib/cost-rates'\n\nexport async function indexDocument(args: {\n  vault: string\n  id: string\n  text: string\n  source: string\n  capturedAt?: string\n  identity?: string\n  op?: string\n}): Promise<{ chunks: number; tokens: number; costUsd: number }> {\n  await ensureIndex()\n\n  const doc = MDocument.fromText(args.text)\n  const chunks = await doc.chunk({ strategy: 'recursive', maxSize: 512, overlap: 50 })\n\n  if (chunks.length === 0) {\n    return { chunks: 0, tokens: 0, costUsd: 0 }\n  }\n\n  const { embeddings, usage } = await embedMany({\n    model: jinaEmbedder,\n    values: chunks.map((c) => c.text),\n  })\n\n  const tokens = usage?.tokens ?? 0\n  const costUsd = embedCostUsd(tokens)\n  if (args.identity) {\n    try {\n      await recordCost({ identity: args.identity, source: 'embed', op: args.op ?? 'index', tokens, costUsd, vault: args.vault })\n    } catch {\n      void 0\n    }\n  }\n\n  const chunkIds = chunks.map((_, i) =>\n    createHash('sha256').update(`${args.vault}::${args.id}::${i}`).digest('hex'),\n  )\n\n  const capturedAt = args.capturedAt ?? new Date().toISOString()\n\n  await memoryVectors.upsert({\n    indexName: INDEX_NAME,\n    vectors: embeddings,\n    ids: chunkIds,\n    metadata: chunks.map((c) => ({\n      text: c.text,\n      source: args.source,\n      vault: args.vault,\n      docId: args.id,\n      capturedAt,\n    })),\n  })\n\n  return { chunks: chunks.length, tokens, costUsd }\n}\n\nexport async function removeDocument(args: { vault: string; id: string }): Promise<void> {\n  await memoryVectors.deleteVectors({\n    indexName: INDEX_NAME,\n    filter: { vault: args.vault, docId: args.id },\n  })\n}\n\nexport async function removeVaultVectors(args: { vault: string }): Promise<void> {\n  await memoryVectors.deleteVectors({\n    indexName: INDEX_NAME,\n    filter: { vault: args.vault },\n  })\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,iBAAiB;AAC1B,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAC7B,SAAS,eAAe,YAAY,mBAAmB;AACvD,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAE7B,eAAsB,cAAc,MAQ6B;AAC/D,QAAM,YAAY;AAElB,QAAM,MAAM,UAAU,SAAS,KAAK,IAAI;AACxC,QAAM,SAAS,MAAM,IAAI,MAAM,EAAE,UAAU,aAAa,SAAS,KAAK,SAAS,GAAG,CAAC;AAEnF,MAAI,OAAO,WAAW,GAAG;AACvB,WAAO,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,EAAE;AAAA,EAC5C;AAEA,QAAM,EAAE,YAAY,MAAM,IAAI,MAAM,UAAU;AAAA,IAC5C,OAAO;AAAA,IACP,QAAQ,OAAO,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,EAClC,CAAC;AAED,QAAM,SAAS,OAAO,UAAU;AAChC,QAAM,UAAU,aAAa,MAAM;AACnC,MAAI,KAAK,UAAU;AACjB,QAAI;AACF,YAAM,WAAW,EAAE,UAAU,KAAK,UAAU,QAAQ,SAAS,IAAI,KAAK,MAAM,SAAS,QAAQ,SAAS,OAAO,KAAK,MAAM,CAAC;AAAA,IAC3H,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,QAAM,WAAW,OAAO;AAAA,IAAI,CAAC,GAAG,MAC9B,WAAW,QAAQ,EAAE,OAAO,GAAG,KAAK,KAAK,KAAK,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,KAAK;AAAA,EAC7E;AAEA,QAAM,aAAa,KAAK,eAAc,oBAAI,KAAK,GAAE,YAAY;AAE7D,QAAM,cAAc,OAAO;AAAA,IACzB,WAAW;AAAA,IACX,SAAS;AAAA,IACT,KAAK;AAAA,IACL,UAAU,OAAO,IAAI,CAAC,OAAO;AAAA,MAC3B,MAAM,EAAE;AAAA,MACR,QAAQ,KAAK;AAAA,MACb,OAAO,KAAK;AAAA,MACZ,OAAO,KAAK;AAAA,MACZ;AAAA,IACF,EAAE;AAAA,EACJ,CAAC;AAED,SAAO,EAAE,QAAQ,OAAO,QAAQ,QAAQ,QAAQ;AAClD;AAEA,eAAsB,eAAe,MAAoD;AACvF,QAAM,cAAc,cAAc;AAAA,IAChC,WAAW;AAAA,IACX,QAAQ,EAAE,OAAO,KAAK,OAAO,OAAO,KAAK,GAAG;AAAA,EAC9C,CAAC;AACH;AAEA,eAAsB,mBAAmB,MAAwC;AAC/E,QAAM,cAAc,cAAc;AAAA,IAChC,WAAW;AAAA,IACX,QAAQ,EAAE,OAAO,KAAK,MAAM;AAAA,EAC9B,CAAC;AACH;","names":[]}

package/dist/rag/query-tool.d.ts

@@ -0,0 +1,47 @@
+import { RagTool } from './vector-search.js';
+
+declare const vectorQueryTool: RagTool<{
+    queryText: string;
+    filter?: z.ZodCoercedString<unknown>;
+    topK?: z.ZodCoercedNumber<unknown>;
+} | {
+    queryText: string;
+    topK?: z.ZodCoercedNumber<unknown>;
+}, any>;
+declare const graphQueryTool: RagTool<{
+    queryText: string;
+    filter?: z.ZodCoercedString<unknown>;
+    topK?: z.ZodCoercedNumber<unknown>;
+} | {
+    queryText: string;
+    topK?: z.ZodCoercedNumber<unknown>;
+}, any>;
+declare function embedQuery(query: string): Promise<number[]>;
+declare function queryWithEmbedding(args: {
+    embedding: number[];
+    vault: string | string[];
+    topK?: number;
+    docId?: string;
+}): Promise<{
+    results: {
+        text: string;
+        source: string;
+        score: number;
+        vault?: string;
+    }[];
+}>;
+declare function semanticSearch(args: {
+    vault: string | string[];
+    query: string;
+    topK?: number;
+    docId?: string;
+}): Promise<{
+    results: {
+        text: string;
+        source: string;
+        score: number;
+        vault?: string;
+    }[];
+}>;
+
+export { embedQuery, graphQueryTool, queryWithEmbedding, semanticSearch, vectorQueryTool };

package/dist/rag/query-tool.js

@@ -0,0 +1,47 @@
+import { createVectorQueryTool, createGraphRAGTool } from "@mastra/rag";
+import { embed } from "ai";
+import { jinaEmbedder } from "./embedder";
+import { memoryVectors, INDEX_NAME } from "./vector-store";
+const vectorQueryTool = createVectorQueryTool({
+  vectorStoreName: "memoryVectors",
+  indexName: INDEX_NAME,
+  model: jinaEmbedder
+});
+const graphQueryTool = createGraphRAGTool({
+  vectorStoreName: "memoryVectors",
+  indexName: INDEX_NAME,
+  model: jinaEmbedder,
+  graphOptions: { dimension: 1024 }
+});
+async function embedQuery(query) {
+  const { embedding } = await embed({ model: jinaEmbedder, value: query });
+  return embedding;
+}
+async function queryWithEmbedding(args) {
+  const vaultFilter = Array.isArray(args.vault) ? { $in: args.vault } : args.vault;
+  const hits = await memoryVectors.query({
+    indexName: INDEX_NAME,
+    queryVector: args.embedding,
+    topK: args.topK ?? 8,
+    filter: args.docId ? { vault: vaultFilter, docId: args.docId } : { vault: vaultFilter }
+  });
+  const results = hits.map((h) => ({
+    text: String(h.metadata?.text ?? ""),
+    source: String(h.metadata?.source ?? ""),
+    score: Number(h.score ?? 0),
+    vault: h.metadata?.vault ? String(h.metadata.vault) : void 0
+  }));
+  return { results };
+}
+async function semanticSearch(args) {
+  const embedding = await embedQuery(args.query);
+  return queryWithEmbedding({ embedding, vault: args.vault, topK: args.topK, docId: args.docId });
+}
+export {
+  embedQuery,
+  graphQueryTool,
+  queryWithEmbedding,
+  semanticSearch,
+  vectorQueryTool
+};
+//# sourceMappingURL=query-tool.js.map

package/dist/rag/query-tool.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/rag/query-tool.ts"],"sourcesContent":["import { createVectorQueryTool, createGraphRAGTool } from '@mastra/rag'\nimport { embed } from 'ai'\nimport { jinaEmbedder } from './embedder'\nimport { memoryVectors, INDEX_NAME } from './vector-store'\n\nexport const vectorQueryTool = createVectorQueryTool({\n  vectorStoreName: 'memoryVectors',\n  indexName: INDEX_NAME,\n  model: jinaEmbedder,\n})\n\nexport const graphQueryTool = createGraphRAGTool({\n  vectorStoreName: 'memoryVectors',\n  indexName: INDEX_NAME,\n  model: jinaEmbedder,\n  graphOptions: { dimension: 1024 },\n})\n\nexport async function embedQuery(query: string): Promise<number[]> {\n  const { embedding } = await embed({ model: jinaEmbedder, value: query })\n  return embedding\n}\n\nexport async function queryWithEmbedding(args: {\n  embedding: number[]\n  vault: string | string[]\n  topK?: number\n  docId?: string\n}): Promise<{ results: { text: string; source: string; score: number; vault?: string }[] }> {\n  const vaultFilter = Array.isArray(args.vault) ? { $in: args.vault } : args.vault\n  const hits = await memoryVectors.query({\n    indexName: INDEX_NAME,\n    queryVector: args.embedding,\n    topK: args.topK ?? 8,\n    filter: args.docId ? { vault: vaultFilter, docId: args.docId } : { vault: vaultFilter },\n  })\n  const results = hits.map((h) => ({\n    text: String(h.metadata?.text ?? ''),\n    source: String(h.metadata?.source ?? ''),\n    score: Number(h.score ?? 0),\n    vault: h.metadata?.vault ? String(h.metadata.vault) : undefined,\n  }))\n  return { results }\n}\n\nexport async function semanticSearch(args: {\n  vault: string | string[]\n  query: string\n  topK?: number\n  docId?: string\n}): Promise<{ results: { text: string; source: string; score: number; vault?: string }[] }> {\n  const embedding = await embedQuery(args.query)\n  return queryWithEmbedding({ embedding, vault: args.vault, topK: args.topK, docId: args.docId })\n}\n"],"mappings":"AAAA,SAAS,uBAAuB,0BAA0B;AAC1D,SAAS,aAAa;AACtB,SAAS,oBAAoB;AAC7B,SAAS,eAAe,kBAAkB;AAEnC,MAAM,kBAAkB,sBAAsB;AAAA,EACnD,iBAAiB;AAAA,EACjB,WAAW;AAAA,EACX,OAAO;AACT,CAAC;AAEM,MAAM,iBAAiB,mBAAmB;AAAA,EAC/C,iBAAiB;AAAA,EACjB,WAAW;AAAA,EACX,OAAO;AAAA,EACP,cAAc,EAAE,WAAW,KAAK;AAClC,CAAC;AAED,eAAsB,WAAW,OAAkC;AACjE,QAAM,EAAE,UAAU,IAAI,MAAM,MAAM,EAAE,OAAO,cAAc,OAAO,MAAM,CAAC;AACvE,SAAO;AACT;AAEA,eAAsB,mBAAmB,MAKmD;AAC1F,QAAM,cAAc,MAAM,QAAQ,KAAK,KAAK,IAAI,EAAE,KAAK,KAAK,MAAM,IAAI,KAAK;AAC3E,QAAM,OAAO,MAAM,cAAc,MAAM;AAAA,IACrC,WAAW;AAAA,IACX,aAAa,KAAK;AAAA,IAClB,MAAM,KAAK,QAAQ;AAAA,IACnB,QAAQ,KAAK,QAAQ,EAAE,OAAO,aAAa,OAAO,KAAK,MAAM,IAAI,EAAE,OAAO,YAAY;AAAA,EACxF,CAAC;AACD,QAAM,UAAU,KAAK,IAAI,CAAC,OAAO;AAAA,IAC/B,MAAM,OAAO,EAAE,UAAU,QAAQ,EAAE;AAAA,IACnC,QAAQ,OAAO,EAAE,UAAU,UAAU,EAAE;AAAA,IACvC,OAAO,OAAO,EAAE,SAAS,CAAC;AAAA,IAC1B,OAAO,EAAE,UAAU,QAAQ,OAAO,EAAE,SAAS,KAAK,IAAI;AAAA,EACxD,EAAE;AACF,SAAO,EAAE,QAAQ;AACnB;AAEA,eAAsB,eAAe,MAKuD;AAC1F,QAAM,YAAY,MAAM,WAAW,KAAK,KAAK;AAC7C,SAAO,mBAAmB,EAAE,WAAW,OAAO,KAAK,OAAO,MAAM,KAAK,MAAM,OAAO,KAAK,MAAM,CAAC;AAChG;","names":[]}

package/dist/rag/vector-store.d.ts

@@ -0,0 +1,7 @@
+import { PgVector } from '@mastra/pg';
+
+declare const INDEX_NAME = "memory";
+declare const memoryVectors: PgVector;
+declare function ensureIndex(): Promise<void>;
+
+export { INDEX_NAME, ensureIndex, memoryVectors };

package/dist/rag/vector-store.js

@@ -0,0 +1,25 @@
+import { PgVector } from "@mastra/pg";
+import { EMBED_DIM } from "./embedder";
+const INDEX_NAME = "memory";
+const memoryVectors = new PgVector({
+  id: "memoryVectors",
+  connectionString: process.env.DATABASE_URL ?? ""
+});
+async function ensureIndex() {
+  try {
+    await memoryVectors.createIndex({ indexName: INDEX_NAME, dimension: EMBED_DIM });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const normalized = message.toLowerCase();
+    const alreadyExists = normalized.includes("already exists") || normalized.includes("duplicate");
+    if (!alreadyExists) {
+      throw error;
+    }
+  }
+}
+export {
+  INDEX_NAME,
+  ensureIndex,
+  memoryVectors
+};
+//# sourceMappingURL=vector-store.js.map

package/dist/rag/vector-store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/rag/vector-store.ts"],"sourcesContent":["import { PgVector } from '@mastra/pg';\nimport { EMBED_DIM } from './embedder';\n\nexport const INDEX_NAME = 'memory';\n\nexport const memoryVectors = new PgVector({\n  id: 'memoryVectors',\n  connectionString: process.env.DATABASE_URL ?? '',\n});\n\nexport async function ensureIndex(): Promise<void> {\n  try {\n    await memoryVectors.createIndex({ indexName: INDEX_NAME, dimension: EMBED_DIM });\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    const normalized = message.toLowerCase();\n    const alreadyExists =\n      normalized.includes('already exists') || normalized.includes('duplicate');\n    if (!alreadyExists) {\n      throw error;\n    }\n  }\n}\n"],"mappings":"AAAA,SAAS,gBAAgB;AACzB,SAAS,iBAAiB;AAEnB,MAAM,aAAa;AAEnB,MAAM,gBAAgB,IAAI,SAAS;AAAA,EACxC,IAAI;AAAA,EACJ,kBAAkB,QAAQ,IAAI,gBAAgB;AAChD,CAAC;AAED,eAAsB,cAA6B;AACjD,MAAI;AACF,UAAM,cAAc,YAAY,EAAE,WAAW,YAAY,WAAW,UAAU,CAAC;AAAA,EACjF,SAAS,OAAO;AACd,UAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,UAAM,aAAa,QAAQ,YAAY;AACvC,UAAM,gBACJ,WAAW,SAAS,gBAAgB,KAAK,WAAW,SAAS,WAAW;AAC1E,QAAI,CAAC,eAAe;AAClB,YAAM;AAAA,IACR;AAAA,EACF;AACF;","names":[]}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "mcpscraper-memory-db",
+  "version": "1.0.0",
+  "description": "Shared data-access layer for the mcpscraper memory product — Postgres db/*.ts, the auth/crypto/vault lib/*.ts modules it depends on, and the Jina/pgvector rag/*.ts indexing layer. Consumed by mcp-memory and mcp-scraper-scheduler so both stay on one implementation instead of hand-copied duplicates.",
+  "type": "module",
+  "license": "UNLICENSED",
+  "private": false,
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/VilovietaSEO/mcpscraper-memory-db.git"
+  },
+  "exports": {
+    "./db/*": {
+      "types": "./dist/db/*.d.ts",
+      "import": "./dist/db/*.js"
+    },
+    "./lib/*": {
+      "types": "./dist/lib/*.d.ts",
+      "import": "./dist/lib/*.js"
+    },
+    "./rag/*": {
+      "types": "./dist/rag/*.d.ts",
+      "import": "./dist/rag/*.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@ai-sdk/provider": "2.0.3",
+    "@mastra/pg": "latest",
+    "@mastra/rag": "latest",
+    "ai": "latest",
+    "jose": "^6.2.3",
+    "pg": "latest"
+  },
+  "devDependencies": {
+    "tsup": "^8.0.0",
+    "typescript": "^5.6.0",
+    "@types/node": "^22.0.0",
+    "@types/pg": "^8.11.0"
+  }
+}