@monoes/monograph 1.2.8 → 1.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +0 -7
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +0 -7
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcp-tools/index.d.ts +0 -1
- package/dist/src/mcp-tools/index.d.ts.map +1 -1
- package/dist/src/mcp-tools/index.js +0 -1
- package/dist/src/mcp-tools/index.js.map +1 -1
- package/dist/src/search/hybrid-query.d.ts +11 -17
- package/dist/src/search/hybrid-query.d.ts.map +1 -1
- package/dist/src/search/hybrid-query.js +10 -95
- package/dist/src/search/hybrid-query.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/index.ts +0 -7
- package/src/mcp-tools/index.ts +0 -1
- package/src/search/hybrid-query.ts +13 -120
- package/dist/src/__tests__/mcp-tools/embed.test.d.ts +0 -2
- package/dist/src/__tests__/mcp-tools/embed.test.d.ts.map +0 -1
- package/dist/src/__tests__/mcp-tools/embed.test.js +0 -12
- package/dist/src/__tests__/mcp-tools/embed.test.js.map +0 -1
- package/dist/src/__tests__/search/embed-stale.test.d.ts +0 -2
- package/dist/src/__tests__/search/embed-stale.test.d.ts.map +0 -1
- package/dist/src/__tests__/search/embed-stale.test.js +0 -38
- package/dist/src/__tests__/search/embed-stale.test.js.map +0 -1
- package/dist/src/__tests__/search/exact-search.test.d.ts +0 -2
- package/dist/src/__tests__/search/exact-search.test.d.ts.map +0 -1
- package/dist/src/__tests__/search/exact-search.test.js +0 -49
- package/dist/src/__tests__/search/exact-search.test.js.map +0 -1
- package/dist/src/mcp-tools/embed.d.ts +0 -25
- package/dist/src/mcp-tools/embed.d.ts.map +0 -1
- package/dist/src/mcp-tools/embed.js +0 -33
- package/dist/src/mcp-tools/embed.js.map +0 -1
- package/dist/src/pipeline/phases/embeddings.d.ts +0 -6
- package/dist/src/pipeline/phases/embeddings.d.ts.map +0 -1
- package/dist/src/pipeline/phases/embeddings.js +0 -12
- package/dist/src/pipeline/phases/embeddings.js.map +0 -1
- package/dist/src/search/device-config.d.ts +0 -16
- package/dist/src/search/device-config.d.ts.map +0 -1
- package/dist/src/search/device-config.js +0 -22
- package/dist/src/search/device-config.js.map +0 -1
- package/dist/src/search/embed-batch.d.ts +0 -30
- package/dist/src/search/embed-batch.d.ts.map +0 -1
- package/dist/src/search/embed-batch.js +0 -99
- package/dist/src/search/embed-batch.js.map +0 -1
- package/dist/src/search/embedder.d.ts +0 -26
- package/dist/src/search/embedder.d.ts.map +0 -1
- package/dist/src/search/embedder.js +0 -49
- package/dist/src/search/embedder.js.map +0 -1
- package/dist/src/search/exact-search.d.ts +0 -10
- package/dist/src/search/exact-search.d.ts.map +0 -1
- package/dist/src/search/exact-search.js +0 -28
- package/dist/src/search/exact-search.js.map +0 -1
- package/dist/src/search/http-embedder.d.ts +0 -17
- package/dist/src/search/http-embedder.d.ts.map +0 -1
- package/dist/src/search/http-embedder.js +0 -57
- package/dist/src/search/http-embedder.js.map +0 -1
- package/dist/src/storage/embedding-store.d.ts +0 -29
- package/dist/src/storage/embedding-store.d.ts.map +0 -1
- package/dist/src/storage/embedding-store.js +0 -77
- package/dist/src/storage/embedding-store.js.map +0 -1
- package/src/__tests__/mcp-tools/embed.test.ts +0 -13
- package/src/__tests__/search/embed-stale.test.ts +0 -42
- package/src/__tests__/search/exact-search.test.ts +0 -54
- package/src/mcp-tools/embed.ts +0 -56
- package/src/pipeline/phases/embeddings.ts +0 -17
- package/src/search/device-config.ts +0 -31
- package/src/search/embed-batch.ts +0 -151
- package/src/search/embedder.ts +0 -60
- package/src/search/exact-search.ts +0 -45
- package/src/search/http-embedder.ts +0 -78
- package/src/storage/embedding-store.ts +0 -108
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embed-batch.js","sourceRoot":"","sources":["../../../src/search/embed-batch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACxG,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE1C,MAAM,UAAU,GAAG,EAAE,CAAC;AAQtB;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,MAAyB;IAEzB,0EAA0E;IAC1E,MAAM,OAAO,GAAG,MAAM,EAAE,MAAM,CAAC;IAC/B,KAAK,OAAO,CAAC;IAEb,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjD,OAAO,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,sBAAsB;IACtB,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,MAAM,WAAW,EAAE,CAAC;IACvC,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,UAAU,CAAC;IAClD,MAAM,OAAO,GAAe,EAAE,CAAC;IAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACjD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC,CAAC,CACH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;;;;;;;GASG;AACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;AAEvJ,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAqB,EACrB,QAAoB,EACpB,KAAK,GAAG,KAAK,EACb,QAAQ,GAAG,KAAK;IAEhB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,0EAA0E;IAC1E,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN;;sBAEgB,CACjB;SACA,GAAG,EAAiG,CAAC;IAExG,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpF,qDAAqD;IACrD,MAAM,WAAW,GAAG,KAAK;QACvB,CAAC,CAAC,IAAI,GAAG,EAAU;QACnB,CAAC,CAAC,IAAI,GAAG,CAEH,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC,GAAG,EACjD,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CACxB,CAAC;IAEN,0CAA0C;IAC1C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,OAAO,GAAG,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAE/C,qEAAqE;IACrE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,qBAAqB,CAAC,EAAE,CAAC,CAAC;IAElD,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC/C,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAC/C,eAAe,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gBACpC,QAAQ,EAAE,CAAC;YACb,CAAC;YAAC,MAAM,CAAC;gBACP,kDAAkD;gBAClD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAED,iFAAiF;AAEjF,SAAS,aAAa,CAAC,GAKtB;IACC,MAAM,KAAK,GAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC9E,IAAI,GAAG,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,OAAO,EAAE,eAAe,EAAE,CAAC"}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Lazy singleton for the HuggingFace FeatureExtractionPipeline.
|
|
3
|
-
*
|
|
4
|
-
* Uses dynamic import so that:
|
|
5
|
-
* 1. Tests can mock `getEmbedder` without installing the package.
|
|
6
|
-
* 2. @huggingface/transformers is optional at import time; callers receive a
|
|
7
|
-
* clear error if the package is unavailable.
|
|
8
|
-
*
|
|
9
|
-
* Model: Snowflake/snowflake-arctic-embed-xs (384 dimensions)
|
|
10
|
-
*/
|
|
11
|
-
export type EmbedderFn = (text: string | string[], options?: Record<string, unknown>) => Promise<any>;
|
|
12
|
-
/**
|
|
13
|
-
* Returns (and caches) a FeatureExtractionPipeline for
|
|
14
|
-
* 'Snowflake/snowflake-arctic-embed-xs'.
|
|
15
|
-
*
|
|
16
|
-
* Throws a descriptive error if @huggingface/transformers is not installed.
|
|
17
|
-
*/
|
|
18
|
-
export declare function getEmbedder(): Promise<EmbedderFn>;
|
|
19
|
-
/** Reset the cached singleton — used only in tests. */
|
|
20
|
-
export declare function resetEmbedderCache(): void;
|
|
21
|
-
/**
|
|
22
|
-
* Embed a single text string and return a 384-dim Float32Array.
|
|
23
|
-
* Pooling: mean. Normalisation: L2 (so cosine ≡ dot product).
|
|
24
|
-
*/
|
|
25
|
-
export declare function embedText(text: string, embedder: EmbedderFn): Promise<Float32Array>;
|
|
26
|
-
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../../src/search/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;AAItG;;;;;GAKG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,UAAU,CAAC,CAmBvD;AAED,uDAAuD;AACvD,wBAAgB,kBAAkB,IAAI,IAAI,CAEzC;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,CAAC,CAIvB"}
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Lazy singleton for the HuggingFace FeatureExtractionPipeline.
|
|
3
|
-
*
|
|
4
|
-
* Uses dynamic import so that:
|
|
5
|
-
* 1. Tests can mock `getEmbedder` without installing the package.
|
|
6
|
-
* 2. @huggingface/transformers is optional at import time; callers receive a
|
|
7
|
-
* clear error if the package is unavailable.
|
|
8
|
-
*
|
|
9
|
-
* Model: Snowflake/snowflake-arctic-embed-xs (384 dimensions)
|
|
10
|
-
*/
|
|
11
|
-
let cachedEmbedder = null;
|
|
12
|
-
/**
|
|
13
|
-
* Returns (and caches) a FeatureExtractionPipeline for
|
|
14
|
-
* 'Snowflake/snowflake-arctic-embed-xs'.
|
|
15
|
-
*
|
|
16
|
-
* Throws a descriptive error if @huggingface/transformers is not installed.
|
|
17
|
-
*/
|
|
18
|
-
export async function getEmbedder() {
|
|
19
|
-
if (cachedEmbedder)
|
|
20
|
-
return cachedEmbedder;
|
|
21
|
-
let pipeline;
|
|
22
|
-
try {
|
|
23
|
-
// Dynamic import so the dep is optional at bundle time
|
|
24
|
-
const hf = await import('@huggingface/transformers');
|
|
25
|
-
pipeline = hf.pipeline;
|
|
26
|
-
}
|
|
27
|
-
catch {
|
|
28
|
-
throw new Error('@huggingface/transformers is not installed. ' +
|
|
29
|
-
'Run `npm install @huggingface/transformers` to enable embedding support.');
|
|
30
|
-
}
|
|
31
|
-
cachedEmbedder = await pipeline('feature-extraction', 'Snowflake/snowflake-arctic-embed-xs', {
|
|
32
|
-
dtype: 'fp32',
|
|
33
|
-
});
|
|
34
|
-
return cachedEmbedder;
|
|
35
|
-
}
|
|
36
|
-
/** Reset the cached singleton — used only in tests. */
|
|
37
|
-
export function resetEmbedderCache() {
|
|
38
|
-
cachedEmbedder = null;
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Embed a single text string and return a 384-dim Float32Array.
|
|
42
|
-
* Pooling: mean. Normalisation: L2 (so cosine ≡ dot product).
|
|
43
|
-
*/
|
|
44
|
-
export async function embedText(text, embedder) {
|
|
45
|
-
const output = await embedder(text, { pooling: 'mean', normalize: true });
|
|
46
|
-
// HuggingFace Tensor: output.data is Float32Array
|
|
47
|
-
return output.data;
|
|
48
|
-
}
|
|
49
|
-
//# sourceMappingURL=embedder.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../../src/search/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,IAAI,cAAc,GAAsB,IAAI,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,IAAI,cAAc;QAAE,OAAO,cAAc,CAAC;IAE1C,IAAI,QAA8F,CAAC;IACnG,IAAI,CAAC;QACH,uDAAuD;QACvD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACrD,QAAQ,GAAG,EAAE,CAAC,QAA2B,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,8CAA8C;YAC5C,0EAA0E,CAC7E,CAAC;IACJ,CAAC;IAED,cAAc,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,qCAAqC,EAAE;QAC3F,KAAK,EAAE,MAAM;KACd,CAAC,CAAC;IACH,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,kBAAkB;IAChC,cAAc,GAAG,IAAI,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,QAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,kDAAkD;IAClD,OAAO,MAAM,CAAC,IAAoB,CAAC;AACrC,CAAC"}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type Database from 'better-sqlite3';
|
|
2
|
-
export interface ExactSearchResult {
|
|
3
|
-
id: string;
|
|
4
|
-
score: number;
|
|
5
|
-
}
|
|
6
|
-
export interface ExactSearchOptions {
|
|
7
|
-
limit?: number;
|
|
8
|
-
}
|
|
9
|
-
export declare function exactVectorSearch(db: Database.Database, queryVector: Float32Array, options?: ExactSearchOptions): ExactSearchResult[];
|
|
10
|
-
//# sourceMappingURL=exact-search.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"exact-search.d.ts","sourceRoot":"","sources":["../../../src/search/exact-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAcD,wBAAgB,iBAAiB,CAC/B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,WAAW,EAAE,YAAY,EACzB,OAAO,GAAE,kBAAuB,GAC/B,iBAAiB,EAAE,CAiBrB"}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
function cosineSimilarity(a, b) {
|
|
2
|
-
let dot = 0, normA = 0, normB = 0;
|
|
3
|
-
const len = Math.min(a.length, b.length);
|
|
4
|
-
for (let i = 0; i < len; i++) {
|
|
5
|
-
dot += a[i] * b[i];
|
|
6
|
-
normA += a[i] * a[i];
|
|
7
|
-
normB += b[i] * b[i];
|
|
8
|
-
}
|
|
9
|
-
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
10
|
-
return denom === 0 ? 0 : dot / denom;
|
|
11
|
-
}
|
|
12
|
-
export function exactVectorSearch(db, queryVector, options = {}) {
|
|
13
|
-
const { limit = 10 } = options;
|
|
14
|
-
const rows = db.prepare(`
|
|
15
|
-
SELECT e.node_id as id, e.vector FROM embeddings e
|
|
16
|
-
INNER JOIN nodes n ON n.id = e.node_id
|
|
17
|
-
`).all();
|
|
18
|
-
if (rows.length === 0)
|
|
19
|
-
return [];
|
|
20
|
-
return rows
|
|
21
|
-
.map(row => {
|
|
22
|
-
const vec = new Float32Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength / 4);
|
|
23
|
-
return { id: row.id, score: cosineSimilarity(queryVector, vec) };
|
|
24
|
-
})
|
|
25
|
-
.sort((a, b) => b.score - a.score)
|
|
26
|
-
.slice(0, limit);
|
|
27
|
-
}
|
|
28
|
-
//# sourceMappingURL=exact-search.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"exact-search.js","sourceRoot":"","sources":["../../../src/search/exact-search.ts"],"names":[],"mappings":"AAWA,SAAS,gBAAgB,CAAC,CAAe,EAAE,CAAe;IACxD,IAAI,GAAG,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACvB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACzB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,WAAyB,EACzB,UAA8B,EAAE;IAEhC,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAE/B,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAGvB,CAAC,CAAC,GAAG,EAAsC,CAAC;IAE7C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,OAAO,IAAI;SACR,GAAG,CAAC,GAAG,CAAC,EAAE;QACT,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAClG,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;IACnE,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
export interface HttpEmbedderConfig {
|
|
2
|
-
endpoint: string;
|
|
3
|
-
model?: string;
|
|
4
|
-
apiKey?: string;
|
|
5
|
-
timeoutMs?: number;
|
|
6
|
-
batchSize?: number;
|
|
7
|
-
}
|
|
8
|
-
export interface EmbedResponse {
|
|
9
|
-
embedding: number[];
|
|
10
|
-
}
|
|
11
|
-
export declare class HttpEmbedder {
|
|
12
|
-
private config;
|
|
13
|
-
constructor(config: HttpEmbedderConfig);
|
|
14
|
-
embedOne(text: string): Promise<number[]>;
|
|
15
|
-
embedBatch(texts: string[]): Promise<number[][]>;
|
|
16
|
-
}
|
|
17
|
-
//# sourceMappingURL=http-embedder.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"http-embedder.d.ts","sourceRoot":"","sources":["../../../src/search/http-embedder.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAA+B;gBAEjC,MAAM,EAAE,kBAAkB;IAUhC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAuCzC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAavD"}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
export class HttpEmbedder {
|
|
2
|
-
config;
|
|
3
|
-
constructor(config) {
|
|
4
|
-
this.config = {
|
|
5
|
-
model: '',
|
|
6
|
-
apiKey: '',
|
|
7
|
-
timeoutMs: 30_000,
|
|
8
|
-
batchSize: 32,
|
|
9
|
-
...config,
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
async embedOne(text) {
|
|
13
|
-
const controller = new AbortController();
|
|
14
|
-
const timer = setTimeout(() => controller.abort(), this.config.timeoutMs);
|
|
15
|
-
try {
|
|
16
|
-
const headers = {
|
|
17
|
-
'Content-Type': 'application/json',
|
|
18
|
-
};
|
|
19
|
-
if (this.config.apiKey) {
|
|
20
|
-
headers['Authorization'] = `Bearer ${this.config.apiKey}`;
|
|
21
|
-
}
|
|
22
|
-
const body = { prompt: text };
|
|
23
|
-
if (this.config.model) {
|
|
24
|
-
body['model'] = this.config.model;
|
|
25
|
-
}
|
|
26
|
-
const response = await fetch(this.config.endpoint, {
|
|
27
|
-
method: 'POST',
|
|
28
|
-
headers,
|
|
29
|
-
body: JSON.stringify(body),
|
|
30
|
-
signal: controller.signal,
|
|
31
|
-
});
|
|
32
|
-
if (!response.ok) {
|
|
33
|
-
throw new Error(`HTTP embedder: ${response.status} ${response.statusText}`);
|
|
34
|
-
}
|
|
35
|
-
const data = (await response.json());
|
|
36
|
-
if (!data.embedding) {
|
|
37
|
-
throw new Error('HTTP embedder: no embedding in response');
|
|
38
|
-
}
|
|
39
|
-
return data.embedding;
|
|
40
|
-
}
|
|
41
|
-
finally {
|
|
42
|
-
clearTimeout(timer);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
async embedBatch(texts) {
|
|
46
|
-
const results = [];
|
|
47
|
-
const batchSize = this.config.batchSize;
|
|
48
|
-
for (let i = 0; i < texts.length; i += batchSize) {
|
|
49
|
-
const batch = texts.slice(i, i + batchSize);
|
|
50
|
-
for (const text of batch) {
|
|
51
|
-
results.push(await this.embedOne(text));
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
return results;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
//# sourceMappingURL=http-embedder.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"http-embedder.js","sourceRoot":"","sources":["../../../src/search/http-embedder.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,YAAY;IACf,MAAM,CAA+B;IAE7C,YAAY,MAA0B;QACpC,IAAI,CAAC,MAAM,GAAG;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,MAAM;YACjB,SAAS,EAAE,EAAE;YACb,GAAG,MAAM;SACV,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAE1E,IAAI,CAAC;YACH,MAAM,OAAO,GAA2B;gBACtC,cAAc,EAAE,kBAAkB;aACnC,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC5D,CAAC;YAED,MAAM,IAAI,GAA4B,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACvD,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACtB,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACpC,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;gBACjD,MAAM,EAAE,MAAM;gBACd,OAAO;gBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;YAC/D,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YAC7D,CAAC;YAED,OAAO,IAAI,CAAC,SAAS,CAAC;QACxB,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import type Database from 'better-sqlite3';
|
|
2
|
-
/**
|
|
3
|
-
* CRUD operations for the embeddings table.
|
|
4
|
-
*
|
|
5
|
-
* Vectors are stored as BLOBs (raw Float32Array bytes) and reconstructed on read.
|
|
6
|
-
*/
|
|
7
|
-
/**
|
|
8
|
-
* Ensure the embeddings table has the content_hash column introduced in a later
|
|
9
|
-
* schema version. Call this ONCE before a batch of upserts rather than inside
|
|
10
|
-
* upsertEmbedding itself to avoid running ALTER TABLE on every row write.
|
|
11
|
-
*/
|
|
12
|
-
export declare function ensureEmbeddingSchema(db: Database.Database): void;
|
|
13
|
-
export declare function upsertEmbedding(db: Database.Database, nodeId: string, vector: Float32Array, contentHash?: string): void;
|
|
14
|
-
/**
|
|
15
|
-
* Bulk-upsert multiple embeddings in a single transaction.
|
|
16
|
-
* Calls ensureEmbeddingSchema once before writing, then wraps all inserts in
|
|
17
|
-
* a transaction for 10-100x faster throughput vs per-row upsertEmbedding calls.
|
|
18
|
-
*/
|
|
19
|
-
export declare function batchUpsertEmbeddings(db: Database.Database, entries: Array<{
|
|
20
|
-
nodeId: string;
|
|
21
|
-
vector: Float32Array;
|
|
22
|
-
contentHash?: string;
|
|
23
|
-
}>): void;
|
|
24
|
-
export declare function getEmbeddingContentHash(db: Database.Database, nodeId: string): string | null;
|
|
25
|
-
export declare function isEmbeddingStale(db: Database.Database, nodeId: string, currentHash: string): boolean;
|
|
26
|
-
export declare function getEmbedding(db: Database.Database, nodeId: string): Float32Array | null;
|
|
27
|
-
export declare function getAllEmbeddings(db: Database.Database): Map<string, Float32Array>;
|
|
28
|
-
export declare function countEmbeddings(db: Database.Database): number;
|
|
29
|
-
//# sourceMappingURL=embedding-store.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-store.d.ts","sourceRoot":"","sources":["../../../src/storage/embedding-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C;;;;GAIG;AAEH;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAMjE;AAED,wBAAgB,eAAe,CAC7B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,YAAY,EACpB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAON;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAC7E,IAAI,CAeN;AAED,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK5F;AAED,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAClB,OAAO,CAGT;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAMvF;AAED,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAUjF;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,MAAM,CAG7D"}
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CRUD operations for the embeddings table.
|
|
3
|
-
*
|
|
4
|
-
* Vectors are stored as BLOBs (raw Float32Array bytes) and reconstructed on read.
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* Ensure the embeddings table has the content_hash column introduced in a later
|
|
8
|
-
* schema version. Call this ONCE before a batch of upserts rather than inside
|
|
9
|
-
* upsertEmbedding itself to avoid running ALTER TABLE on every row write.
|
|
10
|
-
*/
|
|
11
|
-
export function ensureEmbeddingSchema(db) {
|
|
12
|
-
try {
|
|
13
|
-
db.exec('ALTER TABLE embeddings ADD COLUMN content_hash TEXT');
|
|
14
|
-
}
|
|
15
|
-
catch {
|
|
16
|
-
// Column already exists — ignore.
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
export function upsertEmbedding(db, nodeId, vector, contentHash) {
|
|
20
|
-
const buf = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
21
|
-
db
|
|
22
|
-
.prepare('INSERT OR REPLACE INTO embeddings (node_id, vector, content_hash) VALUES (?, ?, ?)')
|
|
23
|
-
.run(nodeId, buf, contentHash ?? null);
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Bulk-upsert multiple embeddings in a single transaction.
|
|
27
|
-
* Calls ensureEmbeddingSchema once before writing, then wraps all inserts in
|
|
28
|
-
* a transaction for 10-100x faster throughput vs per-row upsertEmbedding calls.
|
|
29
|
-
*/
|
|
30
|
-
export function batchUpsertEmbeddings(db, entries) {
|
|
31
|
-
if (entries.length === 0)
|
|
32
|
-
return;
|
|
33
|
-
ensureEmbeddingSchema(db);
|
|
34
|
-
const stmt = db.prepare('INSERT OR REPLACE INTO embeddings (node_id, vector, content_hash) VALUES (?, ?, ?)');
|
|
35
|
-
const insertMany = db.transaction((rows) => {
|
|
36
|
-
for (const e of rows) {
|
|
37
|
-
const buf = Buffer.from(e.vector.buffer, e.vector.byteOffset, e.vector.byteLength);
|
|
38
|
-
stmt.run(e.nodeId, buf, e.contentHash ?? null);
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
insertMany(entries);
|
|
42
|
-
}
|
|
43
|
-
export function getEmbeddingContentHash(db, nodeId) {
|
|
44
|
-
const row = db
|
|
45
|
-
.prepare('SELECT content_hash FROM embeddings WHERE node_id = ?')
|
|
46
|
-
.get(nodeId);
|
|
47
|
-
return row?.content_hash ?? null;
|
|
48
|
-
}
|
|
49
|
-
export function isEmbeddingStale(db, nodeId, currentHash) {
|
|
50
|
-
const stored = getEmbeddingContentHash(db, nodeId);
|
|
51
|
-
return stored !== currentHash;
|
|
52
|
-
}
|
|
53
|
-
export function getEmbedding(db, nodeId) {
|
|
54
|
-
const row = db.prepare('SELECT vector FROM embeddings WHERE node_id = ?').get(nodeId);
|
|
55
|
-
if (!row)
|
|
56
|
-
return null;
|
|
57
|
-
return bufToFloat32(row.vector);
|
|
58
|
-
}
|
|
59
|
-
export function getAllEmbeddings(db) {
|
|
60
|
-
const rows = db.prepare('SELECT node_id, vector FROM embeddings').all();
|
|
61
|
-
const result = new Map();
|
|
62
|
-
for (const row of rows) {
|
|
63
|
-
result.set(row.node_id, bufToFloat32(row.vector));
|
|
64
|
-
}
|
|
65
|
-
return result;
|
|
66
|
-
}
|
|
67
|
-
export function countEmbeddings(db) {
|
|
68
|
-
const row = db.prepare('SELECT COUNT(*) AS n FROM embeddings').get();
|
|
69
|
-
return row.n;
|
|
70
|
-
}
|
|
71
|
-
// ── Internal helpers ──────────────────────────────────────────────────────────
|
|
72
|
-
function bufToFloat32(buf) {
|
|
73
|
-
// Copy to a fresh ArrayBuffer so slice offset doesn't cause issues
|
|
74
|
-
const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
|
75
|
-
return new Float32Array(ab);
|
|
76
|
-
}
|
|
77
|
-
//# sourceMappingURL=embedding-store.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../../src/storage/embedding-store.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AAEH;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAqB;IACzD,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,EAAqB,EACrB,MAAc,EACd,MAAoB,EACpB,WAAoB;IAEpB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7E,EAAE;SACC,OAAO,CACN,oFAAoF,CACrF;SACA,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CACnC,EAAqB,EACrB,OAA8E;IAE9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACjC,qBAAqB,CAAC,EAAE,CAAC,CAAC;IAC1B,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,oFAAoF,CACrF,CAAC;IACF,MAAM,UAAU,GAAG,EAAE,CAAC,WAAW,CAC/B,CAAC,IAA2E,EAAE,EAAE;QAC9E,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACnF,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CACF,CAAC;IACF,UAAU,CAAC,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,EAAqB,EAAE,MAAc;IAC3E,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CAAC,uDAAuD,CAAC;SAChE,GAAG,CAAC,MAAM,CAAgD,CAAC;IAC9D,OAAO,GAAG,EAAE,YAAY,IAAI,IAAI,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,EAAqB,EACrB,MAAc,EACd,WAAmB;IAEnB,MAAM,MAAM,GAAG,uBAAuB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACnD,OAAO,MAAM,KAAK,WAAW,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,EAAqB,EAAE,MAAc;IAChE,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC,GAAG,CAAC,MAAM,CAEvE,CAAC;IACd,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAqB;IACpD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,EAGlE,CAAC;IACJ,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAqB;IACnD,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACtF,OAAO,GAAG,CAAC,CAAC,CAAC;AACf,CAAC;AAED,iFAAiF;AAEjF,SAAS,YAAY,CAAC,GAAW;IAC/B,mEAAmE;IACnE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IAC7E,OAAO,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
2
|
-
|
|
3
|
-
describe('embed tool exports', () => {
|
|
4
|
-
it('runEmbed is exported from mcp-tools index', async () => {
|
|
5
|
-
const mod = await import('../../mcp-tools/index.js');
|
|
6
|
-
expect(typeof (mod as any).runEmbed).toBe('function');
|
|
7
|
-
});
|
|
8
|
-
|
|
9
|
-
it('runEmbed is directly importable from embed.js', async () => {
|
|
10
|
-
const { runEmbed } = await import('../../mcp-tools/embed.js');
|
|
11
|
-
expect(typeof runEmbed).toBe('function');
|
|
12
|
-
});
|
|
13
|
-
});
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import Database from 'better-sqlite3';
|
|
3
|
-
import { upsertEmbedding, getEmbeddingContentHash, isEmbeddingStale } from '../../storage/embedding-store.js';
|
|
4
|
-
|
|
5
|
-
function makeDb(): Database.Database {
|
|
6
|
-
const db = new Database(':memory:');
|
|
7
|
-
db.exec(`
|
|
8
|
-
CREATE TABLE IF NOT EXISTS embeddings (
|
|
9
|
-
node_id TEXT PRIMARY KEY,
|
|
10
|
-
vector BLOB NOT NULL,
|
|
11
|
-
content_hash TEXT,
|
|
12
|
-
created_at INTEGER DEFAULT (unixepoch())
|
|
13
|
-
);
|
|
14
|
-
`);
|
|
15
|
-
return db;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
describe('embedding stale detection', () => {
|
|
19
|
-
it('stores content hash with embedding', () => {
|
|
20
|
-
const db = makeDb();
|
|
21
|
-
upsertEmbedding(db, 'n1', new Float32Array([0.1, 0.2]), 'hash_abc');
|
|
22
|
-
const stored = getEmbeddingContentHash(db, 'n1');
|
|
23
|
-
expect(stored).toBe('hash_abc');
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
it('isEmbeddingStale returns true when hash differs', () => {
|
|
27
|
-
const db = makeDb();
|
|
28
|
-
upsertEmbedding(db, 'n1', new Float32Array([0.1, 0.2]), 'old_hash');
|
|
29
|
-
expect(isEmbeddingStale(db, 'n1', 'new_hash')).toBe(true);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it('isEmbeddingStale returns false when hash matches', () => {
|
|
33
|
-
const db = makeDb();
|
|
34
|
-
upsertEmbedding(db, 'n1', new Float32Array([0.1, 0.2]), 'same_hash');
|
|
35
|
-
expect(isEmbeddingStale(db, 'n1', 'same_hash')).toBe(false);
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
it('isEmbeddingStale returns true when no embedding exists', () => {
|
|
39
|
-
const db = makeDb();
|
|
40
|
-
expect(isEmbeddingStale(db, 'n1', 'any_hash')).toBe(true);
|
|
41
|
-
});
|
|
42
|
-
});
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import Database from 'better-sqlite3';
|
|
3
|
-
import { exactVectorSearch } from '../../search/exact-search.js';
|
|
4
|
-
|
|
5
|
-
function makeDb(): Database.Database {
|
|
6
|
-
const db = new Database(':memory:');
|
|
7
|
-
db.exec(`
|
|
8
|
-
CREATE TABLE nodes (id TEXT PRIMARY KEY, label TEXT NOT NULL, name TEXT NOT NULL,
|
|
9
|
-
norm_label TEXT, file_path TEXT, start_line INTEGER, end_line INTEGER,
|
|
10
|
-
community_id INTEGER, is_exported INTEGER DEFAULT 0, language TEXT, properties TEXT);
|
|
11
|
-
CREATE TABLE embeddings (node_id TEXT PRIMARY KEY, vector BLOB NOT NULL, content_hash TEXT, created_at INTEGER);
|
|
12
|
-
`);
|
|
13
|
-
db.prepare(`INSERT INTO nodes VALUES ('n1','Function','doAuth',null,'/auth.ts',1,5,null,1,null,null)`).run();
|
|
14
|
-
db.prepare(`INSERT INTO nodes VALUES ('n2','Function','renderUI',null,'/ui.ts',1,3,null,1,null,null)`).run();
|
|
15
|
-
|
|
16
|
-
// Store simple 3-dim embeddings: n1 ~ [1, 0, 0], n2 ~ [0, 1, 0]
|
|
17
|
-
const v1 = new Float32Array([1, 0, 0]);
|
|
18
|
-
const v2 = new Float32Array([0, 1, 0]);
|
|
19
|
-
db.prepare(`INSERT INTO embeddings VALUES ('n1', ?, null, 0)`).run(Buffer.from(v1.buffer));
|
|
20
|
-
db.prepare(`INSERT INTO embeddings VALUES ('n2', ?, null, 0)`).run(Buffer.from(v2.buffer));
|
|
21
|
-
return db;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
describe('exactVectorSearch', () => {
|
|
25
|
-
it('returns closest node to query vector', () => {
|
|
26
|
-
const db = makeDb();
|
|
27
|
-
const query = new Float32Array([0.9, 0.1, 0]);
|
|
28
|
-
const results = exactVectorSearch(db, query, { limit: 2 });
|
|
29
|
-
expect(results[0]?.id).toBe('n1');
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it('respects limit', () => {
|
|
33
|
-
const db = makeDb();
|
|
34
|
-
const query = new Float32Array([1, 0, 0]);
|
|
35
|
-
const results = exactVectorSearch(db, query, { limit: 1 });
|
|
36
|
-
expect(results).toHaveLength(1);
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
it('returns score between 0 and 1', () => {
|
|
40
|
-
const db = makeDb();
|
|
41
|
-
const query = new Float32Array([1, 0, 0]);
|
|
42
|
-
const results = exactVectorSearch(db, query, { limit: 2 });
|
|
43
|
-
expect(results[0]!.score).toBeGreaterThan(0);
|
|
44
|
-
expect(results[0]!.score).toBeLessThanOrEqual(1);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it('returns empty array when no embeddings', () => {
|
|
48
|
-
const db = new Database(':memory:');
|
|
49
|
-
db.exec(`CREATE TABLE embeddings (node_id TEXT PRIMARY KEY, vector BLOB NOT NULL, content_hash TEXT, created_at INTEGER);
|
|
50
|
-
CREATE TABLE nodes (id TEXT PRIMARY KEY, label TEXT NOT NULL, name TEXT NOT NULL, norm_label TEXT, file_path TEXT, start_line INTEGER, end_line INTEGER, community_id INTEGER, is_exported INTEGER DEFAULT 0, language TEXT, properties TEXT);`);
|
|
51
|
-
const results = exactVectorSearch(db, new Float32Array([1, 0, 0]), { limit: 5 });
|
|
52
|
-
expect(results).toHaveLength(0);
|
|
53
|
-
});
|
|
54
|
-
});
|
package/src/mcp-tools/embed.ts
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* monograph_embed MCP tool
|
|
3
|
-
*
|
|
4
|
-
* Embeds all symbol nodes in the Monograph knowledge graph using
|
|
5
|
-
* Snowflake/snowflake-arctic-embed-xs (384 dimensions).
|
|
6
|
-
*
|
|
7
|
-
* Requires @huggingface/transformers to be installed.
|
|
8
|
-
* Returns { embedded, skipped, model } on success or an error message.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import type Database from 'better-sqlite3';
|
|
12
|
-
import type { EmbedAllResult } from '../search/embed-batch.js';
|
|
13
|
-
|
|
14
|
-
export interface EmbedToolInput {
|
|
15
|
-
codeOnly?: boolean;
|
|
16
|
-
force?: boolean;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export interface EmbedToolResult {
|
|
20
|
-
embedded: number;
|
|
21
|
-
skipped: number;
|
|
22
|
-
model: string;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const MODEL = 'Snowflake/snowflake-arctic-embed-xs';
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* Run the embedding pipeline on the open database.
|
|
29
|
-
* Exported so the CLI handler can call it after opening the DB.
|
|
30
|
-
*/
|
|
31
|
-
export async function runEmbed(
|
|
32
|
-
db: Database.Database,
|
|
33
|
-
options: EmbedToolInput = {},
|
|
34
|
-
): Promise<EmbedToolResult> {
|
|
35
|
-
const { force = false, codeOnly = false } = options;
|
|
36
|
-
|
|
37
|
-
let getEmbedder: () => Promise<unknown>;
|
|
38
|
-
let embedAll: (db: Database.Database, embedder: unknown, force: boolean, codeOnly: boolean) => Promise<EmbedAllResult>;
|
|
39
|
-
|
|
40
|
-
try {
|
|
41
|
-
const embedderMod = await import('../search/embedder.js');
|
|
42
|
-
const batchMod = await import('../search/embed-batch.js');
|
|
43
|
-
getEmbedder = embedderMod.getEmbedder;
|
|
44
|
-
embedAll = batchMod.embedAll as typeof embedAll;
|
|
45
|
-
} catch {
|
|
46
|
-
throw new Error(
|
|
47
|
-
'@huggingface/transformers is required for embedding. ' +
|
|
48
|
-
'Install it with: npm install @huggingface/transformers',
|
|
49
|
-
);
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
const embedder = await getEmbedder();
|
|
53
|
-
const result = await embedAll(db, embedder as Parameters<typeof embedAll>[1], force, codeOnly);
|
|
54
|
-
|
|
55
|
-
return { ...result, model: MODEL };
|
|
56
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type { PipelinePhase, PipelineContext } from '../types.js';
|
|
2
|
-
// buildEmbeddings is not yet in embedding-store; phase is a no-op until it's added
|
|
3
|
-
function buildEmbeddings(_db: unknown): void { /* placeholder */ }
|
|
4
|
-
|
|
5
|
-
export interface EmbeddingsOutput {
|
|
6
|
-
embeddingsBuilt: number;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export const embeddingsPhase: PipelinePhase<EmbeddingsOutput> = {
|
|
10
|
-
name: 'embeddings',
|
|
11
|
-
deps: ['communities', 'docs-parse', 'pdf-parse', 'contextual-proximity', 'llm-extract'],
|
|
12
|
-
async execute(ctx: PipelineContext) {
|
|
13
|
-
buildEmbeddings(ctx.db);
|
|
14
|
-
const row = ctx.db.prepare('SELECT COUNT(*) as n FROM nodes WHERE embedding IS NOT NULL').get() as { n: number };
|
|
15
|
-
return { embeddingsBuilt: row.n };
|
|
16
|
-
},
|
|
17
|
-
};
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
export type EmbedDevice = 'cpu' | 'cuda' | 'mps';
|
|
2
|
-
|
|
3
|
-
export interface EmbedDeviceConfig {
|
|
4
|
-
device?: EmbedDevice;
|
|
5
|
-
threads?: number;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Resolves the compute device to use for local embedders.
|
|
10
|
-
*
|
|
11
|
-
* Priority:
|
|
12
|
-
* 1. Explicit `config.device` if provided
|
|
13
|
-
* 2. 'mps' on macOS (darwin) when not explicitly overridden
|
|
14
|
-
* 3. 'cuda' when CUDA_VISIBLE_DEVICES env var is set (non-empty)
|
|
15
|
-
* 4. 'cpu' as fallback
|
|
16
|
-
*/
|
|
17
|
-
export function resolveDevice(config?: EmbedDeviceConfig): EmbedDevice {
|
|
18
|
-
if (config?.device) {
|
|
19
|
-
return config.device;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
if (process.platform === 'darwin') {
|
|
23
|
-
return 'mps';
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
if (process.env['CUDA_VISIBLE_DEVICES']) {
|
|
27
|
-
return 'cuda';
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
return 'cpu';
|
|
31
|
-
}
|