@jafreck/lore 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/index.d.ts +1 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -2
- package/dist/index.js.map +1 -1
- package/dist/indexer/embedder.d.ts +20 -31
- package/dist/indexer/embedder.d.ts.map +1 -1
- package/dist/indexer/embedder.js +34 -152
- package/dist/indexer/embedder.js.map +1 -1
- package/dist/lore-server/server.d.ts.map +1 -1
- package/dist/lore-server/server.js +3 -3
- package/dist/lore-server/server.js.map +1 -1
- package/dist/runtime.js +2 -2
- package/dist/runtime.js.map +1 -1
- package/package.json +2 -1
- package/dist/indexer/ensure-python-deps.d.ts +0 -22
- package/dist/indexer/ensure-python-deps.d.ts.map +0 -1
- package/dist/indexer/ensure-python-deps.js +0 -47
- package/dist/indexer/ensure-python-deps.js.map +0 -1
package/README.md
CHANGED
|
@@ -378,13 +378,14 @@ npx @jafreck/lore ingest-coverage --db ./lore.db --root ./my-project \
|
|
|
378
378
|
|
|
379
379
|
### Embeddings
|
|
380
380
|
|
|
381
|
-
Lore optionally generates dense vector embeddings for semantic search using
|
|
382
|
-
|
|
383
|
-
|
|
381
|
+
Lore optionally generates dense vector embeddings for semantic search using
|
|
382
|
+
`@huggingface/transformers` (Transformers.js), which runs ONNX models natively
|
|
383
|
+
in Node.js — no Python or external processes required. Specify the model with
|
|
384
|
+
`--embedding-model`:
|
|
384
385
|
|
|
385
386
|
```bash
|
|
386
387
|
npx @jafreck/lore index --root ./my-project --db ./lore.db \
|
|
387
|
-
--embedding-model '
|
|
388
|
+
--embedding-model 'nomic-ai/nomic-embed-text-v1.5'
|
|
388
389
|
```
|
|
389
390
|
|
|
390
391
|
At query time, `lore_search` in `semantic` or `fused` mode embeds the query
|
package/dist/index.d.ts
CHANGED
|
@@ -16,11 +16,10 @@ export { resolveSymbolEdges, topoSort, detectCycles } from './indexer/call-graph
|
|
|
16
16
|
export { walkFiles, detectLanguageForPath } from './indexer/walker.js';
|
|
17
17
|
export type { WalkerConfig, FileEntry } from './indexer/walker.js';
|
|
18
18
|
export { ImportResolver } from './indexer/resolver.js';
|
|
19
|
-
export { ensurePythonDeps } from './indexer/ensure-python-deps.js';
|
|
20
19
|
export { installGitHooks } from './indexer/git-hooks.js';
|
|
21
20
|
export type { InstallGitHooksOptions } from './indexer/git-hooks.js';
|
|
22
21
|
export type { EffectiveLspSettings, LspSettingsOverrides } from './indexer/lsp/config.js';
|
|
23
|
-
export {
|
|
22
|
+
export { TransformersJsProvider, DEFAULT_EMBEDDING_MODEL } from './indexer/embedder.js';
|
|
24
23
|
export type { EmbeddingProvider } from './indexer/embedder.js';
|
|
25
24
|
export type { ExtractionResult, RawCallRef, RawImport, RawSymbol, SymbolExtractor, } from './indexer/extractors/types.js';
|
|
26
25
|
export { RESOLUTION_METHODS, RESOLVED_METHODS, UNRESOLVED_METHODS } from './indexer/resolution-method.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrF,YAAY,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACvE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACrF,YAAY,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACvE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,YAAY,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACrE,YAAY,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AACxF,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,YAAY,EACV,gBAAgB,EAChB,UAAU,EACV,SAAS,EACT,SAAS,EACT,eAAe,GAChB,MAAM,+BAA+B,CAAC;AAGvC,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAC1G,YAAY,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAGvE,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,GACf,MAAM,2BAA2B,CAAC;AAInC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG7D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACxF,YAAY,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,YAAY,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AACjF,OAAO,EACL,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,aAAa,EACb,SAAS,EACT,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAGtG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACxG,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG9F,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -15,9 +15,8 @@ export { openDb, setLoreMeta, getLoreMeta, createVec0Tables } from './indexer/db
|
|
|
15
15
|
export { resolveSymbolEdges, topoSort, detectCycles } from './indexer/call-graph.js';
|
|
16
16
|
export { walkFiles, detectLanguageForPath } from './indexer/walker.js';
|
|
17
17
|
export { ImportResolver } from './indexer/resolver.js';
|
|
18
|
-
export { ensurePythonDeps } from './indexer/ensure-python-deps.js';
|
|
19
18
|
export { installGitHooks } from './indexer/git-hooks.js';
|
|
20
|
-
export {
|
|
19
|
+
export { TransformersJsProvider, DEFAULT_EMBEDDING_MODEL } from './indexer/embedder.js';
|
|
21
20
|
// ── Resolution method taxonomy (shared constant) ─────────────────────────────
|
|
22
21
|
export { RESOLUTION_METHODS, RESOLVED_METHODS, UNRESOLVED_METHODS } from './indexer/resolution-method.js';
|
|
23
22
|
// ── Pipeline (composable indexing stages) ─────────────────────────────────────
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,iFAAiF;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAErF,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,iFAAiF;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAErF,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAGzD,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAUxF,gFAAgF;AAChF,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAG1G,iFAAiF;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,GACf,MAAM,2BAA2B,CAAC;AAGnC,iFAAiF;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAG3C,iFAAiF;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGjD,iFAAiF;AACjF,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAGxF,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,EACL,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,aAAa,EACb,SAAS,EACT,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAG7B,iFAAiF;AACjF,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGxG,iFAAiF;AACjF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
* @module indexer/embedder
|
|
3
3
|
*
|
|
4
4
|
* Provides an `EmbeddingProvider` abstraction for generating dense vector
|
|
5
|
-
* embeddings from text. The primary implementation (`
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* embeddings from text. The primary implementation (`TransformersJsProvider`)
|
|
6
|
+
* uses the `@huggingface/transformers` library to run ONNX models natively
|
|
7
|
+
* in Node.js — no Python or external processes required.
|
|
8
8
|
*
|
|
9
|
-
* The model's embedding dimensionality is auto-detected at startup
|
|
10
|
-
*
|
|
9
|
+
* The model's embedding dimensionality is auto-detected at startup by
|
|
10
|
+
* embedding a probe sentence and inspecting the output shape.
|
|
11
11
|
*/
|
|
12
12
|
export interface EmbeddingProvider {
|
|
13
13
|
/** Embed a batch of texts; returns one float vector per input. */
|
|
@@ -20,11 +20,11 @@ export interface EmbeddingProvider {
|
|
|
20
20
|
*/
|
|
21
21
|
readonly dims: number;
|
|
22
22
|
/**
|
|
23
|
-
*
|
|
23
|
+
* Load the model and detect dimensionality.
|
|
24
24
|
* Must be called (and awaited) before the first `embed()` call.
|
|
25
25
|
*/
|
|
26
26
|
init(): Promise<void>;
|
|
27
|
-
/** Release the underlying
|
|
27
|
+
/** Release the underlying resources. */
|
|
28
28
|
dispose(): Promise<void>;
|
|
29
29
|
}
|
|
30
30
|
export interface StructuralEmbeddingInput {
|
|
@@ -35,43 +35,32 @@ export interface StructuralEmbeddingInput {
|
|
|
35
35
|
}
|
|
36
36
|
export declare function buildStructuralEmbeddingText(input: StructuralEmbeddingInput): string;
|
|
37
37
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
38
|
+
* Generates embeddings using `@huggingface/transformers` (Transformers.js).
|
|
39
|
+
* Runs ONNX models natively in Node — zero Python dependency.
|
|
40
40
|
*
|
|
41
|
-
* Call `init()` first to
|
|
42
|
-
* dimensionality.
|
|
43
|
-
* provider for efficiency.
|
|
41
|
+
* Call `init()` first to download/load the model and detect its embedding
|
|
42
|
+
* dimensionality. The model is kept in memory for the provider's lifetime.
|
|
44
43
|
*/
|
|
45
|
-
export declare class
|
|
44
|
+
export declare class TransformersJsProvider implements EmbeddingProvider {
|
|
46
45
|
readonly modelName: string;
|
|
47
46
|
private _dims;
|
|
48
|
-
private
|
|
49
|
-
private rl;
|
|
50
|
-
private readonly pendingRequests;
|
|
51
|
-
private readonly pythonBin;
|
|
52
|
-
/** Whether the first stdout line (dims handshake) has been consumed. */
|
|
47
|
+
private _pipeline;
|
|
53
48
|
private initialized;
|
|
54
|
-
constructor(modelName: string
|
|
49
|
+
constructor(modelName: string);
|
|
55
50
|
/** Embedding dimensionality — available only after `init()`. */
|
|
56
51
|
get dims(): number;
|
|
57
52
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
53
|
+
* Load the model via Transformers.js and detect embedding dimensionality
|
|
54
|
+
* by running a single probe sentence.
|
|
60
55
|
*/
|
|
61
56
|
init(): Promise<void>;
|
|
62
57
|
embed(texts: string[]): Promise<number[][]>;
|
|
63
58
|
dispose(): Promise<void>;
|
|
64
|
-
/** Spawn the Python subprocess and wire up the readline interface. */
|
|
65
|
-
private spawnProcess;
|
|
66
59
|
}
|
|
67
|
-
/** Default embedding model used when no model is explicitly specified. */
|
|
68
|
-
export declare const DEFAULT_EMBEDDING_MODEL = "Qwen/Qwen3-Embedding-4B";
|
|
69
60
|
/**
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* @param size Model size variant: `'0.6B'`, `'4B'`, or `'8B'`.
|
|
74
|
-
* @param pythonBin Path to the Python executable (default: `'python3'`).
|
|
61
|
+
* Default embedding model — `nomic-ai/nomic-embed-text-v1.5` is a high-quality,
|
|
62
|
+
* 768-dim model with ONNX weights pre-packaged for Transformers.js.
|
|
63
|
+
* Supports Matryoshka dimensions (768, 512, 256, 128, 64).
|
|
75
64
|
*/
|
|
76
|
-
export declare
|
|
65
|
+
export declare const DEFAULT_EMBEDDING_MODEL = "nomic-ai/nomic-embed-text-v1.5";
|
|
77
66
|
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/indexer/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/indexer/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,uCAAuC;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,wCAAwC;IACxC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CASpF;AASD;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,SAAS,CAA0C;IAC3D,OAAO,CAAC,WAAW,CAAS;gBAEhB,SAAS,EAAE,MAAM;IAI7B,gEAAgE;IAChE,IAAI,IAAI,IAAI,MAAM,CAKjB;IAED;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAarB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAS3C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ/B;AAID;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,mCAAmC,CAAC"}
|
package/dist/indexer/embedder.js
CHANGED
|
@@ -2,16 +2,13 @@
|
|
|
2
2
|
* @module indexer/embedder
|
|
3
3
|
*
|
|
4
4
|
* Provides an `EmbeddingProvider` abstraction for generating dense vector
|
|
5
|
-
* embeddings from text. The primary implementation (`
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* embeddings from text. The primary implementation (`TransformersJsProvider`)
|
|
6
|
+
* uses the `@huggingface/transformers` library to run ONNX models natively
|
|
7
|
+
* in Node.js — no Python or external processes required.
|
|
8
8
|
*
|
|
9
|
-
* The model's embedding dimensionality is auto-detected at startup
|
|
10
|
-
*
|
|
9
|
+
* The model's embedding dimensionality is auto-detected at startup by
|
|
10
|
+
* embedding a probe sentence and inspecting the output shape.
|
|
11
11
|
*/
|
|
12
|
-
import { spawn } from 'node:child_process';
|
|
13
|
-
import * as readline from 'node:readline';
|
|
14
|
-
import { trackProcess } from '../process-tracker.js';
|
|
15
12
|
export function buildStructuralEmbeddingText(input) {
|
|
16
13
|
const parts = [
|
|
17
14
|
input.signature?.trim() ?? '',
|
|
@@ -22,51 +19,20 @@ export function buildStructuralEmbeddingText(input) {
|
|
|
22
19
|
const uniqueParts = [...new Set(parts)];
|
|
23
20
|
return uniqueParts.join('\n');
|
|
24
21
|
}
|
|
25
|
-
// ─── Python bootstrap script ──────────────────────────────────────────────────
|
|
26
22
|
/**
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* 2. Prints `{"dims": <N>}` on the first stdout line.
|
|
30
|
-
* 3. Enters an NDJSON request loop (stdin → stdout).
|
|
23
|
+
* Generates embeddings using `@huggingface/transformers` (Transformers.js).
|
|
24
|
+
* Runs ONNX models natively in Node — zero Python dependency.
|
|
31
25
|
*
|
|
32
|
-
*
|
|
26
|
+
* Call `init()` first to download/load the model and detect its embedding
|
|
27
|
+
* dimensionality. The model is kept in memory for the provider's lifetime.
|
|
33
28
|
*/
|
|
34
|
-
|
|
35
|
-
import sys, json
|
|
36
|
-
from sentence_transformers import SentenceTransformer
|
|
37
|
-
model = SentenceTransformer(sys.argv[1], trust_remote_code=True)
|
|
38
|
-
dims = model.get_sentence_embedding_dimension()
|
|
39
|
-
sys.stdout.write(json.dumps({"dims": dims}) + "\\n")
|
|
40
|
-
sys.stdout.flush()
|
|
41
|
-
for line in sys.stdin:
|
|
42
|
-
line = line.strip()
|
|
43
|
-
if not line:
|
|
44
|
-
continue
|
|
45
|
-
req = json.loads(line)
|
|
46
|
-
vecs = model.encode(req.get('texts', []), normalize_embeddings=True).tolist()
|
|
47
|
-
sys.stdout.write(json.dumps({'embeddings': vecs}) + '\\n')
|
|
48
|
-
sys.stdout.flush()
|
|
49
|
-
`.trimStart();
|
|
50
|
-
/**
|
|
51
|
-
* Communicates with a Python subprocess via stdin/stdout NDJSON to produce
|
|
52
|
-
* embeddings using a sentence-transformers compatible model.
|
|
53
|
-
*
|
|
54
|
-
* Call `init()` first to spawn the process and detect the model's embedding
|
|
55
|
-
* dimensionality. The subprocess is kept alive for the lifetime of the
|
|
56
|
-
* provider for efficiency.
|
|
57
|
-
*/
|
|
58
|
-
export class SentenceTransformersProvider {
|
|
29
|
+
export class TransformersJsProvider {
|
|
59
30
|
modelName;
|
|
60
31
|
_dims = null;
|
|
61
|
-
|
|
62
|
-
rl = null;
|
|
63
|
-
pendingRequests = [];
|
|
64
|
-
pythonBin;
|
|
65
|
-
/** Whether the first stdout line (dims handshake) has been consumed. */
|
|
32
|
+
_pipeline = null;
|
|
66
33
|
initialized = false;
|
|
67
|
-
constructor(modelName
|
|
34
|
+
constructor(modelName) {
|
|
68
35
|
this.modelName = modelName;
|
|
69
|
-
this.pythonBin = pythonBin;
|
|
70
36
|
}
|
|
71
37
|
/** Embedding dimensionality — available only after `init()`. */
|
|
72
38
|
get dims() {
|
|
@@ -76,128 +42,44 @@ export class SentenceTransformersProvider {
|
|
|
76
42
|
return this._dims;
|
|
77
43
|
}
|
|
78
44
|
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
45
|
+
* Load the model via Transformers.js and detect embedding dimensionality
|
|
46
|
+
* by running a single probe sentence.
|
|
81
47
|
*/
|
|
82
48
|
async init() {
|
|
83
49
|
if (this.initialized)
|
|
84
50
|
return;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
this._dims = msg.dims;
|
|
93
|
-
this.initialized = true;
|
|
94
|
-
resolve();
|
|
95
|
-
}
|
|
96
|
-
else {
|
|
97
|
-
reject(new Error(`Unexpected handshake from embedding subprocess: ${line}`));
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
catch (err) {
|
|
101
|
-
reject(new Error(`Failed to parse embedding handshake: ${line}`));
|
|
102
|
-
}
|
|
103
|
-
};
|
|
104
|
-
// Read exactly one line for the handshake, then re-wire for embed requests.
|
|
105
|
-
this.rl.once('line', onLine);
|
|
106
|
-
// If the process dies before the handshake, reject.
|
|
107
|
-
this.proc.once('exit', (code) => {
|
|
108
|
-
if (!this.initialized) {
|
|
109
|
-
reject(new Error(`Embedding subprocess exited with code ${code} before handshake`));
|
|
110
|
-
}
|
|
111
|
-
});
|
|
112
|
-
});
|
|
51
|
+
const { pipeline } = await import('@huggingface/transformers');
|
|
52
|
+
this._pipeline = await pipeline('feature-extraction', this.modelName);
|
|
53
|
+
// Probe the model to detect dimensionality.
|
|
54
|
+
const probe = await this._pipeline('dimensionality probe', { pooling: 'mean', normalize: true });
|
|
55
|
+
const probeList = probe.tolist();
|
|
56
|
+
this._dims = probeList[0].length;
|
|
57
|
+
this.initialized = true;
|
|
113
58
|
}
|
|
114
59
|
async embed(texts) {
|
|
115
60
|
if (texts.length === 0)
|
|
116
61
|
return [];
|
|
117
|
-
if (!this.initialized) {
|
|
62
|
+
if (!this.initialized || !this._pipeline) {
|
|
118
63
|
throw new Error('EmbeddingProvider not initialised — call init() first');
|
|
119
64
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
this.proc.stdin.write(JSON.stringify({ texts }) + '\n');
|
|
123
|
-
});
|
|
65
|
+
const output = await this._pipeline(texts, { pooling: 'mean', normalize: true });
|
|
66
|
+
return output.tolist();
|
|
124
67
|
}
|
|
125
68
|
async dispose() {
|
|
126
|
-
if (
|
|
127
|
-
return;
|
|
128
|
-
const proc = this.proc;
|
|
129
|
-
// Reject all pending requests before tearing down.
|
|
130
|
-
const pending = this.pendingRequests.splice(0);
|
|
131
|
-
const err = new Error('EmbeddingProvider disposed');
|
|
132
|
-
for (const r of pending)
|
|
133
|
-
r.reject(err);
|
|
134
|
-
this.proc = null;
|
|
135
|
-
this.rl?.close();
|
|
136
|
-
this.rl = null;
|
|
137
|
-
proc.stdin?.end();
|
|
138
|
-
await new Promise(resolve => {
|
|
139
|
-
const timeout = setTimeout(() => { proc.kill(); resolve(); }, 5_000);
|
|
140
|
-
proc.once('close', () => { clearTimeout(timeout); resolve(); });
|
|
141
|
-
});
|
|
142
|
-
}
|
|
143
|
-
/** Spawn the Python subprocess and wire up the readline interface. */
|
|
144
|
-
spawnProcess() {
|
|
145
|
-
if (this.proc)
|
|
146
|
-
return;
|
|
147
|
-
this.proc = spawn(this.pythonBin, ['-c', BOOTSTRAP_SCRIPT, this.modelName], {
|
|
148
|
-
stdio: ['pipe', 'pipe', 'inherit'],
|
|
149
|
-
});
|
|
150
|
-
trackProcess(this.proc);
|
|
151
|
-
this.rl = readline.createInterface({ input: this.proc.stdout });
|
|
152
|
-
// After init(), all subsequent lines are embed responses.
|
|
153
|
-
this.rl.on('line', (line) => {
|
|
154
|
-
// Skip lines until init handshake is done (handled by init's once listener).
|
|
155
|
-
if (!this.initialized)
|
|
156
|
-
return;
|
|
157
|
-
const pending = this.pendingRequests.shift();
|
|
158
|
-
if (!pending)
|
|
159
|
-
return;
|
|
69
|
+
if (this._pipeline) {
|
|
160
70
|
try {
|
|
161
|
-
|
|
162
|
-
pending.resolve(embeddings);
|
|
71
|
+
await this._pipeline.dispose?.();
|
|
163
72
|
}
|
|
164
|
-
catch
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
});
|
|
168
|
-
this.proc.on('error', (err) => {
|
|
169
|
-
const reqs = this.pendingRequests.splice(0);
|
|
170
|
-
this.proc = null;
|
|
171
|
-
this.rl = null;
|
|
172
|
-
for (const r of reqs)
|
|
173
|
-
r.reject(err);
|
|
174
|
-
});
|
|
175
|
-
this.proc.on('exit', (code) => {
|
|
176
|
-
if (code !== 0 && this.pendingRequests.length > 0) {
|
|
177
|
-
const err = new Error(`Embedding subprocess exited with code ${code}`);
|
|
178
|
-
const reqs = this.pendingRequests.splice(0);
|
|
179
|
-
for (const r of reqs)
|
|
180
|
-
r.reject(err);
|
|
181
|
-
}
|
|
182
|
-
if (this.proc) {
|
|
183
|
-
this.proc = null;
|
|
184
|
-
this.rl = null;
|
|
185
|
-
}
|
|
186
|
-
});
|
|
73
|
+
catch { /* best-effort cleanup */ }
|
|
74
|
+
this._pipeline = null;
|
|
75
|
+
}
|
|
187
76
|
}
|
|
188
77
|
}
|
|
189
|
-
// ───
|
|
190
|
-
/** Default embedding model used when no model is explicitly specified. */
|
|
191
|
-
export const DEFAULT_EMBEDDING_MODEL = 'Qwen/Qwen3-Embedding-4B';
|
|
78
|
+
// ─── Default model ────────────────────────────────────────────────────────────
|
|
192
79
|
/**
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
* @param size Model size variant: `'0.6B'`, `'4B'`, or `'8B'`.
|
|
197
|
-
* @param pythonBin Path to the Python executable (default: `'python3'`).
|
|
80
|
+
* Default embedding model — `nomic-ai/nomic-embed-text-v1.5` is a high-quality,
|
|
81
|
+
* 768-dim model with ONNX weights pre-packaged for Transformers.js.
|
|
82
|
+
* Supports Matryoshka dimensions (768, 512, 256, 128, 64).
|
|
198
83
|
*/
|
|
199
|
-
export
|
|
200
|
-
const modelName = `Qwen/Qwen3-Embedding-${size}`;
|
|
201
|
-
return new SentenceTransformersProvider(modelName, pythonBin);
|
|
202
|
-
}
|
|
84
|
+
export const DEFAULT_EMBEDDING_MODEL = 'nomic-ai/nomic-embed-text-v1.5';
|
|
203
85
|
//# sourceMappingURL=embedder.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/indexer/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/indexer/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AA8BH,MAAM,UAAU,4BAA4B,CAAC,KAA+B;IAC1E,MAAM,KAAK,GAAG;QACZ,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE;QAC7B,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE;QACzC,KAAK,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,EAAE;QACtC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE;KAClB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AASD;;;;;;GAMG;AACH,MAAM,OAAO,sBAAsB;IACxB,SAAS,CAAS;IACnB,KAAK,GAAkB,IAAI,CAAC;IAC5B,SAAS,GAAqC,IAAI,CAAC;IACnD,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,SAAiB;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,gEAAgE;IAChE,IAAI,IAAI;QACN,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,GAAG,MAAO,QAAuB,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtF,4CAA4C;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjG,MAAM,SAAS,GAAI,KAAkC,CAAC,MAAM,EAAE,CAAC;QAC/D,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,OAAQ,MAAmC,CAAC,MAAM,EAAE,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAO,IAAI,CAAC,SAA0D,CAAC,OAAO,EAAE,EAAE,CAAC;YACrF,CAAC;YAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;YACrC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;CACF;AAED,iFAAiF;AAEjF;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,gCAAgC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/lore-server/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,SAAS,CAAC;AAEjB,OAAO,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/lore-server/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,SAAS,CAAC;AAEjB,OAAO,EAA0B,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAa,KAAK,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAoBxD,wDAAwD;AACxD,MAAM,WAAW,iBAAiB;IAChC,+FAA+F;IAC/F,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAID;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,iBAAiB,EAC5B,OAAO,CAAC,EAAE,iBAAiB,GAC1B,SAAS,CAsBX;AAED;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,iBAAiB,EAC5B,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,SAAS,CAAC,CAYpB"}
|
|
@@ -27,7 +27,7 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
|
|
|
27
27
|
import { fileURLToPath } from 'url';
|
|
28
28
|
import { openReadOnly, } from './db.js';
|
|
29
29
|
import { getLoreMeta } from '../indexer/db.js';
|
|
30
|
-
import {
|
|
30
|
+
import { TransformersJsProvider } from '../indexer/embedder.js';
|
|
31
31
|
import { getLogger } from '../logger.js';
|
|
32
32
|
import { buildToolModules, registerTools } from './tool-registry.js';
|
|
33
33
|
import * as lookup from './tools/lookup.js';
|
|
@@ -110,14 +110,14 @@ function buildToolModulesSync() {
|
|
|
110
110
|
// ─── Embedding helper ─────────────────────────────────────────────────────────
|
|
111
111
|
/**
|
|
112
112
|
* Read the embedding model stored in `lore_meta` at index time and spin up a
|
|
113
|
-
* `
|
|
113
|
+
* `TransformersJsProvider` instance for it. Returns `undefined` when no
|
|
114
114
|
* embedding model is recorded in the database.
|
|
115
115
|
*/
|
|
116
116
|
async function buildEmbedder(db) {
|
|
117
117
|
const modelName = getLoreMeta(db, 'embedding_model');
|
|
118
118
|
if (!modelName)
|
|
119
119
|
return undefined;
|
|
120
|
-
const provider = new
|
|
120
|
+
const provider = new TransformersJsProvider(modelName);
|
|
121
121
|
try {
|
|
122
122
|
await provider.init();
|
|
123
123
|
return provider;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/lore-server/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EACL,YAAY,GAEb,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/lore-server/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EACL,YAAY,GAEb,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAA0B,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,SAAS,EAAmB,MAAM,cAAc,CAAC;AAE1D,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAmB,MAAM,oBAAoB,CAAC;AACtF,OAAO,KAAK,MAAM,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,KAAK,MAAM,kBAAkB,CAAC;AAC1C,OAAO,KAAK,MAAM,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,MAAM,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,KAAK,MAAM,kBAAkB,CAAC;AAC1C,OAAO,KAAK,YAAY,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,OAAO,MAAM,qBAAqB,CAAC;AAC/C,OAAO,KAAK,OAAO,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,KAAK,MAAM,kBAAkB,CAAC;AAC1C,OAAO,KAAK,OAAO,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAC;AAChD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,OAAO,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,cAAc,MAAM,wBAAwB,CAAC;AAYzD,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CACjC,EAAqB,EACrB,MAAc,EACd,QAA4B,EAC5B,OAA2B;IAE3B,MAAM,GAAG,GAAG,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;IAE3C,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,EACzC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAChC,CAAC;IAEF,4EAA4E;IAC5E,iEAAiE;IACjE,uEAAuE;IACvE,2EAA2E;IAC3E,wEAAwE;IACxE,uEAAuE;IACvE,sCAAsC;IACtC,EAAE;IACF,2EAA2E;IAC3E,0EAA0E;IAC1E,MAAM,WAAW,GAAG,oBAAoB,EAAE,CAAC;IAC3C,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAEnH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,EAAqB,EACrB,MAAc,EACd,QAA4B,EAC5B,OAA2B;IAE3B,MAAM,GAAG,GAAG,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;IAE3C,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,EACzC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAChC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAC7C,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAEnH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,iFAAiF;AAEjF;;;GAGG;AACH,SAAS,oBAAoB;IAC3B,OAAO;QACL,EAAE,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE;QACzG,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QACxF,EAAE,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE;QAC9H,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE;QAC3G,EAAE,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QAC1F,EAAE,GAAG,EAAE,KAAK,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;QACtG,EAAE,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QAChG,EAAE,GAAG,EAAE,YAAY,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QACtG,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QAC5F,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QAC5F,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QACxF,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,EAAE;QAClG,EAAE,GAAG,EAAE,QAAQ,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QAC9F,EAAE,GAAG,EAAE,SAAS,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;QACpG,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE;QAC3G,EAAE,GAAG,EAAE,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;KAC3G,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;;GAIG;AACH,KAAK,UAAU,aAAa,CAAC,EAAqB;IAChD,MAAM,SAAS,GAAG,WAAW,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACrD,IAAI,CAAC,SAAS;QAAE,OAAO,SAAS,CAAC;IAEjC,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,SAAS,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,6DAA6D;QAC7D,IAAI,CAAC;YAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACxD,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF,SAAS,SAAS;IAChB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAE,EAAE,CAAC;AACtC,CAAC;AAED,iFAAiF;AAEjF,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;IAE/B,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,EAAE,CAAC,CAAC;IAEzC,MAAM,MAAM,GAAG,mBAAmB,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEzD,uDAAuD;IACvD,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEhC,sDAAsD;IACtD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,gFAAgF;AAChF,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACvD,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/runtime.js
CHANGED
|
@@ -58,8 +58,8 @@ export class LoreRuntime {
|
|
|
58
58
|
// ── Embedder ─────────────────────────────────────────────────────────────
|
|
59
59
|
if (this.config.embeddingModel) {
|
|
60
60
|
try {
|
|
61
|
-
const {
|
|
62
|
-
const provider = new
|
|
61
|
+
const { TransformersJsProvider } = await import('./indexer/embedder.js');
|
|
62
|
+
const provider = new TransformersJsProvider(this.config.embeddingModel);
|
|
63
63
|
await provider.init();
|
|
64
64
|
this._embedder = provider;
|
|
65
65
|
this.log.startup('embedding model loaded', {
|
package/dist/runtime.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,EAAE,SAAS,EAAmB,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAsCtD,iFAAiF;AAEjF;;;;;;;;;;GAUG;AACH,MAAM,OAAO,WAAW;IACb,MAAM,CAAgB;IACtB,GAAG,CAAa;IAEjB,SAAS,GAA6B,IAAI,CAAC;IAC3C,UAAU,GAAqB,IAAI,CAAC;IACpC,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAqB,EAAE,MAAmB;QACpD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,GAAG,GAAG,MAAM,IAAI,SAAS,EAAE,CAAC;IACnC,CAAC;IAED,8EAA8E;IAE9E,8EAA8E;IAC9E,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC;IACrC,CAAC;IAED,kFAAkF;IAClF,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,UAAU,IAAI,SAAS,CAAC;IACtC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,8EAA8E;IAE9E;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,EAAE,SAAS,EAAmB,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAsCtD,iFAAiF;AAEjF;;;;;;;;;;GAUG;AACH,MAAM,OAAO,WAAW;IACb,MAAM,CAAgB;IACtB,GAAG,CAAa;IAEjB,SAAS,GAA6B,IAAI,CAAC;IAC3C,UAAU,GAAqB,IAAI,CAAC;IACpC,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAqB,EAAE,MAAmB;QACpD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,GAAG,GAAG,MAAM,IAAI,SAAS,EAAE,CAAC;IACnC,CAAC;IAED,8EAA8E;IAE9E,8EAA8E;IAC9E,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC;IACrC,CAAC;IAED,kFAAkF;IAClF,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,UAAU,IAAI,SAAS,CAAC;IACtC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,8EAA8E;IAE9E;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;gBACzE,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBACxE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;gBAC1B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE;oBACzC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;oBAC1C,cAAc,EAAE,IAAI;iBACrB,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,GAAG,CAAC,IAAI,CACX,SAAS,EACT,4DAA4D,EAC5D,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAC/C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,4EAA4E;QAC5E,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACxC,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;gBAC5E,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;gBAChD,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;gBACjC,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;aACtC,CAAC,CAAC;YACH,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,oBAAoB,EAAE;gBACrC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS;aACnC,CAAC,CAAC;YACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,GAAG,IAAI,CACzJ,CAAC;QACJ,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YAC9C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;YAC3D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;gBAC1E,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;gBAChD,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,SAAS;gBACjC,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;aACtC,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mBAAmB,EAAE;gBACpC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS;aACnC,CAAC,CAAC;YACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,GAAG,IAAI,CACxJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAEtB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB;YACnB,CAAC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,qBAAqB;QACnB,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,YAAY,EAAE,CAAC;gBACjB,gDAAgD;gBAChD,cAAc,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,YAAY,GAAG,IAAI,CAAC;YAEpB,IAAI,CAAC,QAAQ,EAAE;iBACZ,KAAK,CAAC,GAAG,EAAE,GAAqB,CAAC,CAAC;iBAClC,OAAO,CAAC,GAAG,EAAE;gBACZ,cAAc,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAE/B,+DAA+D;QAC/D,yEAAyE;QACzE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,cAAc,EAAE,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jafreck/lore",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.4",
|
|
4
4
|
"description": "Language-aware codebase indexer — tree-sitter parsing, symbol extraction, call-graph construction, and optional embeddings for semantic search",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -58,6 +58,7 @@
|
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
60
|
"@elm-tooling/tree-sitter-elm": "^5.9.0",
|
|
61
|
+
"@huggingface/transformers": "^3.8.1",
|
|
61
62
|
"@modelcontextprotocol/sdk": "^1.27.1",
|
|
62
63
|
"@tree-sitter-grammars/tree-sitter-lua": "^0.4.1",
|
|
63
64
|
"@tree-sitter-grammars/tree-sitter-zig": "^1.1.2",
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @module indexer/ensure-python-deps
|
|
3
|
-
*
|
|
4
|
-
* Validates that the Python environment has the packages required for
|
|
5
|
-
* embedding generation (`sentence-transformers`, `torch`). If missing,
|
|
6
|
-
* attempts an automatic `pip install` before the first embedding run.
|
|
7
|
-
*
|
|
8
|
-
* This is only used when `loreIndex.embeddings.enabled` is `true` in the
|
|
9
|
-
* migration config.
|
|
10
|
-
*/
|
|
11
|
-
/**
|
|
12
|
-
* Checks whether each of `REQUIRED_PACKAGES` is importable by the given Python
|
|
13
|
-
* binary. For any missing package, runs `pip install sentence-transformers`
|
|
14
|
-
* (which pulls in `torch` as a transitive dependency).
|
|
15
|
-
*
|
|
16
|
-
* Throws if the Python binary is not found or if `pip install` fails.
|
|
17
|
-
*
|
|
18
|
-
* @param pythonBin Path to the Python 3 interpreter (default: `'python3'`).
|
|
19
|
-
* @param timeout Maximum milliseconds to wait for the install (default: 5 min).
|
|
20
|
-
*/
|
|
21
|
-
export declare function ensurePythonDeps(pythonBin?: string, timeout?: number): Promise<void>;
|
|
22
|
-
//# sourceMappingURL=ensure-python-deps.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ensure-python-deps.d.ts","sourceRoot":"","sources":["../../src/indexer/ensure-python-deps.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAUH;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CACpC,SAAS,SAAY,EACrB,OAAO,SAAa,GACnB,OAAO,CAAC,IAAI,CAAC,CAyBf"}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @module indexer/ensure-python-deps
|
|
3
|
-
*
|
|
4
|
-
* Validates that the Python environment has the packages required for
|
|
5
|
-
* embedding generation (`sentence-transformers`, `torch`). If missing,
|
|
6
|
-
* attempts an automatic `pip install` before the first embedding run.
|
|
7
|
-
*
|
|
8
|
-
* This is only used when `loreIndex.embeddings.enabled` is `true` in the
|
|
9
|
-
* migration config.
|
|
10
|
-
*/
|
|
11
|
-
import { execFile } from 'node:child_process';
|
|
12
|
-
import { promisify } from 'node:util';
|
|
13
|
-
const execFileAsync = promisify(execFile);
|
|
14
|
-
/** Python packages required by the sentence-transformers embedding provider. */
|
|
15
|
-
const REQUIRED_PACKAGES = ['sentence_transformers', 'torch'];
|
|
16
|
-
/**
|
|
17
|
-
* Checks whether each of `REQUIRED_PACKAGES` is importable by the given Python
|
|
18
|
-
* binary. For any missing package, runs `pip install sentence-transformers`
|
|
19
|
-
* (which pulls in `torch` as a transitive dependency).
|
|
20
|
-
*
|
|
21
|
-
* Throws if the Python binary is not found or if `pip install` fails.
|
|
22
|
-
*
|
|
23
|
-
* @param pythonBin Path to the Python 3 interpreter (default: `'python3'`).
|
|
24
|
-
* @param timeout Maximum milliseconds to wait for the install (default: 5 min).
|
|
25
|
-
*/
|
|
26
|
-
export async function ensurePythonDeps(pythonBin = 'python3', timeout = 5 * 60_000) {
|
|
27
|
-
// Probe which packages are already available.
|
|
28
|
-
const missing = [];
|
|
29
|
-
for (const pkg of REQUIRED_PACKAGES) {
|
|
30
|
-
try {
|
|
31
|
-
await execFileAsync(pythonBin, ['-c', `import ${pkg}`], { timeout: 10_000 });
|
|
32
|
-
}
|
|
33
|
-
catch {
|
|
34
|
-
missing.push(pkg);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
if (missing.length === 0)
|
|
38
|
-
return;
|
|
39
|
-
// `sentence-transformers` transitively installs torch, so a single pip
|
|
40
|
-
// install call is sufficient.
|
|
41
|
-
await execFileAsync(pythonBin, ['-m', 'pip', 'install', '--quiet', 'sentence-transformers'], { timeout });
|
|
42
|
-
// Verify the install succeeded.
|
|
43
|
-
for (const pkg of REQUIRED_PACKAGES) {
|
|
44
|
-
await execFileAsync(pythonBin, ['-c', `import ${pkg}`], { timeout: 10_000 });
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
//# sourceMappingURL=ensure-python-deps.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ensure-python-deps.js","sourceRoot":"","sources":["../../src/indexer/ensure-python-deps.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE1C,gFAAgF;AAChF,MAAM,iBAAiB,GAAG,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;AAE7D;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAS,GAAG,SAAS,EACrB,OAAO,GAAG,CAAC,GAAG,MAAM;IAEpB,8CAA8C;IAC9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/E,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEjC,uEAAuE;IACvE,8BAA8B;IAC9B,MAAM,aAAa,CACjB,SAAS,EACT,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,uBAAuB,CAAC,EAC5D,EAAE,OAAO,EAAE,CACZ,CAAC;IAEF,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACpC,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC"}
|