@getplumb/core 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedder.d.ts +4 -2
- package/dist/embedder.d.ts.map +1 -1
- package/dist/embedder.js +28 -9
- package/dist/embedder.js.map +1 -1
- package/package.json +3 -3
package/dist/embedder.d.ts
CHANGED
|
@@ -16,12 +16,14 @@
|
|
|
16
16
|
export declare const EMBED_DIM = 384;
|
|
17
17
|
/**
|
|
18
18
|
* Embed a passage for indexing (no query prefix).
|
|
19
|
-
* Returns a normalized Float32Array of length EMBED_DIM
|
|
19
|
+
* Returns a normalized Float32Array of length EMBED_DIM, or a zero vector
|
|
20
|
+
* if @xenova/transformers is not available (graceful degradation).
|
|
20
21
|
*/
|
|
21
22
|
export declare function embed(text: string): Promise<Float32Array>;
|
|
22
23
|
/**
|
|
23
24
|
* Embed a search query with BGE "query: " prefix.
|
|
24
|
-
* Returns a normalized Float32Array of length EMBED_DIM
|
|
25
|
+
* Returns a normalized Float32Array of length EMBED_DIM, or a zero vector
|
|
26
|
+
* if @xenova/transformers is not available (graceful degradation).
|
|
25
27
|
*/
|
|
26
28
|
export declare function embedQuery(query: string): Promise<Float32Array>;
|
|
27
29
|
/**
|
package/dist/embedder.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,sDAAsD;AACtD,eAAO,MAAM,SAAS,MAAM,CAAC;AA2B7B;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAK/D;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAKrE;AA+BD;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBvF"}
|
package/dist/embedder.js
CHANGED
|
@@ -12,36 +12,52 @@
|
|
|
12
12
|
* First call downloads the model (~100 MB for bge-small). Subsequent calls
|
|
13
13
|
* use the local cache at ~/.cache/huggingface/hub/.
|
|
14
14
|
*/
|
|
15
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
16
|
-
// @ts-ignore — @xenova/transformers has incomplete typings
|
|
17
|
-
import { pipeline, env } from '@xenova/transformers';
|
|
18
15
|
/** Embedding dimension for BAAI/bge-small-en-v1.5. */
|
|
19
16
|
export const EMBED_DIM = 384;
|
|
20
|
-
// Disable the remote model check in test/offline environments to use cache.
|
|
21
|
-
// env.allowRemoteModels is already true by default; this line is a no-op but documents intent.
|
|
22
|
-
env.allowLocalModels = true;
|
|
23
17
|
let _embedPipeline = null;
|
|
18
|
+
let _embedLoadFailed = false;
|
|
24
19
|
async function getEmbedPipeline() {
|
|
20
|
+
if (_embedLoadFailed)
|
|
21
|
+
return null;
|
|
25
22
|
if (_embedPipeline === null) {
|
|
26
|
-
|
|
23
|
+
try {
|
|
24
|
+
// Dynamic import so the module is optional — if @xenova/transformers is not
|
|
25
|
+
// installed (e.g. on Windows via openclaw plugins install --ignore-scripts),
|
|
26
|
+
// we fall back to zero-vectors and skip embedding entirely.
|
|
27
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
28
|
+
// @ts-ignore — @xenova/transformers has incomplete typings
|
|
29
|
+
const { pipeline, env } = await import('@xenova/transformers');
|
|
30
|
+
env.allowLocalModels = true;
|
|
31
|
+
_embedPipeline = (await pipeline('feature-extraction', 'Xenova/bge-small-en-v1.5'));
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
_embedLoadFailed = true;
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
27
37
|
}
|
|
28
38
|
return _embedPipeline;
|
|
29
39
|
}
|
|
30
40
|
/**
|
|
31
41
|
* Embed a passage for indexing (no query prefix).
|
|
32
|
-
* Returns a normalized Float32Array of length EMBED_DIM
|
|
42
|
+
* Returns a normalized Float32Array of length EMBED_DIM, or a zero vector
|
|
43
|
+
* if @xenova/transformers is not available (graceful degradation).
|
|
33
44
|
*/
|
|
34
45
|
export async function embed(text) {
|
|
35
46
|
const pipe = await getEmbedPipeline();
|
|
47
|
+
if (pipe === null)
|
|
48
|
+
return new Float32Array(EMBED_DIM);
|
|
36
49
|
const output = await pipe(text, { pooling: 'mean', normalize: true });
|
|
37
50
|
return new Float32Array(output.data);
|
|
38
51
|
}
|
|
39
52
|
/**
|
|
40
53
|
* Embed a search query with BGE "query: " prefix.
|
|
41
|
-
* Returns a normalized Float32Array of length EMBED_DIM
|
|
54
|
+
* Returns a normalized Float32Array of length EMBED_DIM, or a zero vector
|
|
55
|
+
* if @xenova/transformers is not available (graceful degradation).
|
|
42
56
|
*/
|
|
43
57
|
export async function embedQuery(query) {
|
|
44
58
|
const pipe = await getEmbedPipeline();
|
|
59
|
+
if (pipe === null)
|
|
60
|
+
return new Float32Array(EMBED_DIM);
|
|
45
61
|
const output = await pipe(`query: ${query}`, { pooling: 'mean', normalize: true });
|
|
46
62
|
return new Float32Array(output.data);
|
|
47
63
|
}
|
|
@@ -52,6 +68,9 @@ async function getRerankPipeline() {
|
|
|
52
68
|
return null;
|
|
53
69
|
if (_rerankPipeline === null) {
|
|
54
70
|
try {
|
|
71
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
72
|
+
// @ts-ignore — @xenova/transformers has incomplete typings
|
|
73
|
+
const { pipeline } = await import('@xenova/transformers');
|
|
55
74
|
_rerankPipeline = (await pipeline('text-classification', 'Xenova/ms-marco-MiniLM-L-6-v2'));
|
|
56
75
|
}
|
|
57
76
|
catch {
|
package/dist/embedder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,sDAAsD;AACtD,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAI7B,IAAI,cAAc,GAAoB,IAAI,CAAC;AAC3C,IAAI,gBAAgB,GAAG,KAAK,CAAC;AAE7B,KAAK,UAAU,gBAAgB;IAC7B,IAAI,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,4EAA4E;YAC5E,6EAA6E;YAC7E,4DAA4D;YAC5D,6DAA6D;YAC7D,2DAA2D;YAC3D,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC9D,GAAqC,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/D,cAAc,GAAG,CAAC,MAAM,QAAQ,CAAC,oBAAoB,EAAE,0BAA0B,CAAC,CAAa,CAAC;QAClG,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB,GAAG,IAAI,CAAC;YACxB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,MAAM,gBAAgB,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtE,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa;IAC5C,MAAM,IAAI,GAAG,MAAM,gBAAgB,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AASD,IAAI,eAAe,GAA0B,IAAI,CAAC;AAClD,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAE9B,KAAK,UAAU,iBAAiB;IAC9B,IAAI,iBAAiB;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,6DAA6D;YAC7D,2DAA2D;YAC3D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC1D,eAAe,GAAG,CAAC,MAAM,QAAQ,CAC/B,qBAAqB,EACrB,+BAA+B,CAChC,CAAmB,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,GAAG,IAAI,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa,EAAE,QAAkB;IAClE,MAAM,IAAI,GAAG,MAAM,iBAAiB,EAAE,CAAC;IACvC,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3E,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAkC,CAAC;YAC1F,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@getplumb/core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Plumb memory engine — storage abstraction, types, and local SQLite driver",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -33,12 +33,12 @@
|
|
|
33
33
|
"typescript": "^5.4.0"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@xenova/transformers": "^2.17.2",
|
|
37
36
|
"better-sqlite3": "^12.6.2",
|
|
38
37
|
"openai": "^4.80.0",
|
|
39
38
|
"sqlite-vec": "0.1.7-alpha.2"
|
|
40
39
|
},
|
|
41
40
|
"optionalDependencies": {
|
|
42
|
-
"@anthropic-ai/sdk": "^0.78.0"
|
|
41
|
+
"@anthropic-ai/sdk": "^0.78.0",
|
|
42
|
+
"@xenova/transformers": "^2.17.2"
|
|
43
43
|
}
|
|
44
44
|
}
|