@mneme-ai/embeddings 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundled.d.ts +40 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +104 -0
- package/dist/bundled.js.map +1 -0
- package/dist/enrich.d.ts.map +1 -1
- package/dist/enrich.js +6 -2
- package/dist/enrich.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/ollama.d.ts +11 -0
- package/dist/ollama.d.ts.map +1 -1
- package/dist/ollama.js +90 -6
- package/dist/ollama.js.map +1 -1
- package/dist/resolve.d.ts +18 -7
- package/dist/resolve.d.ts.map +1 -1
- package/dist/resolve.js +48 -13
- package/dist/resolve.js.map +1 -1
- package/package.json +4 -3
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from "@mneme-ai/core";
|
|
2
|
+
export interface BundledOptions {
|
|
3
|
+
/** HuggingFace repo id. all-MiniLM-L6-v2 = 25MB / 384-dim — best size/quality. */
|
|
4
|
+
model?: string;
|
|
5
|
+
/** Override cache dir. Default: ~/.cache/mneme/models. */
|
|
6
|
+
cacheDir?: string;
|
|
7
|
+
/** Hook for download/load progress (lazy load can take ~5–60s on cold start). */
|
|
8
|
+
onProgress?: (info: {
|
|
9
|
+
status: string;
|
|
10
|
+
loaded?: number;
|
|
11
|
+
total?: number;
|
|
12
|
+
file?: string;
|
|
13
|
+
}) => void;
|
|
14
|
+
}
|
|
15
|
+
export declare class BundledEmbedder implements EmbeddingProvider {
|
|
16
|
+
readonly name: string;
|
|
17
|
+
readonly dimensions: number;
|
|
18
|
+
private readonly model;
|
|
19
|
+
private readonly cacheDir;
|
|
20
|
+
private readonly onProgress?;
|
|
21
|
+
private extractor;
|
|
22
|
+
private loadPromise;
|
|
23
|
+
constructor(opts?: BundledOptions);
|
|
24
|
+
/** Cheap pre-flight — instantiates the pipeline (downloads if needed) and
|
|
25
|
+
* runs a 1-token sanity embed. Use BEFORE the long indexer loop. */
|
|
26
|
+
verify(): Promise<{
|
|
27
|
+
ok: true;
|
|
28
|
+
} | {
|
|
29
|
+
ok: false;
|
|
30
|
+
reason: string;
|
|
31
|
+
remedy: string;
|
|
32
|
+
}>;
|
|
33
|
+
embed(texts: string[]): Promise<Float32Array[]>;
|
|
34
|
+
/** Lazy load — done at most once, even under concurrent embed() calls. */
|
|
35
|
+
private load;
|
|
36
|
+
private bootPipeline;
|
|
37
|
+
}
|
|
38
|
+
/** Default cache dir: ~/.cache/mneme/models. User can `rm -rf` to reset. */
|
|
39
|
+
export declare function defaultCacheDir(): string;
|
|
40
|
+
//# sourceMappingURL=bundled.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bundled.d.ts","sourceRoot":"","sources":["../src/bundled.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,WAAW,cAAc;IAC7B,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACjG;AAeD,qBAAa,eAAgB,YAAW,iBAAiB;IACvD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA+B;IAC3D,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,WAAW,CAA0C;gBAEjD,IAAI,GAAE,cAAmB;IAQrC;yEACqE;IAC/D,MAAM,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAiB/E,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAcrD,0EAA0E;YAC5D,IAAI;YAQJ,YAAY;CAmC3B;AAED,4EAA4E;AAC5E,wBAAgB,eAAe,IAAI,MAAM,CAExC"}
|
package/dist/bundled.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BundledEmbedder — zero-install WASM embeddings.
|
|
3
|
+
*
|
|
4
|
+
* Uses @xenova/transformers (pure JS + ONNX-WASM, no native deps) so the
|
|
5
|
+
* tool runs on Windows / Mac / Linux out of the box. The model file is
|
|
6
|
+
* lazy-downloaded on first use (~25MB to a local cache), giving users a
|
|
7
|
+
* "★★★ semantic quality" path without installing Ollama.
|
|
8
|
+
*
|
|
9
|
+
* Fallback order in resolveEmbedder():
|
|
10
|
+
* OpenAI (★★★★★ paid) → Ollama (★★★★ free local)
|
|
11
|
+
* → Bundled WASM (★★★ free, no install) → Hash (★★ deterministic)
|
|
12
|
+
*/
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
import { mkdirSync } from "node:fs";
|
|
16
|
+
const DEFAULT_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
17
|
+
const DEFAULT_DIMS = 384;
|
|
18
|
+
export class BundledEmbedder {
|
|
19
|
+
name;
|
|
20
|
+
dimensions;
|
|
21
|
+
model;
|
|
22
|
+
cacheDir;
|
|
23
|
+
onProgress;
|
|
24
|
+
extractor = null;
|
|
25
|
+
loadPromise = null;
|
|
26
|
+
constructor(opts = {}) {
|
|
27
|
+
this.model = opts.model ?? DEFAULT_MODEL;
|
|
28
|
+
this.dimensions = DEFAULT_DIMS;
|
|
29
|
+
this.cacheDir = opts.cacheDir ?? defaultCacheDir();
|
|
30
|
+
this.onProgress = opts.onProgress;
|
|
31
|
+
this.name = `bundled:${this.model}`;
|
|
32
|
+
}
|
|
33
|
+
/** Cheap pre-flight — instantiates the pipeline (downloads if needed) and
|
|
34
|
+
* runs a 1-token sanity embed. Use BEFORE the long indexer loop. */
|
|
35
|
+
async verify() {
|
|
36
|
+
try {
|
|
37
|
+
await this.load();
|
|
38
|
+
await this.embed(["ok"]);
|
|
39
|
+
return { ok: true };
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
const msg = err.message ?? String(err);
|
|
43
|
+
return {
|
|
44
|
+
ok: false,
|
|
45
|
+
reason: `Bundled WASM model failed: ${msg}`,
|
|
46
|
+
remedy: "First run requires internet to download ~25MB. " +
|
|
47
|
+
"If you're offline, set --embedder hash to use the deterministic fallback.",
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async embed(texts) {
|
|
52
|
+
const ext = await this.load();
|
|
53
|
+
const out = new Array(texts.length);
|
|
54
|
+
// Sequential: the WASM pipeline isn't designed for concurrency; one at a
|
|
55
|
+
// time is the predictable, memory-safe path. For 1000 chunks this takes
|
|
56
|
+
// ~10–20s on consumer laptops — acceptable for a one-time index.
|
|
57
|
+
for (let i = 0; i < texts.length; i++) {
|
|
58
|
+
const result = await ext(texts[i], { pooling: "mean", normalize: true });
|
|
59
|
+
// result.data is Float32Array-like — copy into a fresh typed array.
|
|
60
|
+
out[i] = Float32Array.from(result.data);
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
/** Lazy load — done at most once, even under concurrent embed() calls. */
|
|
65
|
+
async load() {
|
|
66
|
+
if (this.extractor)
|
|
67
|
+
return this.extractor;
|
|
68
|
+
if (this.loadPromise)
|
|
69
|
+
return this.loadPromise;
|
|
70
|
+
this.loadPromise = this.bootPipeline();
|
|
71
|
+
this.extractor = await this.loadPromise;
|
|
72
|
+
return this.extractor;
|
|
73
|
+
}
|
|
74
|
+
async bootPipeline() {
|
|
75
|
+
// Ensure cache dir exists so transformers.js can write the ONNX model.
|
|
76
|
+
try {
|
|
77
|
+
mkdirSync(this.cacheDir, { recursive: true });
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
/* permission errors surface later via the pipeline */
|
|
81
|
+
}
|
|
82
|
+
// Dynamic import — keeps the heavy WASM init out of cold-start path
|
|
83
|
+
// for users who never touch this provider.
|
|
84
|
+
const transformers = (await import("@xenova/transformers"));
|
|
85
|
+
transformers.env.cacheDir = this.cacheDir;
|
|
86
|
+
transformers.env.allowRemoteModels = true;
|
|
87
|
+
const onProgress = this.onProgress;
|
|
88
|
+
return await transformers.pipeline("feature-extraction", this.model, {
|
|
89
|
+
progress_callback: onProgress
|
|
90
|
+
? (info) => onProgress({
|
|
91
|
+
status: String(info["status"] ?? ""),
|
|
92
|
+
loaded: typeof info["loaded"] === "number" ? info["loaded"] : undefined,
|
|
93
|
+
total: typeof info["total"] === "number" ? info["total"] : undefined,
|
|
94
|
+
file: typeof info["file"] === "string" ? info["file"] : undefined,
|
|
95
|
+
})
|
|
96
|
+
: undefined,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** Default cache dir: ~/.cache/mneme/models. User can `rm -rf` to reset. */
|
|
101
|
+
export function defaultCacheDir() {
|
|
102
|
+
return join(homedir(), ".cache", "mneme", "models");
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=bundled.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bundled.js","sourceRoot":"","sources":["../src/bundled.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAYpC,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAChD,MAAM,YAAY,GAAG,GAAG,CAAC;AAYzB,MAAM,OAAO,eAAe;IACjB,IAAI,CAAS;IACb,UAAU,CAAS;IACX,KAAK,CAAS;IACd,QAAQ,CAAS;IACjB,UAAU,CAAgC;IACnD,SAAS,GAA4B,IAAI,CAAC;IAC1C,WAAW,GAAqC,IAAI,CAAC;IAE7D,YAAY,OAAuB,EAAE;QACnC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAC;QACnD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,WAAW,IAAI,CAAC,KAAK,EAAE,CAAC;IACtC,CAAC;IAED;yEACqE;IACrE,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACzB,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAI,GAAa,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;YAClD,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,8BAA8B,GAAG,EAAE;gBAC3C,MAAM,EACJ,iDAAiD;oBACjD,2EAA2E;aAC9E,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,yEAAyE;QACzE,wEAAwE;QACxE,iEAAiE;QACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,oEAAoE;YACpE,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,0EAA0E;IAClE,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QAC1C,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;QACxC,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,uEAAuE;QACvE,IAAI,CAAC;YACH,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;QAED,oEAAoE;QACpE,2CAA2C;QAC3C,MAAM,YAAY,GAAG,CAAC,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAOzD,CAAC;QAEF,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1C,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAE1C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,OAAO,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,EAAE;YACnE,iBAAiB,EAAE,UAAU;gBAC3B,CAAC,CAAC,CAAC,IAA6B,EAAE,EAAE,CAChC,UAAU,CAAC;oBACT,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACpC,MAAM,EAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAI,CAAC,QAAQ,CAAY,CAAC,CAAC,CAAC,SAAS;oBACnF,KAAK,EAAE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAI,CAAC,OAAO,CAAY,CAAC,CAAC,CAAC,SAAS;oBAChF,IAAI,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAI,CAAC,MAAM,CAAY,CAAC,CAAC,CAAC,SAAS;iBAC9E,CAAC;gBACN,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;IACL,CAAC;CACF;AAED,4EAA4E;AAC5E,MAAM,UAAU,eAAe;IAC7B,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AACtD,CAAC"}
|
package/dist/enrich.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enrich.d.ts","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,WAAW;IAC1B,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAID,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;
|
|
1
|
+
{"version":3,"file":"enrich.d.ts","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,WAAW;IAC1B,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAID,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAOD,qBAAa,cAAe,YAAW,gBAAgB;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,IAAI,GAAE,qBAA0B;IAStC,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IA0CjD,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;CAQ/B;AAID,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAKD,qBAAa,cAAe,YAAW,gBAAgB;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,IAAI,EAAE,qBAAqB;IAQjC,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;CAwCxD;AAID,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,eAAe,CACnC,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAgB3B"}
|
package/dist/enrich.js
CHANGED
|
@@ -10,7 +10,9 @@
|
|
|
10
10
|
* - OpenAI needs OPENAI_API_KEY. ~$0.10 to enrich a 1k-bad-commit repo.
|
|
11
11
|
*/
|
|
12
12
|
const OLLAMA_DEFAULT_MODEL = "llama3.2:1b";
|
|
13
|
-
|
|
13
|
+
// 127.0.0.1 (not localhost) — Node 18+/undici prefers IPv6 (::1) but Ollama
|
|
14
|
+
// only listens on IPv4 by default. localhost causes silent fetch failures on Windows.
|
|
15
|
+
const OLLAMA_DEFAULT_URL = "http://127.0.0.1:11434";
|
|
14
16
|
export class OllamaEnricher {
|
|
15
17
|
name;
|
|
16
18
|
baseUrl;
|
|
@@ -18,7 +20,9 @@ export class OllamaEnricher {
|
|
|
18
20
|
timeoutMs;
|
|
19
21
|
constructor(opts = {}) {
|
|
20
22
|
this.model = opts.model ?? OLLAMA_DEFAULT_MODEL;
|
|
21
|
-
|
|
23
|
+
const raw = (opts.baseUrl ?? OLLAMA_DEFAULT_URL).replace(/\/$/, "");
|
|
24
|
+
// Auto-rewrite localhost → 127.0.0.1 (see comment on OLLAMA_DEFAULT_URL)
|
|
25
|
+
this.baseUrl = raw.replace(/^http:\/\/localhost(:|$|\/)/i, "http://127.0.0.1$1");
|
|
22
26
|
this.timeoutMs = opts.timeoutMs ?? 60_000;
|
|
23
27
|
this.name = `ollama:${this.model}`;
|
|
24
28
|
}
|
package/dist/enrich.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enrich.js","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAkCH,MAAM,oBAAoB,GAAG,aAAa,CAAC;AAC3C,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAEpD,MAAM,OAAO,cAAc;IAChB,IAAI,CAAS;IACL,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,SAAS,CAAS;IAEnC,YAAY,OAA8B,EAAE;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,oBAAoB,CAAC;QAChD,
|
|
1
|
+
{"version":3,"file":"enrich.js","sourceRoot":"","sources":["../src/enrich.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAkCH,MAAM,oBAAoB,GAAG,aAAa,CAAC;AAC3C,4EAA4E;AAC5E,sFAAsF;AACtF,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAEpD,MAAM,OAAO,cAAc;IAChB,IAAI,CAAS;IACL,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,SAAS,CAAS;IAEnC,YAAY,OAA8B,EAAE;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,oBAAoB,CAAC;QAChD,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACpE,yEAAyE;QACzE,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,8BAA8B,EAAE,oBAAoB,CAAC,CAAC;QACjF,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;QAC1C,IAAI,CAAC,IAAI,GAAG,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAkB;QAC7B,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,EAAE;gBAClD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE;wBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;qBACtC;oBACD,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,GAAG;wBACrC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,GAAG;qBACpC;iBACF,CAAC;gBACF,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAI7B,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;gBACzC,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,KAAK,EAAE;oBACL,YAAY,EAAE,IAAI,CAAC,iBAAiB;oBACpC,gBAAgB,EAAE,IAAI,CAAC,UAAU;iBAClC;aACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,CAAC,CAAC;YACpD,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF;AAWD,MAAM,oBAAoB,GAAG,aAAa,CAAC;AAC3C,MAAM,kBAAkB,GAAG,2BAA2B,CAAC;AAEvD,MAAM,OAAO,cAAc;IAChB,IAAI,CAAS;IACL,MAAM,CAAS;IACf,KAAK,CAAS;IACd,OAAO,CAAS;IAChB,SAAS,CAAS;IAEnC,YAAY,IAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,oBAAoB,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;QAC1C,IAAI,CAAC,IAAI,GAAG,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAkB;QAC7B,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,mBAAmB,EAAE;gBAC1D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE;wBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;qBACtC;oBACD,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,GAAG;oBACrC,UAAU,EAAE,KAAK,CAAC,SAAS,IAAI,GAAG;iBACnC,CAAC;gBACF,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAG7B,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;gBACvD,MAAM,EAAE,IAAI,CAAC,IAAI;gBACjB,KAAK,EAAE;oBACL,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa;oBACvC,gBAAgB,EAAE,IAAI,CAAC,KAAK,EAAE,iBAAiB;iBAChD;aACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;CACF;AAWD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAA+B,EAAE;IAEjC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC;IACzC,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,IAAI,MAAM,MAAM,CAAC,IAAI,EAAE;YAAE,OAAO,MAAM,CAAC;QACvC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC5D,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QAC7D,OAAO,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,IAAI,KAAK,CACb,4EAA4E,CAC7E,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
package/dist/ollama.d.ts
CHANGED
|
@@ -18,6 +18,17 @@ export declare class OllamaEmbedder implements EmbeddingProvider {
|
|
|
18
18
|
/** Once we learn the server lacks /api/embed (404), skip it on every later batch. */
|
|
19
19
|
private batchEndpointBroken;
|
|
20
20
|
constructor(opts?: OllamaOptions);
|
|
21
|
+
/** Cheap pre-flight: verify Ollama is reachable AND the model is available
|
|
22
|
+
* AND a 1-token embed actually returns a vector. Run this BEFORE the
|
|
23
|
+
* long-running indexer loop so failures surface in seconds, not after
|
|
24
|
+
* minutes of redaction work. */
|
|
25
|
+
verify(): Promise<{
|
|
26
|
+
ok: true;
|
|
27
|
+
} | {
|
|
28
|
+
ok: false;
|
|
29
|
+
reason: string;
|
|
30
|
+
remedy: string;
|
|
31
|
+
}>;
|
|
21
32
|
embed(texts: string[]): Promise<Float32Array[]>;
|
|
22
33
|
private embedBatch;
|
|
23
34
|
private embedOne;
|
package/dist/ollama.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../src/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iGAAiG;IACjG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yGAAyG;IACzG,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;
|
|
1
|
+
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../src/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iGAAiG;IACjG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yGAAyG;IACzG,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AAkBD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAwC;IACpE,qFAAqF;IACrF,OAAO,CAAC,mBAAmB,CAAS;gBAExB,IAAI,GAAE,aAAkB;IAYpC;;;qCAGiC;IAC3B,MAAM,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA6B/E,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YA4BvC,UAAU;YAwBV,QAAQ;YAcR,gBAAgB;IAqCxB,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;CAQ/B"}
|
package/dist/ollama.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
const DEFAULT_MODEL = "nomic-embed-text";
|
|
2
2
|
const DEFAULT_DIMS = 768;
|
|
3
|
-
|
|
3
|
+
// Use 127.0.0.1 (NOT localhost) — Node 18+/undici prefers IPv6 (::1) which
|
|
4
|
+
// Ollama doesn't listen on by default. Causes silent "fetch failed" on Windows.
|
|
5
|
+
const DEFAULT_URL = "http://127.0.0.1:11434";
|
|
4
6
|
const DEFAULT_TIMEOUT_MS = 180_000;
|
|
7
|
+
const RETRYABLE_FETCH = ["fetch failed", "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EAI_AGAIN", "socket hang up"];
|
|
5
8
|
export class OllamaEmbedder {
|
|
6
9
|
name;
|
|
7
10
|
dimensions;
|
|
@@ -13,12 +16,48 @@ export class OllamaEmbedder {
|
|
|
13
16
|
batchEndpointBroken = false;
|
|
14
17
|
constructor(opts = {}) {
|
|
15
18
|
this.model = opts.model ?? DEFAULT_MODEL;
|
|
16
|
-
|
|
19
|
+
// Auto-rewrite localhost → 127.0.0.1 so users who pass their own URL
|
|
20
|
+
// don't trip the same IPv6 trap that bit us with the default.
|
|
21
|
+
const raw = (opts.baseUrl ?? DEFAULT_URL).replace(/\/$/, "");
|
|
22
|
+
this.baseUrl = raw.replace(/^http:\/\/localhost(:|$|\/)/i, "http://127.0.0.1$1");
|
|
17
23
|
this.dimensions = opts.dimensions ?? DEFAULT_DIMS;
|
|
18
24
|
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
19
25
|
this.onItemDone = opts.onItemDone;
|
|
20
26
|
this.name = `ollama:${this.model}`;
|
|
21
27
|
}
|
|
28
|
+
/** Cheap pre-flight: verify Ollama is reachable AND the model is available
|
|
29
|
+
* AND a 1-token embed actually returns a vector. Run this BEFORE the
|
|
30
|
+
* long-running indexer loop so failures surface in seconds, not after
|
|
31
|
+
* minutes of redaction work. */
|
|
32
|
+
async verify() {
|
|
33
|
+
try {
|
|
34
|
+
const tags = await this.fetchWithTimeout(`${this.baseUrl}/api/tags`, { method: "GET" });
|
|
35
|
+
if (!tags.ok) {
|
|
36
|
+
return {
|
|
37
|
+
ok: false,
|
|
38
|
+
reason: `Ollama responded with HTTP ${tags.status} on /api/tags.`,
|
|
39
|
+
remedy: "Restart Ollama: `ollama serve` in a new terminal.",
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const list = (await tags.json());
|
|
43
|
+
const have = (list.models ?? []).map((m) => m.name);
|
|
44
|
+
const matches = have.some((n) => n === this.model || n.startsWith(this.model + ":"));
|
|
45
|
+
if (!matches) {
|
|
46
|
+
return {
|
|
47
|
+
ok: false,
|
|
48
|
+
reason: `Model '${this.model}' is not pulled. Available: ${have.join(", ") || "(none)"}.`,
|
|
49
|
+
remedy: `Run: ollama pull ${this.model}`,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
// One-token sanity embed — proves the model is loaded + responds.
|
|
53
|
+
await this.embedOne("ok");
|
|
54
|
+
return { ok: true };
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
const msg = friendlyError(err, this.baseUrl, this.model);
|
|
58
|
+
return { ok: false, reason: msg.reason, remedy: msg.remedy };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
22
61
|
async embed(texts) {
|
|
23
62
|
if (texts.length === 0)
|
|
24
63
|
return [];
|
|
@@ -84,18 +123,35 @@ export class OllamaEmbedder {
|
|
|
84
123
|
throw new Error("Ollama returned no embedding");
|
|
85
124
|
return Float32Array.from(json.embedding);
|
|
86
125
|
}
|
|
87
|
-
async fetchWithTimeout(url, init) {
|
|
126
|
+
async fetchWithTimeout(url, init, attempt = 0) {
|
|
88
127
|
const ctrl = new AbortController();
|
|
89
128
|
const timer = setTimeout(() => ctrl.abort(), this.timeoutMs);
|
|
90
129
|
try {
|
|
91
130
|
return await fetch(url, { ...init, signal: ctrl.signal });
|
|
92
131
|
}
|
|
93
132
|
catch (err) {
|
|
133
|
+
const e = err;
|
|
94
134
|
// Surface a clearer message than "AbortError: This operation was aborted"
|
|
95
|
-
if (
|
|
96
|
-
throw new Error(`Ollama did not respond within ${this.timeoutMs}ms
|
|
135
|
+
if (e?.name === "AbortError") {
|
|
136
|
+
throw new Error(`Ollama did not respond within ${this.timeoutMs}ms at ${this.baseUrl}.\n` +
|
|
137
|
+
` Check that 'ollama serve' is running and the model is loaded ('ollama list').`);
|
|
138
|
+
}
|
|
139
|
+
// Retry once on transient network errors — covers cold-start socket churn.
|
|
140
|
+
const code = e?.cause?.code ?? "";
|
|
141
|
+
const msg = e?.message ?? "";
|
|
142
|
+
const transient = RETRYABLE_FETCH.some((s) => code.includes(s) || msg.includes(s));
|
|
143
|
+
if (transient && attempt === 0) {
|
|
144
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
145
|
+
return this.fetchWithTimeout(url, init, 1);
|
|
97
146
|
}
|
|
98
|
-
throw
|
|
147
|
+
// Re-throw with the underlying cause stitched in so users see WHY it failed.
|
|
148
|
+
const why = e?.cause?.code ?? e?.cause?.message ?? "";
|
|
149
|
+
const detail = why ? `${msg} (${why})` : msg || String(err);
|
|
150
|
+
throw new Error(`Cannot reach Ollama at ${this.baseUrl}: ${detail}.\n` +
|
|
151
|
+
` Fixes:\n` +
|
|
152
|
+
` 1. Start Ollama: ollama serve\n` +
|
|
153
|
+
` 2. Verify model: ollama list (must show ${this.model})\n` +
|
|
154
|
+
` 3. If 'localhost' was used, switch to 127.0.0.1 (Node prefers IPv6 on Windows).`);
|
|
99
155
|
}
|
|
100
156
|
finally {
|
|
101
157
|
clearTimeout(timer);
|
|
@@ -114,4 +170,32 @@ export class OllamaEmbedder {
|
|
|
114
170
|
/** Internal sentinel for "server doesn't support /api/embed yet". */
|
|
115
171
|
class OllamaNotFoundError extends Error {
|
|
116
172
|
}
|
|
173
|
+
/** Translate any fetch/server error into a (reason, remedy) pair the CLI can show.
|
|
174
|
+
* Decoupled from CLI rendering so we can also use it in MCP / programmatic callers. */
|
|
175
|
+
function friendlyError(err, baseUrl, model) {
|
|
176
|
+
const e = err;
|
|
177
|
+
const code = e?.cause?.code ?? "";
|
|
178
|
+
if (code === "ECONNREFUSED") {
|
|
179
|
+
return {
|
|
180
|
+
reason: `Ollama isn't running at ${baseUrl}.`,
|
|
181
|
+
remedy: "Open a new terminal and run: ollama serve",
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
if (code === "ENOTFOUND" || code === "EAI_AGAIN") {
|
|
185
|
+
return {
|
|
186
|
+
reason: `Cannot resolve host in ${baseUrl}.`,
|
|
187
|
+
remedy: "Check the URL — for local Ollama use http://127.0.0.1:11434.",
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
if (e?.message?.includes("did not respond within")) {
|
|
191
|
+
return {
|
|
192
|
+
reason: e.message,
|
|
193
|
+
remedy: `First call may load the model into memory. Try: ollama run ${model} 'hi' to warm it up, then re-run.`,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
reason: e?.message ?? String(err),
|
|
198
|
+
remedy: `Verify Ollama is up: ollama list · if ${model} is missing: ollama pull ${model}`,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
117
201
|
//# sourceMappingURL=ollama.js.map
|
package/dist/ollama.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../src/ollama.ts"],"names":[],"mappings":"AAYA,MAAM,aAAa,GAAG,kBAAkB,CAAC;AACzC,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,MAAM,WAAW,GAAG,wBAAwB,CAAC;AAC7C,MAAM,kBAAkB,GAAG,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../src/ollama.ts"],"names":[],"mappings":"AAYA,MAAM,aAAa,GAAG,kBAAkB,CAAC;AACzC,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,WAAW,GAAG,wBAAwB,CAAC;AAC7C,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,eAAe,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;AAUnH,MAAM,OAAO,cAAc;IAChB,IAAI,CAAS;IACb,UAAU,CAAS;IACX,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,SAAS,CAAS;IAClB,UAAU,CAAyC;IACpE,qFAAqF;IAC7E,mBAAmB,GAAG,KAAK,CAAC;IAEpC,YAAY,OAAsB,EAAE;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC;QACzC,qEAAqE;QACrE,8DAA8D;QAC9D,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,WAAW,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,8BAA8B,EAAE,oBAAoB,CAAC,CAAC;QACjF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,YAAY,CAAC;QAClD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC;IAED;;;qCAGiC;IACjC,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACxF,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACb,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,MAAM,EAAE,8BAA8B,IAAI,CAAC,MAAM,gBAAgB;oBACjE,MAAM,EAAE,mDAAmD;iBAC5D,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAyC,CAAC;YACzE,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC;YACrF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,MAAM,EAAE,UAAU,IAAI,CAAC,KAAK,+BAA+B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,GAAG;oBACzF,MAAM,EAAE,qBAAqB,IAAI,CAAC,KAAK,EAAE;iBAC1C,CAAC;YACJ,CAAC;YACD,kEAAkE;YAClE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,2EAA2E;QAC3E,qEAAqE;QACrE,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC1C,IAAI,IAAI,CAAC,UAAU;oBAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjE,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,uEAAuE;gBACvE,4DAA4D;gBAC5D,IAAI,CAAC,CAAC,GAAG,YAAY,mBAAmB,CAAC;oBAAE,MAAM,GAAG,CAAC;gBACrD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAClC,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,qDAAqD;QACrD,MAAM,GAAG,GAAmB,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,UAAU;gBAAE,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe;QACtC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,YAAY,EAAE;YACnE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;SAC1D,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,IAAI,mBAAmB,CAAC,iDAAiD,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAc,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACb,mBAAmB,IAAI,CAAC,UAAU,CAAC,MAAM,mBAAmB,KAAK,CAAC,MAAM,SAAS,CAClF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,IAAY;QACjC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,iBAAiB,EAAE;YACxE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SAC1D,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAiB,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACrE,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,IAAiB,EAAE,OAAO,GAAG,CAAC;QACxE,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,GAAuF,CAAC;YAClG,0EAA0E;YAC1E,IAAI,CAAC,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,iCAAiC,IAAI,CAAC,SAAS,SAAS,IAAI,CAAC,OAAO,KAAK;oBACvE,kFAAkF,CACrF,CAAC;YACJ,CAAC;YACD,2EAA2E;YAC3E,MAAM,IAAI,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnF,IAAI,SAAS,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC7C,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7C,CAAC;YACD,6EAA6E;YAC7E,MAAM,GAAG,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,CAAC;YACtD,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5D,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,CAAC,OAAO,KAAK,MAAM,KAAK;gBACpD,aAAa;gBACb,uCAAuC;gBACvC,kDAAkD,IAAI,CAAC,KAAK,KAAK;gBACjE,sFAAsF,CACzF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACvF,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF;AAED,qEAAqE;AACrE,MAAM,mBAAoB,SAAQ,KAAK;CAAG;AAE1C;wFACwF;AACxF,SAAS,aAAa,CACpB,GAAY,EACZ,OAAe,EACf,KAAa;IAEb,MAAM,CAAC,GAAG,GAAwE,CAAC;IACnF,MAAM,IAAI,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;IAClC,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,2BAA2B,OAAO,GAAG;YAC7C,MAAM,EAAE,4CAA4C;SACrD,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,0BAA0B,OAAO,GAAG;YAC5C,MAAM,EAAE,8DAA8D;SACvE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACnD,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,OAAO;YACjB,MAAM,EAAE,+DAA+D,KAAK,oCAAoC;SACjH,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM,EAAE,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;QACjC,MAAM,EAAE,8CAA8C,KAAK,6BAA6B,KAAK,EAAE;KAChG,CAAC;AACJ,CAAC"}
|
package/dist/resolve.d.ts
CHANGED
|
@@ -1,15 +1,26 @@
|
|
|
1
1
|
import type { EmbeddingProvider } from "@mneme-ai/core";
|
|
2
|
+
import { BundledEmbedder } from "./bundled.js";
|
|
2
3
|
export interface ResolveOptions {
|
|
3
|
-
|
|
4
|
+
/**
|
|
5
|
+
* `auto` (default) walks the fallback ladder so the user gets the highest
|
|
6
|
+
* quality embedder that's actually available, with hash always reachable
|
|
7
|
+
* as the last resort:
|
|
8
|
+
*
|
|
9
|
+
* 1. OpenAI (★★★★★ paid) — if OPENAI_API_KEY is set
|
|
10
|
+
* 2. Ollama (★★★★ free) — if /api/tags responds + model pulled
|
|
11
|
+
* 3. Bundled (★★★ free) — WASM model, ~25MB lazy download
|
|
12
|
+
* 4. Hash (★★) — deterministic, zero deps, always works
|
|
13
|
+
*
|
|
14
|
+
* Pass an explicit value to skip the ladder.
|
|
15
|
+
*/
|
|
16
|
+
provider?: "auto" | "ollama" | "openai" | "bundled" | "hash";
|
|
4
17
|
model?: string;
|
|
5
18
|
apiKey?: string;
|
|
6
19
|
baseUrl?: string;
|
|
20
|
+
/** Optional callback for bundled-model download progress. */
|
|
21
|
+
onBundledProgress?: NonNullable<ConstructorParameters<typeof BundledEmbedder>[0]>["onProgress"];
|
|
7
22
|
}
|
|
8
|
-
/**
|
|
9
|
-
* Auto-select an embedder.
|
|
10
|
-
*
|
|
11
|
-
* auto: prefer Ollama → OpenAI (if key) → hash fallback
|
|
12
|
-
* ollama / openai / hash: explicit
|
|
13
|
-
*/
|
|
14
23
|
export declare function resolveEmbedder(opts?: ResolveOptions): Promise<EmbeddingProvider>;
|
|
24
|
+
/** Helper for callers that explicitly want the deterministic offline path. */
|
|
25
|
+
export declare function hashEmbedder(): EmbeddingProvider;
|
|
15
26
|
//# sourceMappingURL=resolve.d.ts.map
|
package/dist/resolve.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve.d.ts","sourceRoot":"","sources":["../src/resolve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"resolve.d.ts","sourceRoot":"","sources":["../src/resolve.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGxD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,iBAAiB,CAAC,EAAE,WAAW,CAC7B,qBAAqB,CAAC,OAAO,eAAe,CAAC,CAAC,CAAC,CAAC,CACjD,CAAC,YAAY,CAAC,CAAC;CACjB;AAED,wBAAsB,eAAe,CAAC,IAAI,GAAE,cAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAgE3F;AAED,8EAA8E;AAC9E,wBAAgB,YAAY,IAAI,iBAAiB,CAEhD"}
|
package/dist/resolve.js
CHANGED
|
@@ -1,29 +1,64 @@
|
|
|
1
1
|
import { OllamaEmbedder } from "./ollama.js";
|
|
2
2
|
import { OpenAIEmbedder } from "./openai.js";
|
|
3
|
+
import { BundledEmbedder } from "./bundled.js";
|
|
3
4
|
import { HashEmbedder } from "./hash.js";
|
|
4
|
-
/**
|
|
5
|
-
* Auto-select an embedder.
|
|
6
|
-
*
|
|
7
|
-
* auto: prefer Ollama → OpenAI (if key) → hash fallback
|
|
8
|
-
* ollama / openai / hash: explicit
|
|
9
|
-
*/
|
|
10
5
|
export async function resolveEmbedder(opts = {}) {
|
|
11
6
|
const provider = opts.provider ?? "auto";
|
|
12
|
-
|
|
7
|
+
// ── Explicit picks ──────────────────────────────────────────────────
|
|
8
|
+
if (provider === "openai") {
|
|
9
|
+
const apiKey = opts.apiKey ?? process.env["OPENAI_API_KEY"];
|
|
10
|
+
if (!apiKey) {
|
|
11
|
+
throw new Error("No OpenAI API key. Set OPENAI_API_KEY or pass --api-key.");
|
|
12
|
+
}
|
|
13
|
+
return new OpenAIEmbedder({ apiKey, model: opts.model, baseUrl: opts.baseUrl });
|
|
14
|
+
}
|
|
15
|
+
if (provider === "ollama") {
|
|
13
16
|
const ollama = new OllamaEmbedder({ model: opts.model, baseUrl: opts.baseUrl });
|
|
14
17
|
if (await ollama.ping())
|
|
15
18
|
return ollama;
|
|
16
|
-
|
|
17
|
-
throw new Error(`Ollama not reachable at ${opts.baseUrl ?? "http://localhost:11434"}. Start it with: ollama serve`);
|
|
18
|
-
}
|
|
19
|
+
throw new Error(`Ollama not reachable at ${opts.baseUrl ?? "http://127.0.0.1:11434"}. Start it with: ollama serve`);
|
|
19
20
|
}
|
|
21
|
+
if (provider === "bundled") {
|
|
22
|
+
return new BundledEmbedder({ model: opts.model, onProgress: opts.onBundledProgress });
|
|
23
|
+
}
|
|
24
|
+
if (provider === "hash") {
|
|
25
|
+
return new HashEmbedder();
|
|
26
|
+
}
|
|
27
|
+
// ── auto: walk the ladder, NEVER block the user ────────────────────
|
|
28
|
+
// Each step is health-checked. A failing step quietly falls to the next.
|
|
29
|
+
// The user always ends up with a working embedder (worst case = hash).
|
|
30
|
+
// 1. OpenAI key wins — best quality, no install.
|
|
20
31
|
const apiKey = opts.apiKey ?? process.env["OPENAI_API_KEY"];
|
|
21
|
-
if (
|
|
32
|
+
if (apiKey) {
|
|
22
33
|
return new OpenAIEmbedder({ apiKey, model: opts.model, baseUrl: opts.baseUrl });
|
|
23
34
|
}
|
|
24
|
-
if
|
|
25
|
-
|
|
35
|
+
// 2. Ollama — only if a SHORT sanity embed succeeds. We had real users
|
|
36
|
+
// where /api/tags responded but /api/embeddings hung for minutes; that
|
|
37
|
+
// used to surface as a hard error. Now we just fall through silently.
|
|
38
|
+
const ollama = new OllamaEmbedder({
|
|
39
|
+
model: opts.model,
|
|
40
|
+
baseUrl: opts.baseUrl,
|
|
41
|
+
timeoutMs: 10_000, // short for the auto-detect probe — full timeout used after
|
|
42
|
+
});
|
|
43
|
+
if (await ollama.ping()) {
|
|
44
|
+
const ver = await ollama.verify();
|
|
45
|
+
if (ver.ok) {
|
|
46
|
+
// Reset to the normal long timeout for the real workload.
|
|
47
|
+
return new OllamaEmbedder({ model: opts.model, baseUrl: opts.baseUrl });
|
|
48
|
+
}
|
|
49
|
+
// Ollama is reachable but unhealthy — explicitly skip and try the next step.
|
|
50
|
+
// We swallow the reason: the user picked auto, they want it to JUST WORK.
|
|
26
51
|
}
|
|
52
|
+
// 3. Bundled WASM — ★★★, ~25MB auto-download. Doesn't need a "ping" — the
|
|
53
|
+
// module is in the npm install. We instantiate eagerly; the actual model
|
|
54
|
+
// download happens lazily on first embed (or via verify()).
|
|
55
|
+
return new BundledEmbedder({ model: opts.model, onProgress: opts.onBundledProgress });
|
|
56
|
+
// 4. Hash is the FINAL escape hatch — only chosen by explicit `--embedder hash`
|
|
57
|
+
// or by callers that don't want network/download. Auto-detect prefers
|
|
58
|
+
// bundled because it returns true semantic vectors with no setup.
|
|
59
|
+
}
|
|
60
|
+
/** Helper for callers that explicitly want the deterministic offline path. */
|
|
61
|
+
export function hashEmbedder() {
|
|
27
62
|
return new HashEmbedder();
|
|
28
63
|
}
|
|
29
64
|
//# sourceMappingURL=resolve.js.map
|
package/dist/resolve.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../src/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"resolve.js","sourceRoot":"","sources":["../src/resolve.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAyBzC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAuB,EAAE;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC;IAEzC,uEAAuE;IACvE,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,IAAI,MAAM,MAAM,CAAC,IAAI,EAAE;YAAE,OAAO,MAAM,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,2BAA2B,IAAI,CAAC,OAAO,IAAI,wBAAwB,+BAA+B,CACnG,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACxB,OAAO,IAAI,YAAY,EAAE,CAAC;IAC5B,CAAC;IAED,sEAAsE;IACtE,yEAAyE;IACzE,uEAAuE;IAEvE,iDAAiD;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC5D,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,uEAAuE;IACvE,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,SAAS,EAAE,MAAM,EAAE,4DAA4D;KAChF,CAAC,CAAC;IACH,IAAI,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,0DAA0D;YAC1D,OAAO,IAAI,cAAc,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,6EAA6E;QAC7E,0EAA0E;IAC5E,CAAC;IAED,0EAA0E;IAC1E,4EAA4E;IAC5E,+DAA+D;IAC/D,OAAO,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAEtF,gFAAgF;IAChF,yEAAyE;IACzE,qEAAqE;AACvE,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,YAAY;IAC1B,OAAO,IAAI,YAAY,EAAE,CAAC;AAC5B,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mneme-ai/embeddings",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Embedding providers (Ollama,
|
|
3
|
+
"version": "0.19.0",
|
|
4
|
+
"description": "Embedding providers (OpenAI, Ollama, bundled-WASM, hash-fallback) for Mneme",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"clean": "tsc -b --clean"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@mneme-ai/core": "0.
|
|
33
|
+
"@mneme-ai/core": "0.19.0",
|
|
34
|
+
"@xenova/transformers": "^2.17.2"
|
|
34
35
|
}
|
|
35
36
|
}
|