@vivantel/virage-embedder-transformers 0.2.32 → 0.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedder.d.ts +1 -0
- package/dist/embedder.d.ts.map +1 -1
- package/dist/embedder.js +26 -1
- package/dist/embedder.js.map +1 -1
- package/package.json +2 -2
package/dist/embedder.d.ts
CHANGED
|
@@ -27,6 +27,7 @@ export declare class TransformersEmbedder implements EmbeddingProvider {
|
|
|
27
27
|
private logger;
|
|
28
28
|
constructor(options: TransformersEmbedderOptions);
|
|
29
29
|
setLogger(logger: Logger): void;
|
|
30
|
+
preWarm(onProgress?: (loaded: number, total: number) => void): Promise<void>;
|
|
30
31
|
private getPipeline;
|
|
31
32
|
private _loadPipeline;
|
|
32
33
|
embed(text: string): Promise<number[]>;
|
package/dist/embedder.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEvE,MAAM,WAAW,mBAAmB;IAClC,oFAAoF;IACpF,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAC1B,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wFAAwF;IACxF,YAAY,CAAC,EAAE,mBAAmB,CAAC;CACpC;AAWD,qBAAa,oBAAqB,YAAW,iBAAiB;IAC5D,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,kBAAkB,MAAM;IAEjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAsB;IACpD,OAAO,CAAC,gBAAgB,CAAkC;IAC1D,OAAO,CAAC,MAAM,CAAuB;gBAEzB,OAAO,EAAE,2BAA2B;IAWhD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEvE,MAAM,WAAW,mBAAmB;IAClC,oFAAoF;IACpF,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,2DAA2D;IAC3D,KAAK,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC;IAC1B,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wFAAwF;IACxF,YAAY,CAAC,EAAE,mBAAmB,CAAC;CACpC;AAWD,qBAAa,oBAAqB,YAAW,iBAAiB;IAC5D,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,kBAAkB,MAAM;IAEjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAsB;IACpD,OAAO,CAAC,gBAAgB,CAAkC;IAC1D,OAAO,CAAC,MAAM,CAAuB;gBAEzB,OAAO,EAAE,2BAA2B;IAWhD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIzB,OAAO,CACX,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACnD,OAAO,CAAC,IAAI,CAAC;IAOhB,OAAO,CAAC,WAAW;YAOL,aAAa;IAwDrB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAMtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAuBhD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;CAQtC"}
|
package/dist/embedder.js
CHANGED
|
@@ -23,21 +23,46 @@ export class TransformersEmbedder {
|
|
|
23
23
|
setLogger(logger) {
|
|
24
24
|
this.logger = logger.withTag("transformers");
|
|
25
25
|
}
|
|
26
|
+
async preWarm(onProgress) {
|
|
27
|
+
if (!this._pipelinePromise) {
|
|
28
|
+
this._pipelinePromise = this._loadPipeline(onProgress);
|
|
29
|
+
}
|
|
30
|
+
await this._pipelinePromise;
|
|
31
|
+
}
|
|
26
32
|
getPipeline() {
|
|
27
33
|
if (!this._pipelinePromise) {
|
|
28
34
|
this._pipelinePromise = this._loadPipeline();
|
|
29
35
|
}
|
|
30
36
|
return this._pipelinePromise;
|
|
31
37
|
}
|
|
32
|
-
async _loadPipeline() {
|
|
38
|
+
async _loadPipeline(onProgress) {
|
|
33
39
|
this.logger?.info(`Loading model ${this.model}`);
|
|
34
40
|
this.logger?.debug(`Device: ${this.device}${this.quantization ? `, dtype: ${this.quantization.dtype}` : ""}`);
|
|
35
41
|
// Lazy import — consumers must install @huggingface/transformers
|
|
36
42
|
const { pipeline, env } = await import("@huggingface/transformers");
|
|
37
43
|
env.cacheDir = this.cacheDir;
|
|
44
|
+
// Track per-file download progress and aggregate into a single callback
|
|
45
|
+
const fileTotals = new Map();
|
|
46
|
+
const fileLoaded = new Map();
|
|
47
|
+
const progressCallback = onProgress
|
|
48
|
+
? (event) => {
|
|
49
|
+
if (event.status === "progress" &&
|
|
50
|
+
event.name &&
|
|
51
|
+
typeof event.loaded === "number" &&
|
|
52
|
+
typeof event.total === "number" &&
|
|
53
|
+
event.total > 0) {
|
|
54
|
+
fileTotals.set(event.name, event.total);
|
|
55
|
+
fileLoaded.set(event.name, event.loaded);
|
|
56
|
+
const sumTotal = [...fileTotals.values()].reduce((a, b) => a + b, 0);
|
|
57
|
+
const sumLoaded = [...fileLoaded.values()].reduce((a, b) => a + b, 0);
|
|
58
|
+
onProgress(sumLoaded, sumTotal);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
: undefined;
|
|
38
62
|
const pipe = (await pipeline("feature-extraction", this.model, {
|
|
39
63
|
device: this.device,
|
|
40
64
|
...(this.quantization ? { dtype: this.quantization.dtype } : {}),
|
|
65
|
+
...(progressCallback ? { progress_callback: progressCallback } : {}),
|
|
41
66
|
}));
|
|
42
67
|
this.logger?.info(`Model ${this.model} ready (${this.dimensions}d, device=${this.device})`);
|
|
43
68
|
return pipe;
|
package/dist/embedder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA8BjC,MAAM,OAAO,oBAAoB;IACtB,IAAI,GAAG,cAAc,CAAC;IACtB,UAAU,CAAS;IACnB,KAAK,CAAS;IACd,kBAAkB,GAAG,EAAE,CAAC;IAEhB,MAAM,CAAmB;IACzB,QAAQ,CAAS;IACjB,YAAY,CAAuB;IAC5C,gBAAgB,GAA6B,IAAI,CAAC;IAClD,MAAM,GAAkB,IAAI,CAAC;IAErC,YAAY,OAAoC;QAC9C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;QACtC,sEAAsE;QACtE,0EAA0E;QAC1E,IAAI,CAAC,QAAQ;YACX,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;QACtE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,SAAS,CAAC,MAAc;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,WAAW;QACjB,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,aAAa;
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA8BjC,MAAM,OAAO,oBAAoB;IACtB,IAAI,GAAG,cAAc,CAAC;IACtB,UAAU,CAAS;IACnB,KAAK,CAAS;IACd,kBAAkB,GAAG,EAAE,CAAC;IAEhB,MAAM,CAAmB;IACzB,QAAQ,CAAS;IACjB,YAAY,CAAuB;IAC5C,gBAAgB,GAA6B,IAAI,CAAC;IAClD,MAAM,GAAkB,IAAI,CAAC;IAErC,YAAY,OAAoC;QAC9C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;QACtC,sEAAsE;QACtE,0EAA0E;QAC1E,IAAI,CAAC,QAAQ;YACX,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;QACtE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,SAAS,CAAC,MAAc;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,OAAO,CACX,UAAoD;QAEpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,IAAI,CAAC,gBAAgB,CAAC;IAC9B,CAAC;IAEO,WAAW;QACjB,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,UAAoD;QAEpD,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,iBAAiB,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB,WAAW,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1F,CAAC;QAEF,iEAAiE;QACjE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACpE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE7B,wEAAwE;QACxE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC7C,MAAM,gBAAgB,GAAG,UAAU;YACjC,CAAC,CAAC,CAAC,KAKA,EAAE,EAAE;gBACH,IACE,KAAK,CAAC,MAAM,KAAK,UAAU;oBAC3B,KAAK,CAAC,IAAI;oBACV,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;oBAChC,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;oBAC/B,KAAK,CAAC,KAAK,GAAG,CAAC,EACf,CAAC;oBACD,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBACxC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;oBACzC,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAC9C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EACf,CAAC,CACF,CAAC;oBACF,MAAM,SAAS,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAC/C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EACf,CAAC,CACF,CAAC;oBACF,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YACH,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,EAAE;YAC7D,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAwB,CAAC;QAE3B,IAAI,CAAC,MAAM,EAAE,IAAI,CACf,SAAS,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,UAAU,aAAa,IAAI,CAAC,MAAM,GAAG,CACzE,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxE,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAa,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB,iBAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACzD,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB,oBAAoB,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,UAAU,KAAK,CAAC,MAAM,QAAQ,CACrE,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACjD,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CACR,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAChD,CACd,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vivantel/virage-embedder-transformers",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.34",
|
|
4
4
|
"description": "Local transformers.js embedding provider for @vivantel/rag-core",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"@vivantel/virage-core": "*"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
|
-
"@vivantel/virage-core": "0.2.
|
|
55
|
+
"@vivantel/virage-core": "0.2.32",
|
|
56
56
|
"@types/node": "^25.9.3",
|
|
57
57
|
"typescript": "^6.0.3",
|
|
58
58
|
"vitest": "^4.1.9"
|