@mneme-ai/core 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,MAAM,EACN,WAAW,EACX,iBAAiB,EACjB,eAAe,EAChB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAqB,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAE1E,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IAC1C;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,aAAa,CAAC;CAClC;AAED,qBAAa,OAAO;IACN,OAAO,CAAC,IAAI;gBAAJ,IAAI,EAAE,cAAc;IAElC,GAAG,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,MAAM,EACN,WAAW,EACX,iBAAiB,EACjB,eAAe,EAChB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAqB,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAE1E,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IAC1C;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,aAAa,CAAC;CAClC;AAED,qBAAa,OAAO;IACN,OAAO,CAAC,IAAI;gBAAJ,IAAI,EAAE,cAAc;IAElC,GAAG,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;CA4FjG;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,WAAW,EAAE,CAmB5D;AAOD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,SAAM,GAAG,MAAM,EAAE,CAgBhE"}
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -60,6 +60,13 @@ export class Indexer {
|
|
|
60
60
|
if (this.opts.embedder) {
|
|
61
61
|
const model = this.opts.embedder.name;
|
|
62
62
|
const batchSize = this.opts.embedBatchSize ?? 32;
|
|
63
|
+
// Tell the user the first batch may take longer (cold-start model load).
|
|
64
|
+
report({
|
|
65
|
+
phase: "embedding",
|
|
66
|
+
current: 0,
|
|
67
|
+
total: chunks.length,
|
|
68
|
+
message: `warming up ${model} (first batch may take ~30s)`,
|
|
69
|
+
});
|
|
63
70
|
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
64
71
|
const batch = chunks.slice(i, i + batchSize);
|
|
65
72
|
const vecs = await this.opts.embedder.embed(batch.map((c) => c.text));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAE7D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAsB,MAAM,mBAAmB,CAAC;AAsB1E,MAAM,OAAO,OAAO;IACE;IAApB,YAAoB,IAAoB;QAApB,SAAI,GAAJ,IAAI,CAAgB;IAAG,CAAC;IAE5C,KAAK,CAAC,GAAG;QACP,MAAM,MAAM,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjE,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC,CAAC;QACnF,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG;YAClB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;SAC7B,CAAC,CAAC;QAEH,MAAM,CAAC;YACL,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,OAAO,EAAE,cAAc,OAAO,CAAC,MAAM,UAAU;SAChD,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACtB,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM,CAAC;oBACL,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,OAAO,CAAC,MAAM;oBACrB,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEpC,4EAA4E;QAC5E,6EAA6E;QAC7E,IAAI,aAAa,GAA2B,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAkB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACpC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACpB,aAAa,GAAG,SAAS,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC1E,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,CAAC;oBACL,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,SAAS;oBAClB,KAAK,EAAE,MAAM,CAAC,MAAM;oBACpB,OAAO,EAAE,YAAY,SAAS,qBAAqB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,UAAU;iBAC/F,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAE7D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAsB,MAAM,mBAAmB,CAAC;AAsB1E,MAAM,OAAO,OAAO;IACE;IAApB,YAAoB,IAAoB;QAApB,SAAI,GAAJ,IAAI,CAAgB;IAAG,CAAC;IAE5C,KAAK,CAAC,GAAG;QACP,MAAM,MAAM,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjE,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC,CAAC;QACnF,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG;YAClB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;SAC7B,CAAC,CAAC;QAEH,MAAM,CAAC;YACL,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,OAAO,EAAE,cAAc,OAAO,CAAC,MAAM,UAAU;SAChD,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACtB,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM,CAAC;oBACL,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,OAAO,CAAC,MAAM;oBACrB,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEpC,4EAA4E;QAC5E,6EAA6E;QAC7E,IAAI,aAAa,GAA2B,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAkB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACpC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACpB,aAAa,GAAG,SAAS,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC1E,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,CAAC;oBACL,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,SAAS;oBAClB,KAAK,EAAE,MAAM,CAAC,MAAM;oBACpB,OAAO,EAAE,YAAY,SAAS,qBAAqB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,UAAU;iBAC/F,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;YAEjD,yEAAyE;YACzE,MAAM,CAAC;gBACL,KAAK,EAAE,WAAW;gBAClB,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,OAAO,EAAE,cAAc,KAAK,8BAA8B;aAC3D,CAAC,CAAC;YAEH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBAClD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;oBAAE,KAAK,CAAC,CAAC,CAAE,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC3C,MAAM,CAAC;oBACL,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC;oBAC/C,KAAK,EAAE,MAAM,CAAC,MAAM;oBACpB,OAAO,EAAE,kBAAkB,KAAK,EAAE;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACxE,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa;SACd,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,WAAW,CAAC,OAAiB;IAC3C,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACX,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,UAAkB,EAAE,IAAyB,EAAE,IAAY;IAC5E,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,UAAU,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjG,OAAO,EAAE,EAAE,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,QAAQ,GAAG,GAAG;IACpD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;YAAE,SAAS;QACxB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,IAAI,GAAG,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChB,GAAG,GAAG,CAAC,CAAC;QACV,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,IAAI,GAAG;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,KAAK,CAAC;AACf,CAAC"}
|