@astrofoundry/grimoire 1.15.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedder.d.ts.map +1 -1
- package/dist/embedder.js +8 -3
- package/dist/embedder.js.map +1 -1
- package/package.json +1 -1
package/dist/embedder.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAsBA,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CA0CrE;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG/D"}
|
package/dist/embedder.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { GoogleGenerativeAI } from "@google/generative-ai";
|
|
2
|
-
const BATCH_SIZE =
|
|
2
|
+
const BATCH_SIZE = 50;
|
|
3
3
|
const MODEL = "gemini-embedding-001";
|
|
4
4
|
const OUTPUT_DIMENSIONALITY = 768;
|
|
5
5
|
const MAX_RETRIES = 5;
|
|
6
|
-
const BASE_DELAY_MS =
|
|
6
|
+
const BASE_DELAY_MS = 60000;
|
|
7
|
+
const BATCH_DELAY_MS = 2500;
|
|
7
8
|
let genAI;
|
|
8
9
|
function getClient() {
|
|
9
10
|
if (!genAI) {
|
|
@@ -36,16 +37,20 @@ export async function embedTexts(texts) {
|
|
|
36
37
|
const message = err instanceof Error ? err.message : String(err);
|
|
37
38
|
if (message.includes("429") && attempt < MAX_RETRIES - 1) {
|
|
38
39
|
const delay = BASE_DELAY_MS * Math.pow(2, attempt);
|
|
39
|
-
console.log(` Rate limited, retrying in ${delay / 1000}s...`);
|
|
40
|
+
console.log(` Rate limited (batch ${i / BATCH_SIZE + 1}), retrying in ${delay / 1000}s...`);
|
|
40
41
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
41
42
|
continue;
|
|
42
43
|
}
|
|
44
|
+
console.error(` Embedding failed at batch ${i / BATCH_SIZE + 1} (chunks ${i + 1}-${i + batch.length}): ${message}`);
|
|
43
45
|
throw err;
|
|
44
46
|
}
|
|
45
47
|
}
|
|
46
48
|
for (const embedding of result.embeddings) {
|
|
47
49
|
embeddings.push(embedding.values);
|
|
48
50
|
}
|
|
51
|
+
if (i + BATCH_SIZE < texts.length) {
|
|
52
|
+
await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY_MS));
|
|
53
|
+
}
|
|
49
54
|
}
|
|
50
55
|
return embeddings;
|
|
51
56
|
}
|
package/dist/embedder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,UAAU,GAAG,
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,KAAK,GAAG,sBAAsB,CAAC;AACrC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,cAAc,GAAG,IAAI,CAAC;AAE5B,IAAI,KAAqC,CAAC;AAE1C,SAAS,SAAS;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAe;IAC9C,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAE7C,IAAI,MAAM,CAAC;QACX,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC;oBACtC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;wBAC7B,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;wBAC5C,oBAAoB,EAAE,qBAAqB;qBAC5C,CAAC,CAAC;iBACJ,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;oBACzD,MAAM,KAAK,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;oBACnD,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,GAAG,UAAU,GAAG,CAAC,kBAAkB,KAAK,GAAG,IAAI,MAAM,CAAC,CAAC;oBAC7F,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;oBAC3D,SAAS;gBACX,CAAC;gBACD,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,GAAG,UAAU,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;gBACrH,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,MAAO,CAAC,UAAU,EAAE,CAAC;YAC3C,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC;AACnB,CAAC"}
|