codevault 1.3.0-beta.2 → 1.3.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-indexer.d.ts","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"batch-indexer.d.ts","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AA4BlD,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,iBAAiB,EAAE,GAAG,CAAC;QACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;QAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,WAAW,EAAE,GAAG,CAAC;KAClB,CAAC;CACH;AAED,qBAAa,uBAAuB;IAKhC,OAAO,CAAC,iBAAiB;IACzB,OAAO,CAAC,EAAE;IALZ,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,SAAS,CAAS;gBAGhB,iBAAiB,EAAE,iBAAiB,EACpC,EAAE,EAAE,QAAQ,EACpB,SAAS,GAAE,MAAmB;IAKhC;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B;;OAEG;YACW,YAAY;IAgF1B;;OAEG;IACH,aAAa,IAAI,MAAM;CAGxB"}
|
|
@@ -1,4 +1,23 @@
|
|
|
1
1
|
import { BATCH_SIZE } from '../providers/base.js';
|
|
2
|
+
const MAX_BATCH_RETRIES = 3;
|
|
3
|
+
const INITIAL_RETRY_DELAY_MS = 1000;
|
|
4
|
+
function isRateLimitError(error) {
|
|
5
|
+
const message = error?.message || String(error);
|
|
6
|
+
return (error?.status === 429 ||
|
|
7
|
+
error?.statusCode === 429 ||
|
|
8
|
+
message.includes('rate limit') ||
|
|
9
|
+
message.includes('Rate limit') ||
|
|
10
|
+
message.includes('too many requests') ||
|
|
11
|
+
message.includes('429'));
|
|
12
|
+
}
|
|
13
|
+
function isBatchSizeError(error) {
|
|
14
|
+
const message = error?.message || String(error);
|
|
15
|
+
return (error?.status === 413 ||
|
|
16
|
+
message.includes('too large') ||
|
|
17
|
+
message.includes('payload') ||
|
|
18
|
+
message.includes('request size') ||
|
|
19
|
+
message.includes('token limit'));
|
|
20
|
+
}
|
|
2
21
|
export class BatchEmbeddingProcessor {
|
|
3
22
|
embeddingProvider;
|
|
4
23
|
db;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAIlD,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD,MAAM,OAAO,uBAAuB;IAKxB;IACA;IALF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAE1B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,8CAA8C;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEpC,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAEhB,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAErE,wBAAwB;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,CAAC,MAAM,YAAY,CAAC,CAAC;YAClE,CAAC;YAED,yDAAyD;YACzD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAE1E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;YACzE,CAAC;YAED,mCAAmC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAEhC,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBACxB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,uDAAuD;YACvD,OAAO,CAAC,KAAK,CAAC,mCAAmC,YAAY,CAAC,MAAM,SAAS,CAAC,CAAC;YAC/E,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,OAAO,CAAC,GAAG,CAAC,sEAAsE,CAAC,CAAC;YAEpF,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;oBAE9F,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;wBACxB,EAAE,EAAE,KAAK,CAAC,OAAO;wBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;wBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;wBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;wBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;wBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;wBAClC,SAAS;wBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;wBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;wBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;wBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;wBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;wBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;wBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;wBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;qBACvC,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,eAAe,EAAE,CAAC;oBACzB,OAAO,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,GAAG,EAAE,eAAe,CAAC,CAAC;oBAC5E,MAAM,eAAe,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}
|