lance-context 1.26.1 → 1.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../src/embeddings/gemini.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../src/embeddings/gemini.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAepE;;;;;GAKG;AACH,qBAAa,aAAc,YAAW,gBAAgB;IACpD,IAAI,SAAY;IAChB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAsD;IACrE,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,eAAe;IAkB7B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA+BtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IA6CtD;;;OAGG;YACW,gBAAgB;IAiC9B,aAAa,IAAI,MAAM;IAIvB,QAAQ,IAAI,MAAM;CAGnB"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { chunkArray } from './types.js';
|
|
2
2
|
import { fetchWithRetry } from './retry.js';
|
|
3
3
|
import { RateLimiter } from './rate-limiter.js';
|
|
4
|
+
import { broadcastLog, updateProgressMessage } from '../dashboard/events.js';
|
|
4
5
|
/** Default batch size for Gemini API requests (max 100 per batch request) */
|
|
5
6
|
const DEFAULT_BATCH_SIZE = 100;
|
|
6
7
|
/** Default model for Gemini embeddings */
|
|
@@ -74,13 +75,32 @@ export class GeminiBackend {
|
|
|
74
75
|
if (texts.length <= this.batchSize) {
|
|
75
76
|
return this.embedBatchDirect(texts);
|
|
76
77
|
}
|
|
77
|
-
// For large batches, chunk and process sequentially
|
|
78
|
+
// For large batches, chunk and process sequentially with progress logging
|
|
78
79
|
const chunks = chunkArray(texts, this.batchSize);
|
|
79
80
|
const results = [];
|
|
80
|
-
|
|
81
|
+
const startTime = Date.now();
|
|
82
|
+
const initMsg = `Gemini: embedding ${texts.length} texts in ${chunks.length} batches (${this.batchSize} texts/batch)`;
|
|
83
|
+
console.error(`[lance-context] ${initMsg}`);
|
|
84
|
+
broadcastLog('info', initMsg);
|
|
85
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
86
|
+
const chunk = chunks[i];
|
|
87
|
+
const batchNum = i + 1;
|
|
88
|
+
const batchStart = Date.now();
|
|
89
|
+
updateProgressMessage(`Batch ${batchNum}/${chunks.length}: embedding ${chunk.length} texts...`);
|
|
81
90
|
const chunkResults = await this.embedBatchDirect(chunk);
|
|
82
91
|
results.push(...chunkResults);
|
|
92
|
+
const batchElapsed = ((Date.now() - batchStart) / 1000).toFixed(1);
|
|
93
|
+
const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
94
|
+
const textsProcessed = Math.min((i + 1) * this.batchSize, texts.length);
|
|
95
|
+
const progressMsg = `Batch ${batchNum}/${chunks.length}: done in ${batchElapsed}s (${textsProcessed}/${texts.length} texts, ${totalElapsed}s total)`;
|
|
96
|
+
console.error(`[lance-context] ${progressMsg}`);
|
|
97
|
+
broadcastLog('info', progressMsg);
|
|
98
|
+
updateProgressMessage(progressMsg);
|
|
83
99
|
}
|
|
100
|
+
const totalElapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
101
|
+
const completeMsg = `Gemini: completed embedding ${texts.length} texts in ${totalElapsed}s`;
|
|
102
|
+
console.error(`[lance-context] ${completeMsg}`);
|
|
103
|
+
broadcastLog('info', completeMsg);
|
|
84
104
|
return results;
|
|
85
105
|
}
|
|
86
106
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../src/embeddings/gemini.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../src/embeddings/gemini.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAE7E,6EAA6E;AAC7E,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,0CAA0C;AAC1C,MAAM,oBAAoB,GAAG,sBAAsB,CAAC;AAEpD,+EAA+E;AAC/E,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACxB,IAAI,GAAG,QAAQ,CAAC;IACR,KAAK,CAAS;IACd,MAAM,CAAS;IACf,OAAO,GAAG,kDAAkD,CAAC;IAC7D,UAAU,CAAS;IACnB,WAAW,CAAc;IACzB,SAAS,CAAS;IAE1B,YAAY,MAAuB;QACjC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,oBAAoB,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,mIAAmI,CACpI,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAExD,kEAAkE;QAClE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC;YACjC,iBAAiB,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;YAC5C,aAAa,EAAE,MAAM,CAAC,cAAc,IAAI,EAAE;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,oCAAoC;QACpC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,uDAAuD;QACvD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAEjC,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,WAAW,IAAI,CAAC,KAAK,eAAe,CAAC;QAChE,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,IAAI,CAAC,MAAM;aAC9B;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;gBAC7B,OAAO,EAAE;oBACP,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;iBAClB;gBACD,oBAAoB,EAAE,IAAI,CAAC,UAAU;aACtC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,sCAAsC;QACtC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QAED,0EAA0E;QAC1E,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,OAAO,GAAG,qBAAqB,KAAK,CAAC,MAAM,aAAa,MAAM,CAAC,MAAM,aAAa,IAAI,CAAC,SAAS,eAAe,CAAC;QACtH,OAAO,CAAC,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC;QAC5C,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE9B,qBAAqB,CACnB,SAAS,QAAQ,IAAI,MAAM,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,WAAW,CACzE,CAAC;YAEF,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAE9B,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACnE,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAClE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACxE,MAAM,WAAW,GAAG,SAAS,QAAQ,IAAI,MAAM,CAAC,MAAM,aAAa,YAAY,MAAM,cAAc,IAAI,KAAK,CAAC,MAAM,WAAW,YAAY,UAAU,CAAC;YAErJ,OAAO,CAAC,KAAK,CAAC,mBAAmB,WAAW,EAAE,CAAC,CAAC;YAChD,YAAY,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAClC,qBAAqB,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,GAAG,+BAA+B,KAAK,CAAC,MAAM,aAAa,YAAY,GAAG,CAAC;QAC5F,OAAO,CAAC,KAAK,CAAC,mBAAmB,WAAW,EAAE,CAAC,CAAC;QAChD,YAAY,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAElC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,gBAAgB,CAAC,KAAe;QAC5C,uDAAuD;QACvD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAEjC,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,WAAW,IAAI,CAAC,KAAK,qBAAqB,CAAC;QACtE,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,IAAI,CAAC,MAAM;aAC9B;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC7B,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;oBAC7B,OAAO,EAAE;wBACP,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;qBAClB;oBACD,oBAAoB,EAAE,IAAI,CAAC,UAAU;iBACtC,CAAC,CAAC;aACJ,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QACF,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF"}
|
package/package.json
CHANGED