@vibe-cafe/vibe-usage 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/sync.js +9 -6
package/package.json
CHANGED
package/src/sync.js
CHANGED
|
@@ -4,6 +4,7 @@ import { ingest, fetchSettings } from './api.js';
|
|
|
4
4
|
import { parsers } from './parsers/index.js';
|
|
5
5
|
|
|
6
6
|
const BATCH_SIZE = 100;
|
|
7
|
+
const SESSION_BATCH_SIZE = 500;
|
|
7
8
|
|
|
8
9
|
function formatBytes(bytes) {
|
|
9
10
|
if (bytes < 1024) return `${bytes}B`;
|
|
@@ -85,7 +86,9 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
85
86
|
|
|
86
87
|
let totalIngested = 0;
|
|
87
88
|
let totalSessionsSynced = 0;
|
|
88
|
-
const
|
|
89
|
+
const bucketBatches = Math.ceil(allBuckets.length / BATCH_SIZE);
|
|
90
|
+
const sessionBatches = Math.ceil(allSessions.length / SESSION_BATCH_SIZE);
|
|
91
|
+
const totalBatches = Math.max(bucketBatches, sessionBatches, 1);
|
|
89
92
|
|
|
90
93
|
const parts = [];
|
|
91
94
|
if (allBuckets.length > 0) parts.push(`${allBuckets.length} buckets`);
|
|
@@ -93,18 +96,18 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
93
96
|
console.log(`Uploading ${parts.join(' + ')} (${totalBatches} batch${totalBatches > 1 ? 'es' : ''})...`);
|
|
94
97
|
|
|
95
98
|
try {
|
|
96
|
-
for (let
|
|
97
|
-
const batch = allBuckets.slice(
|
|
98
|
-
const
|
|
99
|
+
for (let batchIdx = 0; batchIdx < totalBatches; batchIdx++) {
|
|
100
|
+
const batch = allBuckets.slice(batchIdx * BATCH_SIZE, (batchIdx + 1) * BATCH_SIZE);
|
|
101
|
+
const batchSessions = allSessions.slice(batchIdx * SESSION_BATCH_SIZE, (batchIdx + 1) * SESSION_BATCH_SIZE);
|
|
102
|
+
const batchNum = batchIdx + 1;
|
|
99
103
|
const prefix = totalBatches > 1 ? ` [${batchNum}/${totalBatches}] ` : ' ';
|
|
100
|
-
const batchSessions = i === 0 ? allSessions : undefined;
|
|
101
104
|
|
|
102
105
|
const result = await ingest(apiUrl, config.apiKey, batch, {
|
|
103
106
|
onProgress(sent, total) {
|
|
104
107
|
const pct = Math.round((sent / total) * 100);
|
|
105
108
|
process.stdout.write(`\r${prefix}${formatBytes(sent)}/${formatBytes(total)} (${pct}%)\x1b[K`);
|
|
106
109
|
},
|
|
107
|
-
}, batchSessions);
|
|
110
|
+
}, batchSessions.length > 0 ? batchSessions : undefined);
|
|
108
111
|
totalIngested += result.ingested ?? batch.length;
|
|
109
112
|
totalSessionsSynced += result.sessions ?? 0;
|
|
110
113
|
}
|