@nocoo/pew 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1 -1
- package/dist/commands/session-upload.d.ts +6 -22
- package/dist/commands/session-upload.d.ts.map +1 -1
- package/dist/commands/session-upload.js +12 -144
- package/dist/commands/session-upload.js.map +1 -1
- package/dist/commands/upload-engine.d.ts +63 -0
- package/dist/commands/upload-engine.d.ts.map +1 -0
- package/dist/commands/upload-engine.js +164 -0
- package/dist/commands/upload-engine.js.map +1 -0
- package/dist/commands/upload.d.ts +5 -21
- package/dist/commands/upload.d.ts.map +1 -1
- package/dist/commands/upload.js +11 -144
- package/dist/commands/upload.js.map +1 -1
- package/dist/storage/base-queue.d.ts +40 -0
- package/dist/storage/base-queue.d.ts.map +1 -0
- package/dist/storage/base-queue.js +89 -0
- package/dist/storage/base-queue.js.map +1 -0
- package/dist/storage/local-queue.d.ts +4 -24
- package/dist/storage/local-queue.d.ts.map +1 -1
- package/dist/storage/local-queue.js +5 -64
- package/dist/storage/local-queue.js.map +1 -1
- package/dist/storage/session-queue.d.ts +3 -21
- package/dist/storage/session-queue.d.ts.map +1 -1
- package/dist/storage/session-queue.js +4 -58
- package/dist/storage/session-queue.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -318,7 +318,7 @@ async function runSessionUpload(stateDir, apiUrl, dev) {
|
|
|
318
318
|
export const main = defineCommand({
|
|
319
319
|
meta: {
|
|
320
320
|
name: "pew",
|
|
321
|
-
version: "0.
|
|
321
|
+
version: "0.6.1",
|
|
322
322
|
description: "Track token usage from your local AI coding tools",
|
|
323
323
|
},
|
|
324
324
|
subCommands: {
|
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* CLI session upload command — sends local session queue records to the Pew SaaS.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 2. Read un-uploaded session records from queue (using saved offset)
|
|
7
|
-
* 3. Deduplicate: keep only latest snapshot per session_key
|
|
8
|
-
* 4. Split into batches of ≤50 (D1 Free plan limit)
|
|
9
|
-
* 5. POST each batch to /api/ingest/sessions with Bearer token
|
|
10
|
-
* 6. Persist offset after all batches succeed
|
|
11
|
-
* 7. Retry on 5xx/429 with exponential backoff
|
|
4
|
+
* Thin wrapper around the generic upload engine with session-specific
|
|
5
|
+
* preprocessing (deduplication: keep only latest snapshot per session_key).
|
|
12
6
|
*/
|
|
7
|
+
import type { UploadResult, UploadProgressEvent } from "./upload-engine.js";
|
|
13
8
|
import type { SessionQueueRecord } from "@pew/core";
|
|
14
9
|
export interface SessionUploadOptions {
|
|
15
10
|
/** Directory for config file and queue state */
|
|
@@ -29,19 +24,8 @@ export interface SessionUploadOptions {
|
|
|
29
24
|
/** Progress callback */
|
|
30
25
|
onProgress?: (event: SessionUploadProgressEvent) => void;
|
|
31
26
|
}
|
|
32
|
-
export
|
|
33
|
-
|
|
34
|
-
batch?: number;
|
|
35
|
-
totalBatches?: number;
|
|
36
|
-
total?: number;
|
|
37
|
-
message?: string;
|
|
38
|
-
}
|
|
39
|
-
export interface SessionUploadResult {
|
|
40
|
-
success: boolean;
|
|
41
|
-
uploaded: number;
|
|
42
|
-
batches: number;
|
|
43
|
-
error?: string;
|
|
44
|
-
}
|
|
27
|
+
export type SessionUploadProgressEvent = UploadProgressEvent;
|
|
28
|
+
export type SessionUploadResult = UploadResult;
|
|
45
29
|
/**
|
|
46
30
|
* Unlike token's aggregateRecords() which SUMS, session dedup
|
|
47
31
|
* keeps only the LATEST snapshot per session_key.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-upload.d.ts","sourceRoot":"","sources":["../../src/commands/session-upload.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"session-upload.d.ts","sourceRoot":"","sources":["../../src/commands/session-upload.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EAEpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAMpD,MAAM,WAAW,oBAAoB;IACnC,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,mCAAmC;IACnC,KAAK,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAC/B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;CAC1D;AAED,MAAM,MAAM,0BAA0B,GAAG,mBAAmB,CAAC;AAC7D,MAAM,MAAM,mBAAmB,GAAG,YAAY,CAAC;AAM/C;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,kBAAkB,EAAE,GAC5B,kBAAkB,EAAE,CAWtB;AAMD,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,mBAAmB,CAAC,CAW9B"}
|
|
@@ -1,25 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* CLI session upload command — sends local session queue records to the Pew SaaS.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 2. Read un-uploaded session records from queue (using saved offset)
|
|
7
|
-
* 3. Deduplicate: keep only latest snapshot per session_key
|
|
8
|
-
* 4. Split into batches of ≤50 (D1 Free plan limit)
|
|
9
|
-
* 5. POST each batch to /api/ingest/sessions with Bearer token
|
|
10
|
-
* 6. Persist offset after all batches succeed
|
|
11
|
-
* 7. Retry on 5xx/429 with exponential backoff
|
|
4
|
+
* Thin wrapper around the generic upload engine with session-specific
|
|
5
|
+
* preprocessing (deduplication: keep only latest snapshot per session_key).
|
|
12
6
|
*/
|
|
13
|
-
import { ConfigManager } from "../config/manager.js";
|
|
14
7
|
import { SessionQueue } from "../storage/session-queue.js";
|
|
8
|
+
import { createUploadEngine } from "./upload-engine.js";
|
|
15
9
|
// ---------------------------------------------------------------------------
|
|
16
|
-
//
|
|
17
|
-
// ---------------------------------------------------------------------------
|
|
18
|
-
const DEFAULT_BATCH_SIZE = 50;
|
|
19
|
-
const DEFAULT_MAX_RETRIES = 2;
|
|
20
|
-
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
// Upload dedup
|
|
10
|
+
// Pre-dedup — keep only the latest snapshot per session_key
|
|
23
11
|
// ---------------------------------------------------------------------------
|
|
24
12
|
/**
|
|
25
13
|
* Unlike token's aggregateRecords() which SUMS, session dedup
|
|
@@ -45,133 +33,13 @@ export function deduplicateSessionRecords(records) {
|
|
|
45
33
|
// Implementation
|
|
46
34
|
// ---------------------------------------------------------------------------
|
|
47
35
|
export async function executeSessionUpload(opts) {
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
success: false,
|
|
55
|
-
uploaded: 0,
|
|
56
|
-
batches: 0,
|
|
57
|
-
error: "Not logged in. Run `pew login` first.",
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
// 2. Read un-uploaded records
|
|
61
|
-
const queue = new SessionQueue(stateDir);
|
|
62
|
-
const currentOffset = await queue.loadOffset();
|
|
63
|
-
const { records: rawRecords, newOffset } = await queue.readFromOffset(currentOffset);
|
|
64
|
-
if (rawRecords.length === 0) {
|
|
65
|
-
return { success: true, uploaded: 0, batches: 0 };
|
|
66
|
-
}
|
|
67
|
-
// 2b. Pre-deduplicate: keep only latest snapshot per session_key
|
|
68
|
-
const records = deduplicateSessionRecords(rawRecords);
|
|
69
|
-
// 3. Split into batches
|
|
70
|
-
const batches = [];
|
|
71
|
-
for (let i = 0; i < records.length; i += batchSize) {
|
|
72
|
-
batches.push(records.slice(i, i + batchSize));
|
|
73
|
-
}
|
|
74
|
-
// 4. Upload each batch
|
|
75
|
-
const endpoint = `${apiUrl}/api/ingest/sessions`;
|
|
76
|
-
let totalUploaded = 0;
|
|
77
|
-
let batchesCompleted = 0;
|
|
78
|
-
for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
|
|
79
|
-
const batch = batches[batchIdx];
|
|
80
|
-
onProgress?.({
|
|
81
|
-
phase: "uploading",
|
|
82
|
-
batch: batchIdx + 1,
|
|
83
|
-
totalBatches: batches.length,
|
|
84
|
-
total: records.length,
|
|
85
|
-
message: `Uploading session batch ${batchIdx + 1}/${batches.length} (${batch.length} records)...`,
|
|
86
|
-
});
|
|
87
|
-
const result = await sendBatchWithRetry({
|
|
88
|
-
endpoint,
|
|
89
|
-
token: config.token,
|
|
90
|
-
batch,
|
|
91
|
-
fetchFn,
|
|
92
|
-
maxRetries,
|
|
93
|
-
retryDelayMs,
|
|
94
|
-
});
|
|
95
|
-
if (!result.ok) {
|
|
96
|
-
return {
|
|
97
|
-
success: false,
|
|
98
|
-
uploaded: totalUploaded,
|
|
99
|
-
batches: batchesCompleted,
|
|
100
|
-
error: result.error,
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
totalUploaded += batch.length;
|
|
104
|
-
batchesCompleted++;
|
|
105
|
-
}
|
|
106
|
-
// 5. All batches succeeded — save final offset
|
|
107
|
-
await queue.saveOffset(newOffset);
|
|
108
|
-
onProgress?.({
|
|
109
|
-
phase: "done",
|
|
110
|
-
total: totalUploaded,
|
|
111
|
-
message: `Uploaded ${totalUploaded} session records in ${batchesCompleted} batch(es).`,
|
|
36
|
+
const queue = new SessionQueue(opts.stateDir);
|
|
37
|
+
const engine = createUploadEngine({
|
|
38
|
+
queue,
|
|
39
|
+
endpoint: "/api/ingest/sessions",
|
|
40
|
+
entityName: "session records",
|
|
41
|
+
preprocess: deduplicateSessionRecords,
|
|
112
42
|
});
|
|
113
|
-
return
|
|
114
|
-
success: true,
|
|
115
|
-
uploaded: totalUploaded,
|
|
116
|
-
batches: batchesCompleted,
|
|
117
|
-
};
|
|
118
|
-
}
|
|
119
|
-
async function sendBatchWithRetry(opts) {
|
|
120
|
-
const { endpoint, token, batch, fetchFn, maxRetries, retryDelayMs } = opts;
|
|
121
|
-
let lastError = "";
|
|
122
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
123
|
-
if (attempt > 0 && retryDelayMs > 0) {
|
|
124
|
-
await sleep(retryDelayMs * 2 ** (attempt - 1));
|
|
125
|
-
}
|
|
126
|
-
try {
|
|
127
|
-
const resp = await fetchFn(endpoint, {
|
|
128
|
-
method: "POST",
|
|
129
|
-
headers: {
|
|
130
|
-
"Content-Type": "application/json",
|
|
131
|
-
Authorization: `Bearer ${token}`,
|
|
132
|
-
},
|
|
133
|
-
body: JSON.stringify(batch),
|
|
134
|
-
});
|
|
135
|
-
if (resp.ok) {
|
|
136
|
-
return { ok: true };
|
|
137
|
-
}
|
|
138
|
-
// 429 — rate limited, retry with Retry-After if available
|
|
139
|
-
if (resp.status === 429) {
|
|
140
|
-
const retryAfter = resp.headers.get("Retry-After");
|
|
141
|
-
const retryMs = retryAfter
|
|
142
|
-
? Math.max(Number(retryAfter) * 1000, retryDelayMs)
|
|
143
|
-
: retryDelayMs * 2 ** attempt;
|
|
144
|
-
if (attempt < maxRetries && retryMs > 0) {
|
|
145
|
-
await sleep(retryMs);
|
|
146
|
-
}
|
|
147
|
-
const body = await resp.json().catch(() => ({}));
|
|
148
|
-
lastError = `429: ${body.error ?? "Too Many Requests"}`;
|
|
149
|
-
continue;
|
|
150
|
-
}
|
|
151
|
-
// 4xx — client error, don't retry
|
|
152
|
-
if (resp.status >= 400 && resp.status < 500) {
|
|
153
|
-
const body = await resp.json().catch(() => ({}));
|
|
154
|
-
const msg = body.error ?? `HTTP ${resp.status}`;
|
|
155
|
-
return { ok: false, error: `${resp.status}: ${msg}` };
|
|
156
|
-
}
|
|
157
|
-
// 5xx — server error, retry
|
|
158
|
-
const body = await resp.json().catch(() => ({}));
|
|
159
|
-
lastError = `${resp.status}: ${body.error ?? "Server Error"}`;
|
|
160
|
-
}
|
|
161
|
-
catch (err) {
|
|
162
|
-
lastError = String(err.message ?? err);
|
|
163
|
-
// Network errors — don't retry if maxRetries is 0
|
|
164
|
-
if (attempt >= maxRetries) {
|
|
165
|
-
return { ok: false, error: lastError };
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
return {
|
|
170
|
-
ok: false,
|
|
171
|
-
error: `Upload failed after ${maxRetries + 1} attempts: ${lastError}`,
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
function sleep(ms) {
|
|
175
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
43
|
+
return engine.execute(opts);
|
|
176
44
|
}
|
|
177
45
|
//# sourceMappingURL=session-upload.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-upload.js","sourceRoot":"","sources":["../../src/commands/session-upload.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"session-upload.js","sourceRoot":"","sources":["../../src/commands/session-upload.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAkCxD,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CACvC,OAA6B;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA8B,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;YACtD,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAA0B;IAE1B,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG,kBAAkB,CAAqB;QACpD,KAAK;QACL,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,iBAAiB;QAC7B,UAAU,EAAE,yBAAyB;KACtC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic upload engine — sends queued records to a Pew SaaS endpoint.
|
|
3
|
+
*
|
|
4
|
+
* This is the shared pipeline used by both token and session upload commands.
|
|
5
|
+
* It handles:
|
|
6
|
+
* - Config/API-key loading
|
|
7
|
+
* - Queue offset management
|
|
8
|
+
* - Preprocessing (aggregation for tokens, dedup for sessions)
|
|
9
|
+
* - Batching (≤50 records per request, D1 free plan limit)
|
|
10
|
+
* - Retry with exponential backoff on 5xx
|
|
11
|
+
* - 429 rate-limit handling with Retry-After (no double-sleep)
|
|
12
|
+
* - Progress callbacks
|
|
13
|
+
*
|
|
14
|
+
* Bug fixes over the original upload/session-upload:
|
|
15
|
+
* - 429 handler no longer double-sleeps (Retry-After + exponential backoff).
|
|
16
|
+
* Now the Retry-After sleep replaces the exponential backoff for that attempt.
|
|
17
|
+
*/
|
|
18
|
+
import type { BaseQueue } from "../storage/base-queue.js";
|
|
19
|
+
export interface UploadEngineConfig<T> {
|
|
20
|
+
/** The queue to read records from */
|
|
21
|
+
queue: BaseQueue<T>;
|
|
22
|
+
/** API endpoint path (e.g. "/api/ingest" or "/api/ingest/sessions") */
|
|
23
|
+
endpoint: string;
|
|
24
|
+
/** Human-readable name for progress messages (e.g. "records", "session records") */
|
|
25
|
+
entityName: string;
|
|
26
|
+
/** Pre-processing step: aggregation for tokens, dedup for sessions */
|
|
27
|
+
preprocess: (records: T[]) => T[];
|
|
28
|
+
}
|
|
29
|
+
export interface UploadExecuteOptions {
|
|
30
|
+
/** Directory for config file and queue state */
|
|
31
|
+
stateDir: string;
|
|
32
|
+
/** Base URL of the Pew SaaS */
|
|
33
|
+
apiUrl: string;
|
|
34
|
+
/** Whether dev mode is active (uses config.dev.json) */
|
|
35
|
+
dev?: boolean;
|
|
36
|
+
/** Injected fetch (for testing) */
|
|
37
|
+
fetch: typeof globalThis.fetch;
|
|
38
|
+
/** Max records per API request (default: 50) */
|
|
39
|
+
batchSize?: number;
|
|
40
|
+
/** Max retries per batch on 5xx (default: 2) */
|
|
41
|
+
maxRetries?: number;
|
|
42
|
+
/** Base retry delay in ms (default: 1000, doubled each retry) */
|
|
43
|
+
retryDelayMs?: number;
|
|
44
|
+
/** Progress callback */
|
|
45
|
+
onProgress?: (event: UploadProgressEvent) => void;
|
|
46
|
+
}
|
|
47
|
+
export interface UploadProgressEvent {
|
|
48
|
+
phase: "uploading" | "done";
|
|
49
|
+
batch?: number;
|
|
50
|
+
totalBatches?: number;
|
|
51
|
+
total?: number;
|
|
52
|
+
message?: string;
|
|
53
|
+
}
|
|
54
|
+
export interface UploadResult {
|
|
55
|
+
success: boolean;
|
|
56
|
+
uploaded: number;
|
|
57
|
+
batches: number;
|
|
58
|
+
error?: string;
|
|
59
|
+
}
|
|
60
|
+
export declare function createUploadEngine<T>(config: UploadEngineConfig<T>): {
|
|
61
|
+
execute: (opts: UploadExecuteOptions) => Promise<UploadResult>;
|
|
62
|
+
};
|
|
63
|
+
//# sourceMappingURL=upload-engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upload-engine.d.ts","sourceRoot":"","sources":["../../src/commands/upload-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAM1D,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,qCAAqC;IACrC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IACpB,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,UAAU,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB;IACnC,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,mCAAmC;IACnC,KAAK,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAC/B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACnD;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAcD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;oBAGpC,oBAAoB,KAAG,OAAO,CAAC,YAAY,CAAC;EAkG1E"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic upload engine — sends queued records to a Pew SaaS endpoint.
|
|
3
|
+
*
|
|
4
|
+
* This is the shared pipeline used by both token and session upload commands.
|
|
5
|
+
* It handles:
|
|
6
|
+
* - Config/API-key loading
|
|
7
|
+
* - Queue offset management
|
|
8
|
+
* - Preprocessing (aggregation for tokens, dedup for sessions)
|
|
9
|
+
* - Batching (≤50 records per request, D1 free plan limit)
|
|
10
|
+
* - Retry with exponential backoff on 5xx
|
|
11
|
+
* - 429 rate-limit handling with Retry-After (no double-sleep)
|
|
12
|
+
* - Progress callbacks
|
|
13
|
+
*
|
|
14
|
+
* Bug fixes over the original upload/session-upload:
|
|
15
|
+
* - 429 handler no longer double-sleeps (Retry-After + exponential backoff).
|
|
16
|
+
* Now the Retry-After sleep replaces the exponential backoff for that attempt.
|
|
17
|
+
*/
|
|
18
|
+
import { ConfigManager } from "../config/manager.js";
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Constants
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
const DEFAULT_BATCH_SIZE = 50;
|
|
23
|
+
const DEFAULT_MAX_RETRIES = 2;
|
|
24
|
+
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Engine factory
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
export function createUploadEngine(config) {
|
|
29
|
+
const { queue, endpoint, entityName, preprocess } = config;
|
|
30
|
+
async function execute(opts) {
|
|
31
|
+
const { stateDir, apiUrl, dev = false, fetch: fetchFn, batchSize = DEFAULT_BATCH_SIZE, maxRetries = DEFAULT_MAX_RETRIES, retryDelayMs = DEFAULT_RETRY_DELAY_MS, onProgress, } = opts;
|
|
32
|
+
// 1. Load API key
|
|
33
|
+
const configManager = new ConfigManager(stateDir, dev);
|
|
34
|
+
const cfg = await configManager.load();
|
|
35
|
+
if (!cfg.token) {
|
|
36
|
+
return {
|
|
37
|
+
success: false,
|
|
38
|
+
uploaded: 0,
|
|
39
|
+
batches: 0,
|
|
40
|
+
error: "Not logged in. Run `pew login` first.",
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
// 2. Read un-uploaded records
|
|
44
|
+
const currentOffset = await queue.loadOffset();
|
|
45
|
+
const { records: rawRecords, newOffset } = await queue.readFromOffset(currentOffset);
|
|
46
|
+
if (rawRecords.length === 0) {
|
|
47
|
+
return { success: true, uploaded: 0, batches: 0 };
|
|
48
|
+
}
|
|
49
|
+
// 2b. Pre-process (aggregate for tokens, dedup for sessions)
|
|
50
|
+
const records = preprocess(rawRecords);
|
|
51
|
+
// 3. Split into batches
|
|
52
|
+
const batches = [];
|
|
53
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
54
|
+
batches.push(records.slice(i, i + batchSize));
|
|
55
|
+
}
|
|
56
|
+
// 4. Upload each batch
|
|
57
|
+
const fullEndpoint = `${apiUrl}${endpoint}`;
|
|
58
|
+
let totalUploaded = 0;
|
|
59
|
+
let batchesCompleted = 0;
|
|
60
|
+
for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
|
|
61
|
+
const batch = batches[batchIdx];
|
|
62
|
+
onProgress?.({
|
|
63
|
+
phase: "uploading",
|
|
64
|
+
batch: batchIdx + 1,
|
|
65
|
+
totalBatches: batches.length,
|
|
66
|
+
total: records.length,
|
|
67
|
+
message: `Uploading ${entityName} batch ${batchIdx + 1}/${batches.length} (${batch.length} records)...`,
|
|
68
|
+
});
|
|
69
|
+
const result = await sendBatchWithRetry({
|
|
70
|
+
endpoint: fullEndpoint,
|
|
71
|
+
token: cfg.token,
|
|
72
|
+
batch,
|
|
73
|
+
fetchFn,
|
|
74
|
+
maxRetries,
|
|
75
|
+
retryDelayMs,
|
|
76
|
+
});
|
|
77
|
+
if (!result.ok) {
|
|
78
|
+
return {
|
|
79
|
+
success: false,
|
|
80
|
+
uploaded: totalUploaded,
|
|
81
|
+
batches: batchesCompleted,
|
|
82
|
+
error: result.error,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
totalUploaded += batch.length;
|
|
86
|
+
batchesCompleted++;
|
|
87
|
+
}
|
|
88
|
+
// 5. All batches succeeded — save final offset
|
|
89
|
+
await queue.saveOffset(newOffset);
|
|
90
|
+
onProgress?.({
|
|
91
|
+
phase: "done",
|
|
92
|
+
total: totalUploaded,
|
|
93
|
+
message: `Uploaded ${totalUploaded} ${entityName} in ${batchesCompleted} batch(es).`,
|
|
94
|
+
});
|
|
95
|
+
return {
|
|
96
|
+
success: true,
|
|
97
|
+
uploaded: totalUploaded,
|
|
98
|
+
batches: batchesCompleted,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return { execute };
|
|
102
|
+
}
|
|
103
|
+
async function sendBatchWithRetry(opts) {
|
|
104
|
+
const { endpoint, token, batch, fetchFn, maxRetries, retryDelayMs } = opts;
|
|
105
|
+
let lastError = "";
|
|
106
|
+
let sleptFor429 = false;
|
|
107
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
108
|
+
// Exponential backoff — but skip if we already slept for a 429 Retry-After
|
|
109
|
+
if (attempt > 0 && !sleptFor429 && retryDelayMs > 0) {
|
|
110
|
+
await sleep(retryDelayMs * 2 ** (attempt - 1));
|
|
111
|
+
}
|
|
112
|
+
sleptFor429 = false; // Reset for next iteration
|
|
113
|
+
try {
|
|
114
|
+
const resp = await fetchFn(endpoint, {
|
|
115
|
+
method: "POST",
|
|
116
|
+
headers: {
|
|
117
|
+
"Content-Type": "application/json",
|
|
118
|
+
Authorization: `Bearer ${token}`,
|
|
119
|
+
},
|
|
120
|
+
body: JSON.stringify(batch),
|
|
121
|
+
});
|
|
122
|
+
if (resp.ok) {
|
|
123
|
+
return { ok: true };
|
|
124
|
+
}
|
|
125
|
+
// 429 — rate limited, retry with Retry-After if available
|
|
126
|
+
if (resp.status === 429) {
|
|
127
|
+
const retryAfter = resp.headers.get("Retry-After");
|
|
128
|
+
const retryMs = retryAfter
|
|
129
|
+
? Math.max(Number(retryAfter) * 1000, retryDelayMs)
|
|
130
|
+
: retryDelayMs * 2 ** attempt;
|
|
131
|
+
if (attempt < maxRetries && retryMs > 0) {
|
|
132
|
+
await sleep(retryMs);
|
|
133
|
+
sleptFor429 = true; // Skip the top-of-loop backoff on next iteration
|
|
134
|
+
}
|
|
135
|
+
const body = await resp.json().catch(() => ({}));
|
|
136
|
+
lastError = `429: ${body.error ?? "Too Many Requests"}`;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
// 4xx — client error, don't retry
|
|
140
|
+
if (resp.status >= 400 && resp.status < 500) {
|
|
141
|
+
const body = await resp.json().catch(() => ({}));
|
|
142
|
+
const msg = body.error ?? `HTTP ${resp.status}`;
|
|
143
|
+
return { ok: false, error: `${resp.status}: ${msg}` };
|
|
144
|
+
}
|
|
145
|
+
// 5xx — server error, retry
|
|
146
|
+
const body = await resp.json().catch(() => ({}));
|
|
147
|
+
lastError = `${resp.status}: ${body.error ?? "Server Error"}`;
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
lastError = String(err.message ?? err);
|
|
151
|
+
if (attempt >= maxRetries) {
|
|
152
|
+
return { ok: false, error: lastError };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
ok: false,
|
|
158
|
+
error: `Upload failed after ${maxRetries + 1} attempts: ${lastError}`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
function sleep(ms) {
|
|
162
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=upload-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upload-engine.js","sourceRoot":"","sources":["../../src/commands/upload-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAoDrD,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,UAAU,kBAAkB,CAAI,MAA6B;IACjE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAE3D,KAAK,UAAU,OAAO,CAAC,IAA0B;QAC/C,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,GAAG,GAAG,KAAK,EACX,KAAK,EAAE,OAAO,EACd,SAAS,GAAG,kBAAkB,EAC9B,UAAU,GAAG,mBAAmB,EAChC,YAAY,GAAG,sBAAsB,EACrC,UAAU,GACX,GAAG,IAAI,CAAC;QAET,kBAAkB;QAClB,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAEvC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,uCAAuC;aAC/C,CAAC;QACJ,CAAC;QAED,8BAA8B;QAC9B,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;QAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GACtC,MAAM,KAAK,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAE5C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,6DAA6D;QAC7D,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QAEvC,wBAAwB;QACxB,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,uBAAuB;QACvB,MAAM,YAAY,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC5C,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAEhC,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,KAAK,EAAE,QAAQ,GAAG,CAAC;gBACnB,YAAY,EAAE,OAAO,CAAC,MAAM;gBAC5B,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,OAAO,EAAE,aAAa,UAAU,UAAU,QAAQ,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,cAAc;aACxG,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC;gBACtC,QAAQ,EAAE,YAAY;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,KAAK;gBACL,OAAO;gBACP,UAAU;gBACV,YAAY;aACb,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,aAAa;oBACvB,OAAO,EAAE,gBAAgB;oBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB,CAAC;YACJ,CAAC;YAED,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;YAC9B,gBAAgB,EAAE,CAAC;QACrB,CAAC;QAED,+CAA+C;QAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAElC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE,YAAY,aAAa,IAAI,UAAU,OAAO,gBAAgB,aAAa;SACrF,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,gBAAgB;SAC1B,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,CAAC;AACrB,CAAC;AAWD,KAAK,UAAU,kBAAkB,CAAI,IAOpC;IACC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAE3E,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,2EAA2E;QAC3E,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,KAAK,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,WAAW,GAAG,KAAK,CAAC,CAAC,2BAA2B;QAEhD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE;gBACnC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,KAAK,EAAE;iBACjC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;gBACZ,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YACtB,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACxB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBACnD,MAAM,OAAO,GAAG,UAAU;oBACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,EAAE,YAAY,CAAC;oBACnD,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,OAAO,CAAC;gBAChC,IAAI,OAAO,GAAG,UAAU,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBACxC,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,WAAW,GAAG,IAAI,CAAC,CAAC,iDAAiD;gBACvE,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACjD,SAAS,GAAG,QAAS,IAA+B,CAAC,KAAK,IAAI,mBAAmB,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YAED,kCAAkC;YAClC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACjD,MAAM,GAAG,GACN,IAA+B,CAAC,KAAK,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,EAAE,CAAC;YACxD,CAAC;YAED,4BAA4B;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjD,SAAS,GAAG,GAAG,IAAI,CAAC,MAAM,KAAM,IAA+B,CAAC,KAAK,IAAI,cAAc,EAAE,CAAC;QAC5F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,MAAM,CAAE,GAAa,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;YAElD,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBAC1B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,uBAAuB,UAAU,GAAG,CAAC,cAAc,SAAS,EAAE;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -1,14 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* CLI upload command — sends local queue records to the Pew SaaS.
|
|
2
|
+
* CLI upload command — sends local token queue records to the Pew SaaS.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 2. Read un-uploaded records from queue (using saved offset)
|
|
7
|
-
* 3. Split into batches of ≤50 (D1 Free plan: 50 queries/Worker invocation)
|
|
8
|
-
* 4. POST each batch to /api/ingest with Bearer token
|
|
9
|
-
* 5. Persist offset after each successful batch (for resume on failure)
|
|
10
|
-
* 6. Retry on 5xx with exponential backoff
|
|
4
|
+
* Thin wrapper around the generic upload engine with token-specific
|
|
5
|
+
* preprocessing (aggregation by source/model/hour_start).
|
|
11
6
|
*/
|
|
7
|
+
import type { UploadResult, UploadProgressEvent } from "./upload-engine.js";
|
|
12
8
|
import type { QueueRecord } from "@pew/core";
|
|
13
9
|
export interface UploadOptions {
|
|
14
10
|
/** Directory for config file and queue state */
|
|
@@ -28,19 +24,7 @@ export interface UploadOptions {
|
|
|
28
24
|
/** Progress callback */
|
|
29
25
|
onProgress?: (event: UploadProgressEvent) => void;
|
|
30
26
|
}
|
|
31
|
-
export
|
|
32
|
-
phase: "uploading" | "done";
|
|
33
|
-
batch?: number;
|
|
34
|
-
totalBatches?: number;
|
|
35
|
-
total?: number;
|
|
36
|
-
message?: string;
|
|
37
|
-
}
|
|
38
|
-
export interface UploadResult {
|
|
39
|
-
success: boolean;
|
|
40
|
-
uploaded: number;
|
|
41
|
-
batches: number;
|
|
42
|
-
error?: string;
|
|
43
|
-
}
|
|
27
|
+
export type { UploadResult, UploadProgressEvent };
|
|
44
28
|
/**
|
|
45
29
|
* Aggregate QueueRecords by (source, model, hour_start), summing token fields.
|
|
46
30
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/commands/upload.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/commands/upload.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EAEpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAM7C,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,mCAAmC;IACnC,KAAK,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAC/B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACnD;AAED,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,CAAC;AAMlD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,WAAW,EAAE,CAoBtE;AAMD,wBAAsB,aAAa,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAW9E"}
|
package/dist/commands/upload.js
CHANGED
|
@@ -1,22 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* CLI upload command — sends local queue records to the Pew SaaS.
|
|
2
|
+
* CLI upload command — sends local token queue records to the Pew SaaS.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 2. Read un-uploaded records from queue (using saved offset)
|
|
7
|
-
* 3. Split into batches of ≤50 (D1 Free plan: 50 queries/Worker invocation)
|
|
8
|
-
* 4. POST each batch to /api/ingest with Bearer token
|
|
9
|
-
* 5. Persist offset after each successful batch (for resume on failure)
|
|
10
|
-
* 6. Retry on 5xx with exponential backoff
|
|
4
|
+
* Thin wrapper around the generic upload engine with token-specific
|
|
5
|
+
* preprocessing (aggregation by source/model/hour_start).
|
|
11
6
|
*/
|
|
12
|
-
import { ConfigManager } from "../config/manager.js";
|
|
13
7
|
import { LocalQueue } from "../storage/local-queue.js";
|
|
14
|
-
|
|
15
|
-
// Constants
|
|
16
|
-
// ---------------------------------------------------------------------------
|
|
17
|
-
const DEFAULT_BATCH_SIZE = 50;
|
|
18
|
-
const DEFAULT_MAX_RETRIES = 2;
|
|
19
|
-
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
8
|
+
import { createUploadEngine } from "./upload-engine.js";
|
|
20
9
|
// ---------------------------------------------------------------------------
|
|
21
10
|
// Pre-aggregation — merge QueueRecords with the same (source, model, hour_start)
|
|
22
11
|
// ---------------------------------------------------------------------------
|
|
@@ -51,135 +40,13 @@ export function aggregateRecords(records) {
|
|
|
51
40
|
// Implementation
|
|
52
41
|
// ---------------------------------------------------------------------------
|
|
53
42
|
export async function executeUpload(opts) {
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
success: false,
|
|
61
|
-
uploaded: 0,
|
|
62
|
-
batches: 0,
|
|
63
|
-
error: "Not logged in. Run `pew login` first.",
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
// 2. Read un-uploaded records
|
|
67
|
-
const queue = new LocalQueue(stateDir);
|
|
68
|
-
const currentOffset = await queue.loadOffset();
|
|
69
|
-
const { records: rawRecords, newOffset } = await queue.readFromOffset(currentOffset);
|
|
70
|
-
if (rawRecords.length === 0) {
|
|
71
|
-
return { success: true, uploaded: 0, batches: 0 };
|
|
72
|
-
}
|
|
73
|
-
// 2b. Pre-aggregate by (source, model, hour_start) for idempotent upsert
|
|
74
|
-
const records = aggregateRecords(rawRecords);
|
|
75
|
-
// 3. Split into batches
|
|
76
|
-
const batches = [];
|
|
77
|
-
for (let i = 0; i < records.length; i += batchSize) {
|
|
78
|
-
batches.push(records.slice(i, i + batchSize));
|
|
79
|
-
}
|
|
80
|
-
// 4. Upload each batch
|
|
81
|
-
const endpoint = `${apiUrl}/api/ingest`;
|
|
82
|
-
let totalUploaded = 0;
|
|
83
|
-
let batchesCompleted = 0;
|
|
84
|
-
for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
|
|
85
|
-
const batch = batches[batchIdx];
|
|
86
|
-
onProgress?.({
|
|
87
|
-
phase: "uploading",
|
|
88
|
-
batch: batchIdx + 1,
|
|
89
|
-
totalBatches: batches.length,
|
|
90
|
-
total: records.length,
|
|
91
|
-
message: `Uploading batch ${batchIdx + 1}/${batches.length} (${batch.length} records)...`,
|
|
92
|
-
});
|
|
93
|
-
const result = await sendBatchWithRetry({
|
|
94
|
-
endpoint,
|
|
95
|
-
token: config.token,
|
|
96
|
-
batch,
|
|
97
|
-
fetchFn,
|
|
98
|
-
maxRetries,
|
|
99
|
-
retryDelayMs,
|
|
100
|
-
});
|
|
101
|
-
if (!result.ok) {
|
|
102
|
-
// With idempotent overwrite upsert, don't save partial offset.
|
|
103
|
-
// Next retry will re-aggregate and re-send everything safely.
|
|
104
|
-
return {
|
|
105
|
-
success: false,
|
|
106
|
-
uploaded: totalUploaded,
|
|
107
|
-
batches: batchesCompleted,
|
|
108
|
-
error: result.error,
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
totalUploaded += batch.length;
|
|
112
|
-
batchesCompleted++;
|
|
113
|
-
}
|
|
114
|
-
// 5. All batches succeeded — save final offset
|
|
115
|
-
await queue.saveOffset(newOffset);
|
|
116
|
-
onProgress?.({
|
|
117
|
-
phase: "done",
|
|
118
|
-
total: totalUploaded,
|
|
119
|
-
message: `Uploaded ${totalUploaded} records in ${batchesCompleted} batch(es).`,
|
|
43
|
+
const queue = new LocalQueue(opts.stateDir);
|
|
44
|
+
const engine = createUploadEngine({
|
|
45
|
+
queue,
|
|
46
|
+
endpoint: "/api/ingest",
|
|
47
|
+
entityName: "records",
|
|
48
|
+
preprocess: aggregateRecords,
|
|
120
49
|
});
|
|
121
|
-
return
|
|
122
|
-
success: true,
|
|
123
|
-
uploaded: totalUploaded,
|
|
124
|
-
batches: batchesCompleted,
|
|
125
|
-
};
|
|
126
|
-
}
|
|
127
|
-
async function sendBatchWithRetry(opts) {
|
|
128
|
-
const { endpoint, token, batch, fetchFn, maxRetries, retryDelayMs } = opts;
|
|
129
|
-
let lastError = "";
|
|
130
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
131
|
-
if (attempt > 0 && retryDelayMs > 0) {
|
|
132
|
-
await sleep(retryDelayMs * 2 ** (attempt - 1));
|
|
133
|
-
}
|
|
134
|
-
try {
|
|
135
|
-
const resp = await fetchFn(endpoint, {
|
|
136
|
-
method: "POST",
|
|
137
|
-
headers: {
|
|
138
|
-
"Content-Type": "application/json",
|
|
139
|
-
Authorization: `Bearer ${token}`,
|
|
140
|
-
},
|
|
141
|
-
body: JSON.stringify(batch),
|
|
142
|
-
});
|
|
143
|
-
if (resp.ok) {
|
|
144
|
-
return { ok: true };
|
|
145
|
-
}
|
|
146
|
-
// 429 — rate limited, retry with Retry-After if available
|
|
147
|
-
if (resp.status === 429) {
|
|
148
|
-
const retryAfter = resp.headers.get("Retry-After");
|
|
149
|
-
const retryMs = retryAfter
|
|
150
|
-
? Math.max(Number(retryAfter) * 1000, retryDelayMs)
|
|
151
|
-
: retryDelayMs * 2 ** attempt;
|
|
152
|
-
if (attempt < maxRetries && retryMs > 0) {
|
|
153
|
-
await sleep(retryMs);
|
|
154
|
-
}
|
|
155
|
-
const body = await resp.json().catch(() => ({}));
|
|
156
|
-
lastError = `429: ${body.error ?? "Too Many Requests"}`;
|
|
157
|
-
continue;
|
|
158
|
-
}
|
|
159
|
-
// 4xx — client error, don't retry
|
|
160
|
-
if (resp.status >= 400 && resp.status < 500) {
|
|
161
|
-
const body = await resp.json().catch(() => ({}));
|
|
162
|
-
const msg = body.error ?? `HTTP ${resp.status}`;
|
|
163
|
-
return { ok: false, error: `${resp.status}: ${msg}` };
|
|
164
|
-
}
|
|
165
|
-
// 5xx — server error, retry
|
|
166
|
-
const body = await resp.json().catch(() => ({}));
|
|
167
|
-
lastError = `${resp.status}: ${body.error ?? "Server Error"}`;
|
|
168
|
-
}
|
|
169
|
-
catch (err) {
|
|
170
|
-
lastError = String(err.message ?? err);
|
|
171
|
-
// Network errors — don't retry if maxRetries is 0
|
|
172
|
-
if (attempt >= maxRetries) {
|
|
173
|
-
return { ok: false, error: lastError };
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
return {
|
|
178
|
-
ok: false,
|
|
179
|
-
error: `Upload failed after ${maxRetries + 1} attempts: ${lastError}`,
|
|
180
|
-
};
|
|
181
|
-
}
|
|
182
|
-
function sleep(ms) {
|
|
183
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
50
|
+
return engine.execute(opts);
|
|
184
51
|
}
|
|
185
52
|
//# sourceMappingURL=upload.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.js","sourceRoot":"","sources":["../../src/commands/upload.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"upload.js","sourceRoot":"","sources":["../../src/commands/upload.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAiCxD,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE3C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,CAAC;YACxC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,CAAC,mBAAmB,CAAC;YACtD,QAAQ,CAAC,aAAa,IAAI,CAAC,CAAC,aAAa,CAAC;YAC1C,QAAQ,CAAC,uBAAuB,IAAI,CAAC,CAAC,uBAAuB,CAAC;YAC9D,QAAQ,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAmB;IACrD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE5C,MAAM,MAAM,GAAG,kBAAkB,CAAc;QAC7C,KAAK;QACL,QAAQ,EAAE,aAAa;QACvB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,gBAAgB;KAC7B,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic append-only JSONL queue with byte-offset tracking.
|
|
3
|
+
*
|
|
4
|
+
* Records are stored as newline-delimited JSON. A separate state file
|
|
5
|
+
* tracks how far the upload cursor has progressed (in bytes).
|
|
6
|
+
*
|
|
7
|
+
* Bug fixes over the original LocalQueue/SessionQueue:
|
|
8
|
+
* - readFromOffset uses Buffer slicing (byte offset) instead of String.slice
|
|
9
|
+
* (character offset), which broke on non-ASCII content.
|
|
10
|
+
* - Per-line JSON.parse error handling: corrupted lines are skipped instead
|
|
11
|
+
* of blocking all subsequent records.
|
|
12
|
+
*/
|
|
13
|
+
export declare class BaseQueue<T> {
|
|
14
|
+
readonly queuePath: string;
|
|
15
|
+
private readonly statePath;
|
|
16
|
+
private readonly dir;
|
|
17
|
+
constructor(storeDir: string, queueFile: string, stateFile: string);
|
|
18
|
+
/** Ensure the directory exists */
|
|
19
|
+
private ensureDir;
|
|
20
|
+
/** Append a single record to the queue */
|
|
21
|
+
append(record: T): Promise<void>;
|
|
22
|
+
/** Append multiple records to the queue in a single write */
|
|
23
|
+
appendBatch(records: T[]): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Read records from the queue starting at a byte offset.
|
|
26
|
+
* Returns parsed records and the new offset (end of file in bytes).
|
|
27
|
+
*
|
|
28
|
+
* - Uses Buffer for slicing to correctly handle multi-byte UTF-8 characters.
|
|
29
|
+
* - Skips lines that fail JSON.parse instead of throwing.
|
|
30
|
+
*/
|
|
31
|
+
readFromOffset(offset: number): Promise<{
|
|
32
|
+
records: T[];
|
|
33
|
+
newOffset: number;
|
|
34
|
+
}>;
|
|
35
|
+
/** Save the upload byte offset to the state file */
|
|
36
|
+
saveOffset(offset: number): Promise<void>;
|
|
37
|
+
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
38
|
+
loadOffset(): Promise<number>;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=base-queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base-queue.d.ts","sourceRoot":"","sources":["../../src/storage/base-queue.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AACH,qBAAa,SAAS,CAAC,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;gBAEjB,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAMlE,kCAAkC;YACpB,SAAS;IAIvB,0CAA0C;IACpC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtC,6DAA6D;IACvD,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9C;;;;;;OAMG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAC5C,OAAO,EAAE,CAAC,EAAE,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IA0BF,oDAAoD;IAC9C,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,wEAAwE;IAClE,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;CASpC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { readFile, writeFile, appendFile, mkdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Generic append-only JSONL queue with byte-offset tracking.
|
|
5
|
+
*
|
|
6
|
+
* Records are stored as newline-delimited JSON. A separate state file
|
|
7
|
+
* tracks how far the upload cursor has progressed (in bytes).
|
|
8
|
+
*
|
|
9
|
+
* Bug fixes over the original LocalQueue/SessionQueue:
|
|
10
|
+
* - readFromOffset uses Buffer slicing (byte offset) instead of String.slice
|
|
11
|
+
* (character offset), which broke on non-ASCII content.
|
|
12
|
+
* - Per-line JSON.parse error handling: corrupted lines are skipped instead
|
|
13
|
+
* of blocking all subsequent records.
|
|
14
|
+
*/
|
|
15
|
+
export class BaseQueue {
|
|
16
|
+
queuePath;
|
|
17
|
+
statePath;
|
|
18
|
+
dir;
|
|
19
|
+
constructor(storeDir, queueFile, stateFile) {
|
|
20
|
+
this.dir = storeDir;
|
|
21
|
+
this.queuePath = join(storeDir, queueFile);
|
|
22
|
+
this.statePath = join(storeDir, stateFile);
|
|
23
|
+
}
|
|
24
|
+
/** Ensure the directory exists */
|
|
25
|
+
async ensureDir() {
|
|
26
|
+
await mkdir(this.dir, { recursive: true });
|
|
27
|
+
}
|
|
28
|
+
/** Append a single record to the queue */
|
|
29
|
+
async append(record) {
|
|
30
|
+
await this.ensureDir();
|
|
31
|
+
await appendFile(this.queuePath, JSON.stringify(record) + "\n");
|
|
32
|
+
}
|
|
33
|
+
/** Append multiple records to the queue in a single write */
|
|
34
|
+
async appendBatch(records) {
|
|
35
|
+
if (records.length === 0)
|
|
36
|
+
return;
|
|
37
|
+
await this.ensureDir();
|
|
38
|
+
const data = records.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
39
|
+
await appendFile(this.queuePath, data);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Read records from the queue starting at a byte offset.
|
|
43
|
+
* Returns parsed records and the new offset (end of file in bytes).
|
|
44
|
+
*
|
|
45
|
+
* - Uses Buffer for slicing to correctly handle multi-byte UTF-8 characters.
|
|
46
|
+
* - Skips lines that fail JSON.parse instead of throwing.
|
|
47
|
+
*/
|
|
48
|
+
async readFromOffset(offset) {
|
|
49
|
+
let buf;
|
|
50
|
+
try {
|
|
51
|
+
buf = await readFile(this.queuePath);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return { records: [], newOffset: 0 };
|
|
55
|
+
}
|
|
56
|
+
const slice = buf.subarray(offset);
|
|
57
|
+
const text = slice.toString("utf-8");
|
|
58
|
+
const lines = text.split("\n").filter((line) => line.trim().length > 0);
|
|
59
|
+
const records = [];
|
|
60
|
+
for (const line of lines) {
|
|
61
|
+
try {
|
|
62
|
+
records.push(JSON.parse(line));
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// Skip corrupted lines — log would be nice but we keep it silent
|
|
66
|
+
// to avoid coupling to a logger. The line is simply lost.
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const newOffset = buf.byteLength;
|
|
70
|
+
return { records, newOffset };
|
|
71
|
+
}
|
|
72
|
+
/** Save the upload byte offset to the state file */
|
|
73
|
+
async saveOffset(offset) {
|
|
74
|
+
await this.ensureDir();
|
|
75
|
+
await writeFile(this.statePath, JSON.stringify({ offset }) + "\n");
|
|
76
|
+
}
|
|
77
|
+
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
78
|
+
async loadOffset() {
|
|
79
|
+
try {
|
|
80
|
+
const raw = await readFile(this.statePath, "utf-8");
|
|
81
|
+
const state = JSON.parse(raw);
|
|
82
|
+
return state.offset ?? 0;
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return 0;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=base-queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base-queue.js","sourceRoot":"","sources":["../../src/storage/base-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,SAAS;IACX,SAAS,CAAS;IACV,SAAS,CAAS;IAClB,GAAG,CAAS;IAE7B,YAAY,QAAgB,EAAE,SAAiB,EAAE,SAAiB;QAChE,IAAI,CAAC,GAAG,GAAG,QAAQ,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC7C,CAAC;IAED,kCAAkC;IAC1B,KAAK,CAAC,SAAS;QACrB,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,0CAA0C;IAC1C,KAAK,CAAC,MAAM,CAAC,MAAS;QACpB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,WAAW,CAAC,OAAY;QAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACrE,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc;QAIjC,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAExE,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,iEAAiE;gBACjE,0DAA0D;YAC5D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;QACjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAChC,CAAC;IAED,oDAAoD;IACpD,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,UAAU;QACd,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAuB,CAAC;YACpD,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,30 +1,10 @@
|
|
|
1
|
+
import { BaseQueue } from "./base-queue.js";
|
|
1
2
|
import type { QueueRecord } from "@pew/core";
|
|
2
3
|
/**
|
|
3
|
-
* Append-only local queue for usage records.
|
|
4
|
-
*
|
|
4
|
+
* Append-only local queue for token usage records.
|
|
5
|
+
* Thin wrapper around BaseQueue with token-specific file names.
|
|
5
6
|
*/
|
|
6
|
-
export declare class LocalQueue {
|
|
7
|
-
readonly queuePath: string;
|
|
8
|
-
private readonly statePath;
|
|
9
|
-
private readonly dir;
|
|
7
|
+
export declare class LocalQueue extends BaseQueue<QueueRecord> {
|
|
10
8
|
constructor(storeDir: string);
|
|
11
|
-
/** Ensure the directory exists */
|
|
12
|
-
private ensureDir;
|
|
13
|
-
/** Append a single record to the queue */
|
|
14
|
-
append(record: QueueRecord): Promise<void>;
|
|
15
|
-
/** Append multiple records to the queue in a single write */
|
|
16
|
-
appendBatch(records: QueueRecord[]): Promise<void>;
|
|
17
|
-
/**
|
|
18
|
-
* Read records from the queue starting at a byte offset.
|
|
19
|
-
* Returns parsed records and the new offset (end of file).
|
|
20
|
-
*/
|
|
21
|
-
readFromOffset(offset: number): Promise<{
|
|
22
|
-
records: QueueRecord[];
|
|
23
|
-
newOffset: number;
|
|
24
|
-
}>;
|
|
25
|
-
/** Save the upload byte offset to the state file */
|
|
26
|
-
saveOffset(offset: number): Promise<void>;
|
|
27
|
-
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
28
|
-
loadOffset(): Promise<number>;
|
|
29
9
|
}
|
|
30
10
|
//# sourceMappingURL=local-queue.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-queue.d.ts","sourceRoot":"","sources":["../../src/storage/local-queue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"local-queue.d.ts","sourceRoot":"","sources":["../../src/storage/local-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C;;;GAGG;AACH,qBAAa,UAAW,SAAQ,SAAS,CAAC,WAAW,CAAC;gBACxC,QAAQ,EAAE,MAAM;CAG7B"}
|
|
@@ -1,70 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
const QUEUE_FILE = "queue.jsonl";
|
|
4
|
-
const STATE_FILE = "queue.state.json";
|
|
1
|
+
import { BaseQueue } from "./base-queue.js";
|
|
5
2
|
/**
|
|
6
|
-
* Append-only local queue for usage records.
|
|
7
|
-
*
|
|
3
|
+
* Append-only local queue for token usage records.
|
|
4
|
+
* Thin wrapper around BaseQueue with token-specific file names.
|
|
8
5
|
*/
|
|
9
|
-
export class LocalQueue {
|
|
10
|
-
queuePath;
|
|
11
|
-
statePath;
|
|
12
|
-
dir;
|
|
6
|
+
export class LocalQueue extends BaseQueue {
|
|
13
7
|
constructor(storeDir) {
|
|
14
|
-
|
|
15
|
-
this.queuePath = join(storeDir, QUEUE_FILE);
|
|
16
|
-
this.statePath = join(storeDir, STATE_FILE);
|
|
17
|
-
}
|
|
18
|
-
/** Ensure the directory exists */
|
|
19
|
-
async ensureDir() {
|
|
20
|
-
await mkdir(this.dir, { recursive: true });
|
|
21
|
-
}
|
|
22
|
-
/** Append a single record to the queue */
|
|
23
|
-
async append(record) {
|
|
24
|
-
await this.ensureDir();
|
|
25
|
-
await appendFile(this.queuePath, JSON.stringify(record) + "\n");
|
|
26
|
-
}
|
|
27
|
-
/** Append multiple records to the queue in a single write */
|
|
28
|
-
async appendBatch(records) {
|
|
29
|
-
if (records.length === 0)
|
|
30
|
-
return;
|
|
31
|
-
await this.ensureDir();
|
|
32
|
-
const data = records.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
33
|
-
await appendFile(this.queuePath, data);
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Read records from the queue starting at a byte offset.
|
|
37
|
-
* Returns parsed records and the new offset (end of file).
|
|
38
|
-
*/
|
|
39
|
-
async readFromOffset(offset) {
|
|
40
|
-
let raw;
|
|
41
|
-
try {
|
|
42
|
-
raw = await readFile(this.queuePath, "utf-8");
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
45
|
-
return { records: [], newOffset: 0 };
|
|
46
|
-
}
|
|
47
|
-
const slice = raw.slice(offset);
|
|
48
|
-
const lines = slice.split("\n").filter((line) => line.trim().length > 0);
|
|
49
|
-
const records = lines.map((line) => JSON.parse(line));
|
|
50
|
-
const newOffset = Buffer.byteLength(raw, "utf-8");
|
|
51
|
-
return { records, newOffset };
|
|
52
|
-
}
|
|
53
|
-
/** Save the upload byte offset to the state file */
|
|
54
|
-
async saveOffset(offset) {
|
|
55
|
-
await this.ensureDir();
|
|
56
|
-
await writeFile(this.statePath, JSON.stringify({ offset }) + "\n");
|
|
57
|
-
}
|
|
58
|
-
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
59
|
-
async loadOffset() {
|
|
60
|
-
try {
|
|
61
|
-
const raw = await readFile(this.statePath, "utf-8");
|
|
62
|
-
const state = JSON.parse(raw);
|
|
63
|
-
return state.offset ?? 0;
|
|
64
|
-
}
|
|
65
|
-
catch {
|
|
66
|
-
return 0;
|
|
67
|
-
}
|
|
8
|
+
super(storeDir, "queue.jsonl", "queue.state.json");
|
|
68
9
|
}
|
|
69
10
|
}
|
|
70
11
|
//# sourceMappingURL=local-queue.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-queue.js","sourceRoot":"","sources":["../../src/storage/local-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"local-queue.js","sourceRoot":"","sources":["../../src/storage/local-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;GAGG;AACH,MAAM,OAAO,UAAW,SAAQ,SAAsB;IACpD,YAAY,QAAgB;QAC1B,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,kBAAkB,CAAC,CAAC;IACrD,CAAC;CACF"}
|
|
@@ -1,28 +1,10 @@
|
|
|
1
|
+
import { BaseQueue } from "./base-queue.js";
|
|
1
2
|
import type { SessionQueueRecord } from "@pew/core";
|
|
2
3
|
/**
|
|
3
4
|
* Append-only local queue for session records.
|
|
4
|
-
*
|
|
5
|
+
* Thin wrapper around BaseQueue with session-specific file names.
|
|
5
6
|
*/
|
|
6
|
-
export declare class SessionQueue {
|
|
7
|
-
readonly queuePath: string;
|
|
8
|
-
private readonly statePath;
|
|
9
|
-
private readonly dir;
|
|
7
|
+
export declare class SessionQueue extends BaseQueue<SessionQueueRecord> {
|
|
10
8
|
constructor(storeDir: string);
|
|
11
|
-
/** Ensure the directory exists */
|
|
12
|
-
private ensureDir;
|
|
13
|
-
/** Append multiple records to the queue in a single write */
|
|
14
|
-
appendBatch(records: SessionQueueRecord[]): Promise<void>;
|
|
15
|
-
/**
|
|
16
|
-
* Read records from the queue starting at a byte offset.
|
|
17
|
-
* Returns parsed records and the new offset (end of file).
|
|
18
|
-
*/
|
|
19
|
-
readFromOffset(offset: number): Promise<{
|
|
20
|
-
records: SessionQueueRecord[];
|
|
21
|
-
newOffset: number;
|
|
22
|
-
}>;
|
|
23
|
-
/** Save the upload byte offset to the state file */
|
|
24
|
-
saveOffset(offset: number): Promise<void>;
|
|
25
|
-
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
26
|
-
loadOffset(): Promise<number>;
|
|
27
9
|
}
|
|
28
10
|
//# sourceMappingURL=session-queue.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-queue.d.ts","sourceRoot":"","sources":["../../src/storage/session-queue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-queue.d.ts","sourceRoot":"","sources":["../../src/storage/session-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAEpD;;;GAGG;AACH,qBAAa,YAAa,SAAQ,SAAS,CAAC,kBAAkB,CAAC;gBACjD,QAAQ,EAAE,MAAM;CAG7B"}
|
|
@@ -1,65 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
const QUEUE_FILE = "session-queue.jsonl";
|
|
4
|
-
const STATE_FILE = "session-queue.state.json";
|
|
1
|
+
import { BaseQueue } from "./base-queue.js";
|
|
5
2
|
/**
|
|
6
3
|
* Append-only local queue for session records.
|
|
7
|
-
*
|
|
4
|
+
* Thin wrapper around BaseQueue with session-specific file names.
|
|
8
5
|
*/
|
|
9
|
-
export class SessionQueue {
|
|
10
|
-
queuePath;
|
|
11
|
-
statePath;
|
|
12
|
-
dir;
|
|
6
|
+
export class SessionQueue extends BaseQueue {
|
|
13
7
|
constructor(storeDir) {
|
|
14
|
-
|
|
15
|
-
this.queuePath = join(storeDir, QUEUE_FILE);
|
|
16
|
-
this.statePath = join(storeDir, STATE_FILE);
|
|
17
|
-
}
|
|
18
|
-
/** Ensure the directory exists */
|
|
19
|
-
async ensureDir() {
|
|
20
|
-
await mkdir(this.dir, { recursive: true });
|
|
21
|
-
}
|
|
22
|
-
/** Append multiple records to the queue in a single write */
|
|
23
|
-
async appendBatch(records) {
|
|
24
|
-
if (records.length === 0)
|
|
25
|
-
return;
|
|
26
|
-
await this.ensureDir();
|
|
27
|
-
const data = records.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
28
|
-
await appendFile(this.queuePath, data);
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Read records from the queue starting at a byte offset.
|
|
32
|
-
* Returns parsed records and the new offset (end of file).
|
|
33
|
-
*/
|
|
34
|
-
async readFromOffset(offset) {
|
|
35
|
-
let raw;
|
|
36
|
-
try {
|
|
37
|
-
raw = await readFile(this.queuePath, "utf-8");
|
|
38
|
-
}
|
|
39
|
-
catch {
|
|
40
|
-
return { records: [], newOffset: 0 };
|
|
41
|
-
}
|
|
42
|
-
const slice = raw.slice(offset);
|
|
43
|
-
const lines = slice.split("\n").filter((line) => line.trim().length > 0);
|
|
44
|
-
const records = lines.map((line) => JSON.parse(line));
|
|
45
|
-
const newOffset = Buffer.byteLength(raw, "utf-8");
|
|
46
|
-
return { records, newOffset };
|
|
47
|
-
}
|
|
48
|
-
/** Save the upload byte offset to the state file */
|
|
49
|
-
async saveOffset(offset) {
|
|
50
|
-
await this.ensureDir();
|
|
51
|
-
await writeFile(this.statePath, JSON.stringify({ offset }) + "\n");
|
|
52
|
-
}
|
|
53
|
-
/** Load the upload byte offset. Returns 0 if not found or corrupted. */
|
|
54
|
-
async loadOffset() {
|
|
55
|
-
try {
|
|
56
|
-
const raw = await readFile(this.statePath, "utf-8");
|
|
57
|
-
const state = JSON.parse(raw);
|
|
58
|
-
return state.offset ?? 0;
|
|
59
|
-
}
|
|
60
|
-
catch {
|
|
61
|
-
return 0;
|
|
62
|
-
}
|
|
8
|
+
super(storeDir, "session-queue.jsonl", "session-queue.state.json");
|
|
63
9
|
}
|
|
64
10
|
}
|
|
65
11
|
//# sourceMappingURL=session-queue.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-queue.js","sourceRoot":"","sources":["../../src/storage/session-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"session-queue.js","sourceRoot":"","sources":["../../src/storage/session-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C;;;GAGG;AACH,MAAM,OAAO,YAAa,SAAQ,SAA6B;IAC7D,YAAY,QAAgB;QAC1B,KAAK,CAAC,QAAQ,EAAE,qBAAqB,EAAE,0BAA0B,CAAC,CAAC;IACrE,CAAC;CACF"}
|