@nocoo/pew 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +179 -1
- package/dist/cli.js.map +1 -1
- package/dist/commands/init.d.ts +38 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +73 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/notify.d.ts +11 -0
- package/dist/commands/notify.d.ts.map +1 -0
- package/dist/commands/notify.js +28 -0
- package/dist/commands/notify.js.map +1 -0
- package/dist/commands/session-upload.d.ts +6 -22
- package/dist/commands/session-upload.d.ts.map +1 -1
- package/dist/commands/session-upload.js +12 -144
- package/dist/commands/session-upload.js.map +1 -1
- package/dist/commands/status.d.ts +4 -0
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +1 -0
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/uninstall.d.ts +39 -0
- package/dist/commands/uninstall.d.ts.map +1 -0
- package/dist/commands/uninstall.js +107 -0
- package/dist/commands/uninstall.js.map +1 -0
- package/dist/commands/upload-engine.d.ts +63 -0
- package/dist/commands/upload-engine.d.ts.map +1 -0
- package/dist/commands/upload-engine.js +164 -0
- package/dist/commands/upload-engine.js.map +1 -0
- package/dist/commands/upload.d.ts +5 -21
- package/dist/commands/upload.d.ts.map +1 -1
- package/dist/commands/upload.js +11 -144
- package/dist/commands/upload.js.map +1 -1
- package/dist/notifier/claude-hook.d.ts +18 -0
- package/dist/notifier/claude-hook.d.ts.map +1 -0
- package/dist/notifier/claude-hook.js +226 -0
- package/dist/notifier/claude-hook.js.map +1 -0
- package/dist/notifier/codex-notifier.d.ts +19 -0
- package/dist/notifier/codex-notifier.d.ts.map +1 -0
- package/dist/notifier/codex-notifier.js +258 -0
- package/dist/notifier/codex-notifier.js.map +1 -0
- package/dist/notifier/coordinator.d.ts +29 -0
- package/dist/notifier/coordinator.d.ts.map +1 -0
- package/dist/notifier/coordinator.js +176 -0
- package/dist/notifier/coordinator.js.map +1 -0
- package/dist/notifier/gemini-hook.d.ts +18 -0
- package/dist/notifier/gemini-hook.d.ts.map +1 -0
- package/dist/notifier/gemini-hook.js +243 -0
- package/dist/notifier/gemini-hook.js.map +1 -0
- package/dist/notifier/notify-handler.d.ts +41 -0
- package/dist/notifier/notify-handler.d.ts.map +1 -0
- package/dist/notifier/notify-handler.js +172 -0
- package/dist/notifier/notify-handler.js.map +1 -0
- package/dist/notifier/openclaw-hook.d.ts +28 -0
- package/dist/notifier/openclaw-hook.d.ts.map +1 -0
- package/dist/notifier/openclaw-hook.js +272 -0
- package/dist/notifier/openclaw-hook.js.map +1 -0
- package/dist/notifier/opencode-plugin.d.ts +20 -0
- package/dist/notifier/opencode-plugin.d.ts.map +1 -0
- package/dist/notifier/opencode-plugin.js +106 -0
- package/dist/notifier/opencode-plugin.js.map +1 -0
- package/dist/notifier/paths.d.ts +21 -0
- package/dist/notifier/paths.d.ts.map +1 -0
- package/dist/notifier/paths.js +36 -0
- package/dist/notifier/paths.js.map +1 -0
- package/dist/notifier/registry.d.ts +21 -0
- package/dist/notifier/registry.d.ts.map +1 -0
- package/dist/notifier/registry.js +145 -0
- package/dist/notifier/registry.js.map +1 -0
- package/dist/storage/base-queue.d.ts +40 -0
- package/dist/storage/base-queue.d.ts.map +1 -0
- package/dist/storage/base-queue.js +89 -0
- package/dist/storage/base-queue.js.map +1 -0
- package/dist/storage/local-queue.d.ts +4 -24
- package/dist/storage/local-queue.d.ts.map +1 -1
- package/dist/storage/local-queue.js +5 -64
- package/dist/storage/local-queue.js.map +1 -1
- package/dist/storage/session-queue.d.ts +3 -21
- package/dist/storage/session-queue.d.ts.map +1 -1
- package/dist/storage/session-queue.js +4 -58
- package/dist/storage/session-queue.js.map +1 -1
- package/dist/utils/paths.d.ts +4 -0
- package/dist/utils/paths.d.ts.map +1 -1
- package/dist/utils/paths.js +4 -0
- package/dist/utils/paths.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,25 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* CLI session upload command — sends local session queue records to the Pew SaaS.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 2. Read un-uploaded session records from queue (using saved offset)
|
|
7
|
-
* 3. Deduplicate: keep only latest snapshot per session_key
|
|
8
|
-
* 4. Split into batches of ≤50 (D1 Free plan limit)
|
|
9
|
-
* 5. POST each batch to /api/ingest/sessions with Bearer token
|
|
10
|
-
* 6. Persist offset after all batches succeed
|
|
11
|
-
* 7. Retry on 5xx/429 with exponential backoff
|
|
4
|
+
* Thin wrapper around the generic upload engine with session-specific
|
|
5
|
+
* preprocessing (deduplication: keep only latest snapshot per session_key).
|
|
12
6
|
*/
|
|
13
|
-
import { ConfigManager } from "../config/manager.js";
|
|
14
7
|
import { SessionQueue } from "../storage/session-queue.js";
|
|
8
|
+
import { createUploadEngine } from "./upload-engine.js";
|
|
15
9
|
// ---------------------------------------------------------------------------
|
|
16
|
-
//
|
|
17
|
-
// ---------------------------------------------------------------------------
|
|
18
|
-
const DEFAULT_BATCH_SIZE = 50;
|
|
19
|
-
const DEFAULT_MAX_RETRIES = 2;
|
|
20
|
-
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
// Upload dedup
|
|
10
|
+
// Pre-dedup — keep only the latest snapshot per session_key
|
|
23
11
|
// ---------------------------------------------------------------------------
|
|
24
12
|
/**
|
|
25
13
|
* Unlike token's aggregateRecords() which SUMS, session dedup
|
|
@@ -45,133 +33,13 @@ export function deduplicateSessionRecords(records) {
|
|
|
45
33
|
// Implementation
|
|
46
34
|
// ---------------------------------------------------------------------------
|
|
47
35
|
export async function executeSessionUpload(opts) {
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
success: false,
|
|
55
|
-
uploaded: 0,
|
|
56
|
-
batches: 0,
|
|
57
|
-
error: "Not logged in. Run `pew login` first.",
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
// 2. Read un-uploaded records
|
|
61
|
-
const queue = new SessionQueue(stateDir);
|
|
62
|
-
const currentOffset = await queue.loadOffset();
|
|
63
|
-
const { records: rawRecords, newOffset } = await queue.readFromOffset(currentOffset);
|
|
64
|
-
if (rawRecords.length === 0) {
|
|
65
|
-
return { success: true, uploaded: 0, batches: 0 };
|
|
66
|
-
}
|
|
67
|
-
// 2b. Pre-deduplicate: keep only latest snapshot per session_key
|
|
68
|
-
const records = deduplicateSessionRecords(rawRecords);
|
|
69
|
-
// 3. Split into batches
|
|
70
|
-
const batches = [];
|
|
71
|
-
for (let i = 0; i < records.length; i += batchSize) {
|
|
72
|
-
batches.push(records.slice(i, i + batchSize));
|
|
73
|
-
}
|
|
74
|
-
// 4. Upload each batch
|
|
75
|
-
const endpoint = `${apiUrl}/api/ingest/sessions`;
|
|
76
|
-
let totalUploaded = 0;
|
|
77
|
-
let batchesCompleted = 0;
|
|
78
|
-
for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
|
|
79
|
-
const batch = batches[batchIdx];
|
|
80
|
-
onProgress?.({
|
|
81
|
-
phase: "uploading",
|
|
82
|
-
batch: batchIdx + 1,
|
|
83
|
-
totalBatches: batches.length,
|
|
84
|
-
total: records.length,
|
|
85
|
-
message: `Uploading session batch ${batchIdx + 1}/${batches.length} (${batch.length} records)...`,
|
|
86
|
-
});
|
|
87
|
-
const result = await sendBatchWithRetry({
|
|
88
|
-
endpoint,
|
|
89
|
-
token: config.token,
|
|
90
|
-
batch,
|
|
91
|
-
fetchFn,
|
|
92
|
-
maxRetries,
|
|
93
|
-
retryDelayMs,
|
|
94
|
-
});
|
|
95
|
-
if (!result.ok) {
|
|
96
|
-
return {
|
|
97
|
-
success: false,
|
|
98
|
-
uploaded: totalUploaded,
|
|
99
|
-
batches: batchesCompleted,
|
|
100
|
-
error: result.error,
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
totalUploaded += batch.length;
|
|
104
|
-
batchesCompleted++;
|
|
105
|
-
}
|
|
106
|
-
// 5. All batches succeeded — save final offset
|
|
107
|
-
await queue.saveOffset(newOffset);
|
|
108
|
-
onProgress?.({
|
|
109
|
-
phase: "done",
|
|
110
|
-
total: totalUploaded,
|
|
111
|
-
message: `Uploaded ${totalUploaded} session records in ${batchesCompleted} batch(es).`,
|
|
36
|
+
const queue = new SessionQueue(opts.stateDir);
|
|
37
|
+
const engine = createUploadEngine({
|
|
38
|
+
queue,
|
|
39
|
+
endpoint: "/api/ingest/sessions",
|
|
40
|
+
entityName: "session records",
|
|
41
|
+
preprocess: deduplicateSessionRecords,
|
|
112
42
|
});
|
|
113
|
-
return
|
|
114
|
-
success: true,
|
|
115
|
-
uploaded: totalUploaded,
|
|
116
|
-
batches: batchesCompleted,
|
|
117
|
-
};
|
|
118
|
-
}
|
|
119
|
-
async function sendBatchWithRetry(opts) {
|
|
120
|
-
const { endpoint, token, batch, fetchFn, maxRetries, retryDelayMs } = opts;
|
|
121
|
-
let lastError = "";
|
|
122
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
123
|
-
if (attempt > 0 && retryDelayMs > 0) {
|
|
124
|
-
await sleep(retryDelayMs * 2 ** (attempt - 1));
|
|
125
|
-
}
|
|
126
|
-
try {
|
|
127
|
-
const resp = await fetchFn(endpoint, {
|
|
128
|
-
method: "POST",
|
|
129
|
-
headers: {
|
|
130
|
-
"Content-Type": "application/json",
|
|
131
|
-
Authorization: `Bearer ${token}`,
|
|
132
|
-
},
|
|
133
|
-
body: JSON.stringify(batch),
|
|
134
|
-
});
|
|
135
|
-
if (resp.ok) {
|
|
136
|
-
return { ok: true };
|
|
137
|
-
}
|
|
138
|
-
// 429 — rate limited, retry with Retry-After if available
|
|
139
|
-
if (resp.status === 429) {
|
|
140
|
-
const retryAfter = resp.headers.get("Retry-After");
|
|
141
|
-
const retryMs = retryAfter
|
|
142
|
-
? Math.max(Number(retryAfter) * 1000, retryDelayMs)
|
|
143
|
-
: retryDelayMs * 2 ** attempt;
|
|
144
|
-
if (attempt < maxRetries && retryMs > 0) {
|
|
145
|
-
await sleep(retryMs);
|
|
146
|
-
}
|
|
147
|
-
const body = await resp.json().catch(() => ({}));
|
|
148
|
-
lastError = `429: ${body.error ?? "Too Many Requests"}`;
|
|
149
|
-
continue;
|
|
150
|
-
}
|
|
151
|
-
// 4xx — client error, don't retry
|
|
152
|
-
if (resp.status >= 400 && resp.status < 500) {
|
|
153
|
-
const body = await resp.json().catch(() => ({}));
|
|
154
|
-
const msg = body.error ?? `HTTP ${resp.status}`;
|
|
155
|
-
return { ok: false, error: `${resp.status}: ${msg}` };
|
|
156
|
-
}
|
|
157
|
-
// 5xx — server error, retry
|
|
158
|
-
const body = await resp.json().catch(() => ({}));
|
|
159
|
-
lastError = `${resp.status}: ${body.error ?? "Server Error"}`;
|
|
160
|
-
}
|
|
161
|
-
catch (err) {
|
|
162
|
-
lastError = String(err.message ?? err);
|
|
163
|
-
// Network errors — don't retry if maxRetries is 0
|
|
164
|
-
if (attempt >= maxRetries) {
|
|
165
|
-
return { ok: false, error: lastError };
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
return {
|
|
170
|
-
ok: false,
|
|
171
|
-
error: `Upload failed after ${maxRetries + 1} attempts: ${lastError}`,
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
function sleep(ms) {
|
|
175
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
43
|
+
return engine.execute(opts);
|
|
176
44
|
}
|
|
177
45
|
//# sourceMappingURL=session-upload.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-upload.js","sourceRoot":"","sources":["../../src/commands/session-upload.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"session-upload.js","sourceRoot":"","sources":["../../src/commands/session-upload.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAkCxD,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CACvC,OAA6B;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA8B,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;YACtD,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAA0B;IAE1B,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG,kBAAkB,CAAqB;QACpD,KAAK;QACL,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,iBAAiB;QAC7B,UAAU,EAAE,yBAAyB;KACtC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { NotifierStatus, Source } from "@pew/core";
|
|
1
2
|
/** Resolved source directory paths used for file classification */
|
|
2
3
|
export interface SourceDirs {
|
|
3
4
|
claudeDir: string;
|
|
@@ -16,6 +17,8 @@ export interface StatusResult {
|
|
|
16
17
|
pendingRecords: number;
|
|
17
18
|
/** Breakdown by source */
|
|
18
19
|
sources: Record<string, number>;
|
|
20
|
+
/** Notifier hook/plugin status by source */
|
|
21
|
+
notifiers: Partial<Record<Source, NotifierStatus>>;
|
|
19
22
|
}
|
|
20
23
|
/**
|
|
21
24
|
* Compute the current sync status.
|
|
@@ -24,5 +27,6 @@ export interface StatusResult {
|
|
|
24
27
|
export declare function executeStatus(opts: {
|
|
25
28
|
stateDir: string;
|
|
26
29
|
sourceDirs: SourceDirs;
|
|
30
|
+
notifierStatuses?: Partial<Record<Source, NotifierStatus>>;
|
|
27
31
|
}): Promise<StatusResult>;
|
|
28
32
|
//# sourceMappingURL=status.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAIxD,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,4CAA4C;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,4CAA4C;IAC5C,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;CACpD;AAiBD;;;GAGG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,UAAU,CAAC;IACvB,gBAAgB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;CAC5D,GAAG,OAAO,CAAC,YAAY,CAAC,CAwBxB"}
|
package/dist/commands/status.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"status.js","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"status.js","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAyBvD;;;;;GAKG;AACH,SAAS,cAAc,CAAC,QAAgB,EAAE,IAAgB;IACxD,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,OAAO,aAAa,CAAC;IAC9D,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAAE,OAAO,OAAO,CAAC;IAC/D,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,OAAO,YAAY,CAAC;IAC7D,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC;QAAE,OAAO,UAAU,CAAC;IACpE,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC;QAAE,OAAO,UAAU,CAAC;IAC7D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAInC;IACC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAEtC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEvC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;IACxC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAEvD,uDAAuD;IACvD,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACpD,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO;QACL,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM;QAC/C,QAAQ,EAAE,OAAO,CAAC,SAAS;QAC3B,cAAc,EAAE,OAAO,CAAC,MAAM;QAC9B,OAAO;QACP,SAAS,EAAE,IAAI,CAAC,gBAAgB,IAAI,EAAE;KACvC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { NotifierOperationResult, Source } from "@pew/core";
|
|
2
|
+
import { resolveNotifierPaths, type NotifierPaths } from "../notifier/paths.js";
|
|
3
|
+
import { getAllDrivers, uninstallAll } from "../notifier/registry.js";
|
|
4
|
+
interface ArtifactRemovalResult {
|
|
5
|
+
changed: boolean;
|
|
6
|
+
path: string;
|
|
7
|
+
detail: string;
|
|
8
|
+
warnings?: string[];
|
|
9
|
+
}
|
|
10
|
+
export interface UninstallOptions {
|
|
11
|
+
stateDir: string;
|
|
12
|
+
home: string;
|
|
13
|
+
env?: Record<string, string | undefined>;
|
|
14
|
+
dryRun?: boolean;
|
|
15
|
+
sources?: Source[];
|
|
16
|
+
resolveNotifierPathsFn?: typeof resolveNotifierPaths;
|
|
17
|
+
uninstallAllFn?: typeof uninstallAll;
|
|
18
|
+
uninstallDriverFn?: (source: Source, paths: NotifierPaths, deps?: {
|
|
19
|
+
spawn?: (cmd: string, args: string[], opts?: object) => {
|
|
20
|
+
status: number | null;
|
|
21
|
+
};
|
|
22
|
+
}) => Promise<NotifierOperationResult>;
|
|
23
|
+
getAllDriversFn?: typeof getAllDrivers;
|
|
24
|
+
spawn?: (cmd: string, args: string[], opts?: object) => {
|
|
25
|
+
status: number | null;
|
|
26
|
+
};
|
|
27
|
+
removeNotifyHandlerFn?: (opts: {
|
|
28
|
+
notifyPath: string;
|
|
29
|
+
}) => Promise<ArtifactRemovalResult>;
|
|
30
|
+
removeCodexBackupFn?: (path: string) => Promise<ArtifactRemovalResult>;
|
|
31
|
+
}
|
|
32
|
+
export interface UninstallResult {
|
|
33
|
+
notifyHandler: ArtifactRemovalResult;
|
|
34
|
+
codexBackup: ArtifactRemovalResult;
|
|
35
|
+
hooks: NotifierOperationResult[];
|
|
36
|
+
}
|
|
37
|
+
export declare function executeUninstall(opts: UninstallOptions): Promise<UninstallResult>;
|
|
38
|
+
export {};
|
|
39
|
+
//# sourceMappingURL=uninstall.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uninstall.d.ts","sourceRoot":"","sources":["../../src/commands/uninstall.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,KAAK,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAChF,OAAO,EACL,aAAa,EAEb,YAAY,EACb,MAAM,yBAAyB,CAAC;AAEjC,UAAU,qBAAqB;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,sBAAsB,CAAC,EAAE,OAAO,oBAAoB,CAAC;IACrD,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;IACrC,iBAAiB,CAAC,EAAE,CAClB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,EACpB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,KAAK;YAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;SAAE,CAAA;KAAE,KACzF,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACtC,eAAe,CAAC,EAAE,OAAO,aAAa,CAAC;IACvC,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,KAAK;QAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAClF,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE;QAC7B,UAAU,EAAE,MAAM,CAAC;KACpB,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACrC,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACxE;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,qBAAqB,CAAC;IACrC,WAAW,EAAE,qBAAqB,CAAC;IACnC,KAAK,EAAE,uBAAuB,EAAE,CAAC;CAClC;AAED,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAuFvF"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { unlink } from "node:fs/promises";
|
|
2
|
+
import { removeNotifyHandler } from "../notifier/notify-handler.js";
|
|
3
|
+
import { resolveNotifierPaths } from "../notifier/paths.js";
|
|
4
|
+
import { getAllDrivers, getDriver, uninstallAll, } from "../notifier/registry.js";
|
|
5
|
+
export async function executeUninstall(opts) {
|
|
6
|
+
const resolveNotifierPathsFn = opts.resolveNotifierPathsFn ?? resolveNotifierPaths;
|
|
7
|
+
const getAllDriversFn = opts.getAllDriversFn ?? getAllDrivers;
|
|
8
|
+
const paths = resolveNotifierPathsFn(opts.home, opts.env);
|
|
9
|
+
const allSources = getAllDriversFn().map((driver) => driver.source);
|
|
10
|
+
const selectedSources = opts.sources && opts.sources.length > 0 ? opts.sources : allSources;
|
|
11
|
+
const fullUninstall = selectedSources.length === allSources.length &&
|
|
12
|
+
allSources.every((source) => selectedSources.includes(source));
|
|
13
|
+
const shouldRemoveCodexBackup = fullUninstall || selectedSources.includes("codex");
|
|
14
|
+
if (opts.dryRun) {
|
|
15
|
+
return {
|
|
16
|
+
notifyHandler: {
|
|
17
|
+
changed: false,
|
|
18
|
+
path: paths.notifyPath,
|
|
19
|
+
detail: fullUninstall ? "dry-run" : "shared artifact kept",
|
|
20
|
+
},
|
|
21
|
+
codexBackup: {
|
|
22
|
+
changed: false,
|
|
23
|
+
path: paths.codexNotifyOriginalPath,
|
|
24
|
+
detail: shouldRemoveCodexBackup ? "dry-run" : "not selected",
|
|
25
|
+
},
|
|
26
|
+
hooks: selectedSources.map((source) => ({
|
|
27
|
+
source,
|
|
28
|
+
action: "skip",
|
|
29
|
+
changed: false,
|
|
30
|
+
detail: "dry-run",
|
|
31
|
+
})),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
let hooks;
|
|
35
|
+
if (fullUninstall) {
|
|
36
|
+
const uninstallAllFn = opts.uninstallAllFn ?? uninstallAll;
|
|
37
|
+
hooks = await uninstallAllFn(paths, { spawn: opts.spawn });
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
const uninstallDriverFn = opts.uninstallDriverFn ??
|
|
41
|
+
(async (source, notifierPaths, deps) => {
|
|
42
|
+
const driver = getDriver(source);
|
|
43
|
+
if (!driver) {
|
|
44
|
+
return {
|
|
45
|
+
source,
|
|
46
|
+
action: "skip",
|
|
47
|
+
changed: false,
|
|
48
|
+
detail: "Unknown source",
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
return driver.uninstall(notifierPaths, deps);
|
|
52
|
+
});
|
|
53
|
+
hooks = [];
|
|
54
|
+
for (const source of selectedSources) {
|
|
55
|
+
try {
|
|
56
|
+
hooks.push(await uninstallDriverFn(source, paths, { spawn: opts.spawn }));
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
hooks.push({
|
|
60
|
+
source,
|
|
61
|
+
action: "skip",
|
|
62
|
+
changed: false,
|
|
63
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
64
|
+
warnings: ["Driver uninstall failed"],
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const removeNotifyHandlerFn = opts.removeNotifyHandlerFn ?? removeNotifyHandler;
|
|
70
|
+
const removeCodexBackupFn = opts.removeCodexBackupFn ?? removeOptionalFile;
|
|
71
|
+
const notifyHandler = fullUninstall
|
|
72
|
+
? await removeNotifyHandlerFn({ notifyPath: paths.notifyPath })
|
|
73
|
+
: {
|
|
74
|
+
changed: false,
|
|
75
|
+
path: paths.notifyPath,
|
|
76
|
+
detail: "shared artifact kept",
|
|
77
|
+
};
|
|
78
|
+
const codexBackup = shouldRemoveCodexBackup
|
|
79
|
+
? await removeCodexBackupFn(paths.codexNotifyOriginalPath)
|
|
80
|
+
: {
|
|
81
|
+
changed: false,
|
|
82
|
+
path: paths.codexNotifyOriginalPath,
|
|
83
|
+
detail: "not selected",
|
|
84
|
+
};
|
|
85
|
+
return { notifyHandler, codexBackup, hooks };
|
|
86
|
+
}
|
|
87
|
+
async function removeOptionalFile(path) {
|
|
88
|
+
try {
|
|
89
|
+
await unlink(path);
|
|
90
|
+
return {
|
|
91
|
+
changed: true,
|
|
92
|
+
path,
|
|
93
|
+
detail: "artifact removed",
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
if (err?.code === "ENOENT") {
|
|
98
|
+
return {
|
|
99
|
+
changed: false,
|
|
100
|
+
path,
|
|
101
|
+
detail: "artifact not found",
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
throw err;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=uninstall.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uninstall.js","sourceRoot":"","sources":["../../src/commands/uninstall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAsB,MAAM,sBAAsB,CAAC;AAChF,OAAO,EACL,aAAa,EACb,SAAS,EACT,YAAY,GACb,MAAM,yBAAyB,CAAC;AAoCjC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAsB;IAC3D,MAAM,sBAAsB,GAAG,IAAI,CAAC,sBAAsB,IAAI,oBAAoB,CAAC;IACnF,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,aAAa,CAAC;IAC9D,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IAC5F,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM;QAChE,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,MAAM,uBAAuB,GAAG,aAAa,IAAI,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEnF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO;YACL,aAAa,EAAE;gBACb,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,KAAK,CAAC,UAAU;gBACtB,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,sBAAsB;aAC3D;YACD,WAAW,EAAE;gBACX,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,KAAK,CAAC,uBAAuB;gBACnC,MAAM,EAAE,uBAAuB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc;aAC7D;YACD,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACtC,MAAM;gBACN,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,SAAS;aAClB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAED,IAAI,KAAgC,CAAC;IACrC,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,YAAY,CAAC;QAC3D,KAAK,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;SAAM,CAAC;QACN,MAAM,iBAAiB,GACrB,IAAI,CAAC,iBAAiB;YACtB,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,EAAE;gBACrC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;gBACjC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO;wBACL,MAAM;wBACN,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,gBAAgB;qBACS,CAAC;gBACtC,CAAC;gBACD,OAAO,MAAM,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC;QAEL,KAAK,GAAG,EAAE,CAAC;QACX,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,KAAK,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC5E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC;oBACT,MAAM;oBACN,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC9D,QAAQ,EAAE,CAAC,yBAAyB,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,IAAI,mBAAmB,CAAC;IAChF,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,kBAAkB,CAAC;IAE3E,MAAM,aAAa,GAAG,aAAa;QACjC,CAAC,CAAC,MAAM,qBAAqB,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC/D,CAAC,CAAC;YACA,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,KAAK,CAAC,UAAU;YACtB,MAAM,EAAE,sBAAsB;SAC/B,CAAC;IAEJ,MAAM,WAAW,GAAG,uBAAuB;QACzC,CAAC,CAAC,MAAM,mBAAmB,CAAC,KAAK,CAAC,uBAAuB,CAAC;QAC1D,CAAC,CAAC;YACA,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,KAAK,CAAC,uBAAuB;YACnC,MAAM,EAAE,cAAc;SACvB,CAAC;IAEJ,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,IAAY;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI;YACJ,MAAM,EAAE,kBAAkB;SAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAAyC,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClE,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,IAAI;gBACJ,MAAM,EAAE,oBAAoB;aAC7B,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic upload engine — sends queued records to a Pew SaaS endpoint.
|
|
3
|
+
*
|
|
4
|
+
* This is the shared pipeline used by both token and session upload commands.
|
|
5
|
+
* It handles:
|
|
6
|
+
* - Config/API-key loading
|
|
7
|
+
* - Queue offset management
|
|
8
|
+
* - Preprocessing (aggregation for tokens, dedup for sessions)
|
|
9
|
+
* - Batching (≤50 records per request, D1 free plan limit)
|
|
10
|
+
* - Retry with exponential backoff on 5xx
|
|
11
|
+
* - 429 rate-limit handling with Retry-After (no double-sleep)
|
|
12
|
+
* - Progress callbacks
|
|
13
|
+
*
|
|
14
|
+
* Bug fixes over the original upload/session-upload:
|
|
15
|
+
* - 429 handler no longer double-sleeps (Retry-After + exponential backoff).
|
|
16
|
+
* Now the Retry-After sleep replaces the exponential backoff for that attempt.
|
|
17
|
+
*/
|
|
18
|
+
import type { BaseQueue } from "../storage/base-queue.js";
|
|
19
|
+
export interface UploadEngineConfig<T> {
|
|
20
|
+
/** The queue to read records from */
|
|
21
|
+
queue: BaseQueue<T>;
|
|
22
|
+
/** API endpoint path (e.g. "/api/ingest" or "/api/ingest/sessions") */
|
|
23
|
+
endpoint: string;
|
|
24
|
+
/** Human-readable name for progress messages (e.g. "records", "session records") */
|
|
25
|
+
entityName: string;
|
|
26
|
+
/** Pre-processing step: aggregation for tokens, dedup for sessions */
|
|
27
|
+
preprocess: (records: T[]) => T[];
|
|
28
|
+
}
|
|
29
|
+
export interface UploadExecuteOptions {
|
|
30
|
+
/** Directory for config file and queue state */
|
|
31
|
+
stateDir: string;
|
|
32
|
+
/** Base URL of the Pew SaaS */
|
|
33
|
+
apiUrl: string;
|
|
34
|
+
/** Whether dev mode is active (uses config.dev.json) */
|
|
35
|
+
dev?: boolean;
|
|
36
|
+
/** Injected fetch (for testing) */
|
|
37
|
+
fetch: typeof globalThis.fetch;
|
|
38
|
+
/** Max records per API request (default: 50) */
|
|
39
|
+
batchSize?: number;
|
|
40
|
+
/** Max retries per batch on 5xx (default: 2) */
|
|
41
|
+
maxRetries?: number;
|
|
42
|
+
/** Base retry delay in ms (default: 1000, doubled each retry) */
|
|
43
|
+
retryDelayMs?: number;
|
|
44
|
+
/** Progress callback */
|
|
45
|
+
onProgress?: (event: UploadProgressEvent) => void;
|
|
46
|
+
}
|
|
47
|
+
export interface UploadProgressEvent {
|
|
48
|
+
phase: "uploading" | "done";
|
|
49
|
+
batch?: number;
|
|
50
|
+
totalBatches?: number;
|
|
51
|
+
total?: number;
|
|
52
|
+
message?: string;
|
|
53
|
+
}
|
|
54
|
+
export interface UploadResult {
|
|
55
|
+
success: boolean;
|
|
56
|
+
uploaded: number;
|
|
57
|
+
batches: number;
|
|
58
|
+
error?: string;
|
|
59
|
+
}
|
|
60
|
+
export declare function createUploadEngine<T>(config: UploadEngineConfig<T>): {
|
|
61
|
+
execute: (opts: UploadExecuteOptions) => Promise<UploadResult>;
|
|
62
|
+
};
|
|
63
|
+
//# sourceMappingURL=upload-engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upload-engine.d.ts","sourceRoot":"","sources":["../../src/commands/upload-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAM1D,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,qCAAqC;IACrC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IACpB,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,UAAU,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB;IACnC,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,mCAAmC;IACnC,KAAK,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAC/B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACnD;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAcD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;oBAGpC,oBAAoB,KAAG,OAAO,CAAC,YAAY,CAAC;EAkG1E"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic upload engine — sends queued records to a Pew SaaS endpoint.
|
|
3
|
+
*
|
|
4
|
+
* This is the shared pipeline used by both token and session upload commands.
|
|
5
|
+
* It handles:
|
|
6
|
+
* - Config/API-key loading
|
|
7
|
+
* - Queue offset management
|
|
8
|
+
* - Preprocessing (aggregation for tokens, dedup for sessions)
|
|
9
|
+
* - Batching (≤50 records per request, D1 free plan limit)
|
|
10
|
+
* - Retry with exponential backoff on 5xx
|
|
11
|
+
* - 429 rate-limit handling with Retry-After (no double-sleep)
|
|
12
|
+
* - Progress callbacks
|
|
13
|
+
*
|
|
14
|
+
* Bug fixes over the original upload/session-upload:
|
|
15
|
+
* - 429 handler no longer double-sleeps (Retry-After + exponential backoff).
|
|
16
|
+
* Now the Retry-After sleep replaces the exponential backoff for that attempt.
|
|
17
|
+
*/
|
|
18
|
+
import { ConfigManager } from "../config/manager.js";
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Constants
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
const DEFAULT_BATCH_SIZE = 50;
|
|
23
|
+
const DEFAULT_MAX_RETRIES = 2;
|
|
24
|
+
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Engine factory
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
export function createUploadEngine(config) {
|
|
29
|
+
const { queue, endpoint, entityName, preprocess } = config;
|
|
30
|
+
async function execute(opts) {
|
|
31
|
+
const { stateDir, apiUrl, dev = false, fetch: fetchFn, batchSize = DEFAULT_BATCH_SIZE, maxRetries = DEFAULT_MAX_RETRIES, retryDelayMs = DEFAULT_RETRY_DELAY_MS, onProgress, } = opts;
|
|
32
|
+
// 1. Load API key
|
|
33
|
+
const configManager = new ConfigManager(stateDir, dev);
|
|
34
|
+
const cfg = await configManager.load();
|
|
35
|
+
if (!cfg.token) {
|
|
36
|
+
return {
|
|
37
|
+
success: false,
|
|
38
|
+
uploaded: 0,
|
|
39
|
+
batches: 0,
|
|
40
|
+
error: "Not logged in. Run `pew login` first.",
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
// 2. Read un-uploaded records
|
|
44
|
+
const currentOffset = await queue.loadOffset();
|
|
45
|
+
const { records: rawRecords, newOffset } = await queue.readFromOffset(currentOffset);
|
|
46
|
+
if (rawRecords.length === 0) {
|
|
47
|
+
return { success: true, uploaded: 0, batches: 0 };
|
|
48
|
+
}
|
|
49
|
+
// 2b. Pre-process (aggregate for tokens, dedup for sessions)
|
|
50
|
+
const records = preprocess(rawRecords);
|
|
51
|
+
// 3. Split into batches
|
|
52
|
+
const batches = [];
|
|
53
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
54
|
+
batches.push(records.slice(i, i + batchSize));
|
|
55
|
+
}
|
|
56
|
+
// 4. Upload each batch
|
|
57
|
+
const fullEndpoint = `${apiUrl}${endpoint}`;
|
|
58
|
+
let totalUploaded = 0;
|
|
59
|
+
let batchesCompleted = 0;
|
|
60
|
+
for (let batchIdx = 0; batchIdx < batches.length; batchIdx++) {
|
|
61
|
+
const batch = batches[batchIdx];
|
|
62
|
+
onProgress?.({
|
|
63
|
+
phase: "uploading",
|
|
64
|
+
batch: batchIdx + 1,
|
|
65
|
+
totalBatches: batches.length,
|
|
66
|
+
total: records.length,
|
|
67
|
+
message: `Uploading ${entityName} batch ${batchIdx + 1}/${batches.length} (${batch.length} records)...`,
|
|
68
|
+
});
|
|
69
|
+
const result = await sendBatchWithRetry({
|
|
70
|
+
endpoint: fullEndpoint,
|
|
71
|
+
token: cfg.token,
|
|
72
|
+
batch,
|
|
73
|
+
fetchFn,
|
|
74
|
+
maxRetries,
|
|
75
|
+
retryDelayMs,
|
|
76
|
+
});
|
|
77
|
+
if (!result.ok) {
|
|
78
|
+
return {
|
|
79
|
+
success: false,
|
|
80
|
+
uploaded: totalUploaded,
|
|
81
|
+
batches: batchesCompleted,
|
|
82
|
+
error: result.error,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
totalUploaded += batch.length;
|
|
86
|
+
batchesCompleted++;
|
|
87
|
+
}
|
|
88
|
+
// 5. All batches succeeded — save final offset
|
|
89
|
+
await queue.saveOffset(newOffset);
|
|
90
|
+
onProgress?.({
|
|
91
|
+
phase: "done",
|
|
92
|
+
total: totalUploaded,
|
|
93
|
+
message: `Uploaded ${totalUploaded} ${entityName} in ${batchesCompleted} batch(es).`,
|
|
94
|
+
});
|
|
95
|
+
return {
|
|
96
|
+
success: true,
|
|
97
|
+
uploaded: totalUploaded,
|
|
98
|
+
batches: batchesCompleted,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return { execute };
|
|
102
|
+
}
|
|
103
|
+
async function sendBatchWithRetry(opts) {
|
|
104
|
+
const { endpoint, token, batch, fetchFn, maxRetries, retryDelayMs } = opts;
|
|
105
|
+
let lastError = "";
|
|
106
|
+
let sleptFor429 = false;
|
|
107
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
108
|
+
// Exponential backoff — but skip if we already slept for a 429 Retry-After
|
|
109
|
+
if (attempt > 0 && !sleptFor429 && retryDelayMs > 0) {
|
|
110
|
+
await sleep(retryDelayMs * 2 ** (attempt - 1));
|
|
111
|
+
}
|
|
112
|
+
sleptFor429 = false; // Reset for next iteration
|
|
113
|
+
try {
|
|
114
|
+
const resp = await fetchFn(endpoint, {
|
|
115
|
+
method: "POST",
|
|
116
|
+
headers: {
|
|
117
|
+
"Content-Type": "application/json",
|
|
118
|
+
Authorization: `Bearer ${token}`,
|
|
119
|
+
},
|
|
120
|
+
body: JSON.stringify(batch),
|
|
121
|
+
});
|
|
122
|
+
if (resp.ok) {
|
|
123
|
+
return { ok: true };
|
|
124
|
+
}
|
|
125
|
+
// 429 — rate limited, retry with Retry-After if available
|
|
126
|
+
if (resp.status === 429) {
|
|
127
|
+
const retryAfter = resp.headers.get("Retry-After");
|
|
128
|
+
const retryMs = retryAfter
|
|
129
|
+
? Math.max(Number(retryAfter) * 1000, retryDelayMs)
|
|
130
|
+
: retryDelayMs * 2 ** attempt;
|
|
131
|
+
if (attempt < maxRetries && retryMs > 0) {
|
|
132
|
+
await sleep(retryMs);
|
|
133
|
+
sleptFor429 = true; // Skip the top-of-loop backoff on next iteration
|
|
134
|
+
}
|
|
135
|
+
const body = await resp.json().catch(() => ({}));
|
|
136
|
+
lastError = `429: ${body.error ?? "Too Many Requests"}`;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
// 4xx — client error, don't retry
|
|
140
|
+
if (resp.status >= 400 && resp.status < 500) {
|
|
141
|
+
const body = await resp.json().catch(() => ({}));
|
|
142
|
+
const msg = body.error ?? `HTTP ${resp.status}`;
|
|
143
|
+
return { ok: false, error: `${resp.status}: ${msg}` };
|
|
144
|
+
}
|
|
145
|
+
// 5xx — server error, retry
|
|
146
|
+
const body = await resp.json().catch(() => ({}));
|
|
147
|
+
lastError = `${resp.status}: ${body.error ?? "Server Error"}`;
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
lastError = String(err.message ?? err);
|
|
151
|
+
if (attempt >= maxRetries) {
|
|
152
|
+
return { ok: false, error: lastError };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
ok: false,
|
|
158
|
+
error: `Upload failed after ${maxRetries + 1} attempts: ${lastError}`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
function sleep(ms) {
|
|
162
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=upload-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upload-engine.js","sourceRoot":"","sources":["../../src/commands/upload-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAoDrD,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,UAAU,kBAAkB,CAAI,MAA6B;IACjE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAE3D,KAAK,UAAU,OAAO,CAAC,IAA0B;QAC/C,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,GAAG,GAAG,KAAK,EACX,KAAK,EAAE,OAAO,EACd,SAAS,GAAG,kBAAkB,EAC9B,UAAU,GAAG,mBAAmB,EAChC,YAAY,GAAG,sBAAsB,EACrC,UAAU,GACX,GAAG,IAAI,CAAC;QAET,kBAAkB;QAClB,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAEvC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,uCAAuC;aAC/C,CAAC;QACJ,CAAC;QAED,8BAA8B;QAC9B,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;QAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GACtC,MAAM,KAAK,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAE5C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,6DAA6D;QAC7D,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QAEvC,wBAAwB;QACxB,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,uBAAuB;QACvB,MAAM,YAAY,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC5C,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAEhC,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,KAAK,EAAE,QAAQ,GAAG,CAAC;gBACnB,YAAY,EAAE,OAAO,CAAC,MAAM;gBAC5B,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,OAAO,EAAE,aAAa,UAAU,UAAU,QAAQ,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,cAAc;aACxG,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC;gBACtC,QAAQ,EAAE,YAAY;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,KAAK;gBACL,OAAO;gBACP,UAAU;gBACV,YAAY;aACb,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,aAAa;oBACvB,OAAO,EAAE,gBAAgB;oBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB,CAAC;YACJ,CAAC;YAED,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC;YAC9B,gBAAgB,EAAE,CAAC;QACrB,CAAC;QAED,+CAA+C;QAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAElC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE,YAAY,aAAa,IAAI,UAAU,OAAO,gBAAgB,aAAa;SACrF,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,gBAAgB;SAC1B,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,CAAC;AACrB,CAAC;AAWD,KAAK,UAAU,kBAAkB,CAAI,IAOpC;IACC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;IAE3E,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,2EAA2E;QAC3E,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,KAAK,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,WAAW,GAAG,KAAK,CAAC,CAAC,2BAA2B;QAEhD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE;gBACnC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,KAAK,EAAE;iBACjC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;gBACZ,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YACtB,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACxB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBACnD,MAAM,OAAO,GAAG,UAAU;oBACxB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,EAAE,YAAY,CAAC;oBACnD,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,OAAO,CAAC;gBAChC,IAAI,OAAO,GAAG,UAAU,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBACxC,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,WAAW,GAAG,IAAI,CAAC,CAAC,iDAAiD;gBACvE,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACjD,SAAS,GAAG,QAAS,IAA+B,CAAC,KAAK,IAAI,mBAAmB,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YAED,kCAAkC;YAClC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACjD,MAAM,GAAG,GACN,IAA+B,CAAC,KAAK,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,EAAE,CAAC;YACxD,CAAC;YAED,4BAA4B;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjD,SAAS,GAAG,GAAG,IAAI,CAAC,MAAM,KAAM,IAA+B,CAAC,KAAK,IAAI,cAAc,EAAE,CAAC;QAC5F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,MAAM,CAAE,GAAa,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;YAElD,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBAC1B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,uBAAuB,UAAU,GAAG,CAAC,cAAc,SAAS,EAAE;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|