@magicpixelart/cli 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +415 -0
- package/README.md +31 -2
- package/dist/api.js +173 -54
- package/dist/commands/doctor.js +201 -56
- package/dist/commands/init.js +25 -15
- package/dist/commands/list.js +6 -5
- package/dist/commands/login.js +3 -34
- package/dist/commands/repair.js +157 -0
- package/dist/commands/start.js +56 -14
- package/dist/commands/status.js +41 -13
- package/dist/commands/sync.js +297 -135
- package/dist/commands/whoami.js +52 -12
- package/dist/config.js +88 -20
- package/dist/index.js +38 -23
- package/dist/util/atomicWrite.js +25 -0
- package/dist/util/auth.js +41 -0
- package/dist/util/authHeaders.js +24 -0
- package/dist/util/credentials.js +27 -10
- package/dist/util/emitIndex.js +78 -70
- package/dist/util/errors.js +49 -0
- package/dist/util/flagValidators.js +49 -0
- package/dist/util/format.js +14 -0
- package/dist/util/framework.js +10 -0
- package/dist/util/iso.js +28 -0
- package/dist/util/paths.js +99 -0
- package/dist/util/security.js +39 -2
- package/dist/util/telemetry.js +182 -0
- package/package.json +11 -4
package/dist/commands/sync.js
CHANGED
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
import kleur from 'kleur';
|
|
2
2
|
import ora from 'ora';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
|
-
import { mkdir, writeFile, unlink,
|
|
4
|
+
import { mkdir, writeFile, unlink, rename } from 'node:fs/promises';
|
|
5
5
|
import { existsSync } from 'node:fs';
|
|
6
6
|
import { dirname, relative, resolve } from 'node:path';
|
|
7
7
|
import { loadConfig, loadState, saveState } from '../config.js';
|
|
8
8
|
import { fetchAllManifest, fetchAssetBytes, ApiError } from '../api.js';
|
|
9
9
|
import { fileSha256 } from '../util/hash.js';
|
|
10
|
-
import { assetDiskPath, assetDiskPathFromKey } from '../util/paths.js';
|
|
10
|
+
import { assetDiskPath, assetDiskPathFromKey, pruneEmptyDirs, walkOutDirPngs } from '../util/paths.js';
|
|
11
11
|
import { createLimit } from '../util/limit.js';
|
|
12
12
|
import { emitTypedIndex, ensureAgentsDoc, scanDiskAssets } from '../util/emitIndex.js';
|
|
13
13
|
import { assertPathInsideRoot, tmpPathFor } from '../util/security.js';
|
|
14
|
+
import { friendlyFsError } from '../util/errors.js';
|
|
15
|
+
import { maxIsoTimestamp } from '../util/iso.js';
|
|
16
|
+
import { formatBytes } from '../util/format.js';
|
|
14
17
|
export async function syncCommand(opts) {
|
|
15
18
|
if (opts.watch) {
|
|
16
19
|
await watchLoop(opts);
|
|
@@ -19,7 +22,13 @@ export async function syncCommand(opts) {
|
|
|
19
22
|
await runOnce(opts);
|
|
20
23
|
}
|
|
21
24
|
async function watchLoop(opts) {
|
|
22
|
-
|
|
25
|
+
// Default 2s — matches the perceived-instant UX users expect when editing
|
|
26
|
+
// sprites in MagicPixel and watching them refresh in their game's dev
|
|
27
|
+
// server. Incremental polls send `?since=<lastSync>` so an empty manifest
|
|
28
|
+
// round-trip is cheap (a single HTTP HEAD-sized response with `count: 0`).
|
|
29
|
+
// Commander already validated the numeric value (2–3600); the `?? 2`
|
|
30
|
+
// covers the bare boolean `-w` form.
|
|
31
|
+
const intervalSec = typeof opts.watch === 'string' ? parseInt(opts.watch, 10) : 2;
|
|
23
32
|
// Header — print once on start. Project/asset count is best-effort; if the
|
|
24
33
|
// first manifest fetch fails we still want the loop to come up and retry.
|
|
25
34
|
let headerCount = null;
|
|
@@ -34,38 +43,97 @@ async function watchLoop(opts) {
|
|
|
34
43
|
console.log(` Edit at: ${kleur.cyan('https://magicpixel.art')}`);
|
|
35
44
|
if (headerCount !== null)
|
|
36
45
|
console.log(kleur.dim(` Sprites: ${headerCount}`));
|
|
37
|
-
console.log(kleur.dim(` Polling: every ${intervalSec}s · Stop: Ctrl+C`));
|
|
46
|
+
console.log(kleur.dim(` Polling: every ${intervalSec}s (slows when idle) · Stop: Ctrl+C`));
|
|
38
47
|
console.log();
|
|
39
48
|
let stopping = false;
|
|
40
49
|
let inFlight = false;
|
|
41
|
-
let resolveIdle = null;
|
|
42
50
|
let backoffSec = intervalSec;
|
|
43
51
|
let pausedForAuth = false;
|
|
44
|
-
|
|
52
|
+
// Mirrors pausedForAuth for the network-offline path: once we've told the
|
|
53
|
+
// user "MagicPixel is offline", the next successful tick prints a single
|
|
54
|
+
// "back online" recovery line. Without this flag a user who walked away
|
|
55
|
+
// during an outage has no signal that things are healthy again.
|
|
56
|
+
let pausedForNetwork = false;
|
|
57
|
+
// After this many consecutive 401/403s we give up and exit non-zero so a
|
|
58
|
+
// parent process (Vite plugin, systemd, pm2) can tell the watcher is
|
|
59
|
+
// genuinely broken (revoked key) rather than transiently blipped.
|
|
60
|
+
const MAX_AUTH_FAILURES = 5;
|
|
61
|
+
let consecutiveAuthFailures = 0;
|
|
62
|
+
// Adaptive idle backoff: after a few minutes of nothing-to-do we slow the
|
|
63
|
+
// poll from intervalSec → 5s → 10s so a dev who walked away isn't hammering
|
|
64
|
+
// the manifest endpoint. ANY change OR error resets this back to intervalSec,
|
|
65
|
+
// so the "edit → see it" promise stays intact the moment the user comes
|
|
66
|
+
// back. Error backoff (2→60s) is separate and continues to win.
|
|
67
|
+
let consecutiveIdleTicks = 0;
|
|
68
|
+
// Thresholds in seconds (NOT ticks) so a `--watch 10` user gets the same
|
|
69
|
+
// ~3 min / ~15 min UX as a default `--watch 2` user. Previously these were
|
|
70
|
+
// tick counts (90 / 300) which made the slowdown timing scale with intervalSec.
|
|
71
|
+
const IDLE_SOFT_BACKOFF_SECONDS = 180;
|
|
72
|
+
const IDLE_HARD_BACKOFF_SECONDS = 900;
|
|
73
|
+
// Cancellable idle sleep — `onStopSignal` calls `wakeStop()` so the next
|
|
74
|
+
// tick exits immediately instead of waiting out the full backoff (which can
|
|
75
|
+
// be 60s at the error ceiling). Initialised to a no-op so the first
|
|
76
|
+
// pre-loop `await tick()` is safe even before the first sleep.
|
|
77
|
+
let wakeStop = () => { };
|
|
78
|
+
// Handle both SIGINT (Ctrl+C) and SIGTERM (`kill`, `docker stop`, systemd,
|
|
79
|
+
// pm2). Without the SIGTERM listener a supervisor-managed watcher would die
|
|
80
|
+
// mid-sync without draining in-flight work or preserving the exit code.
|
|
81
|
+
const onStopSignal = (signal) => {
|
|
45
82
|
if (stopping)
|
|
46
83
|
return;
|
|
47
84
|
stopping = true;
|
|
85
|
+
wakeStop();
|
|
48
86
|
process.stdout.write('\x1b[2K\r');
|
|
49
87
|
if (inFlight) {
|
|
50
|
-
console.log(kleur.dim(
|
|
51
|
-
process.once(
|
|
88
|
+
console.log(kleur.dim(`[watch] finishing current sync… (${signal} again to force quit)`));
|
|
89
|
+
process.once(signal, () => process.exit(signal === 'SIGINT' ? 130 : 143));
|
|
52
90
|
}
|
|
53
91
|
else {
|
|
54
|
-
|
|
55
|
-
|
|
92
|
+
if (!opts.quiet)
|
|
93
|
+
console.log(kleur.dim('[watch] stopped.'));
|
|
94
|
+
// Preserve any exit code already set by a prior failed tick.
|
|
95
|
+
process.exit(process.exitCode ?? 0);
|
|
56
96
|
}
|
|
57
97
|
};
|
|
58
|
-
process.on('SIGINT',
|
|
98
|
+
process.on('SIGINT', onStopSignal);
|
|
99
|
+
process.on('SIGTERM', onStopSignal);
|
|
59
100
|
const tick = async () => {
|
|
60
101
|
if (inFlight || stopping)
|
|
61
102
|
return;
|
|
62
103
|
inFlight = true;
|
|
63
104
|
try {
|
|
64
105
|
const r = await runOnce({ ...opts, watch: false }, { watchMode: true });
|
|
65
|
-
// Reset backoff
|
|
106
|
+
// Reset backoff on any successful tick. Resume message fires once when
|
|
107
|
+
// we recover from an auth pause — `getApiKey()` re-reads
|
|
108
|
+
// .magicpixel/credentials on every call, so a `magicpixel login` in
|
|
109
|
+
// another terminal is picked up automatically by the next tick.
|
|
110
|
+
const wasPausedForAuth = pausedForAuth;
|
|
111
|
+
const wasPausedForNetwork = pausedForNetwork;
|
|
66
112
|
backoffSec = intervalSec;
|
|
67
113
|
pausedForAuth = false;
|
|
114
|
+
pausedForNetwork = false;
|
|
115
|
+
consecutiveAuthFailures = 0;
|
|
116
|
+
if (wasPausedForAuth && !opts.quiet) {
|
|
117
|
+
process.stdout.write('\x1b[2K\r');
|
|
118
|
+
console.log(`${kleur.dim(timestamp())} ${kleur.green('✓')} Key accepted again — resuming.`);
|
|
119
|
+
}
|
|
120
|
+
if (wasPausedForNetwork && !wasPausedForAuth && !opts.quiet) {
|
|
121
|
+
process.stdout.write('\x1b[2K\r');
|
|
122
|
+
console.log(`${kleur.dim(timestamp())} ${kleur.green('✓')} Back online — resuming.`);
|
|
123
|
+
}
|
|
68
124
|
const changedCount = r.added.length + r.modified.length + r.removed.length + r.renamed.length;
|
|
125
|
+
if (changedCount > 0) {
|
|
126
|
+
// Snap back to the fast interval the moment anything changes.
|
|
127
|
+
consecutiveIdleTicks = 0;
|
|
128
|
+
backoffSec = intervalSec;
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
consecutiveIdleTicks++;
|
|
132
|
+
backoffSec = nextBackoffForIdle(consecutiveIdleTicks, intervalSec, {
|
|
133
|
+
softSec: IDLE_SOFT_BACKOFF_SECONDS,
|
|
134
|
+
hardSec: IDLE_HARD_BACKOFF_SECONDS,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
69
137
|
if (opts.quiet)
|
|
70
138
|
return;
|
|
71
139
|
if (changedCount > 0) {
|
|
@@ -84,54 +152,141 @@ async function watchLoop(opts) {
|
|
|
84
152
|
const apiErr = err instanceof ApiError ? err : null;
|
|
85
153
|
const firstLine = err.message.split('\n')[0];
|
|
86
154
|
process.stdout.write('\x1b[2K\r');
|
|
87
|
-
|
|
155
|
+
// Any error breaks the idle streak so we come back fast once it clears.
|
|
156
|
+
consecutiveIdleTicks = 0;
|
|
157
|
+
const decision = classifyTickError(err, {
|
|
158
|
+
backoffSec,
|
|
159
|
+
pausedForAuth,
|
|
160
|
+
pausedForNetwork,
|
|
161
|
+
consecutiveAuthFailures,
|
|
162
|
+
maxAuthFailures: MAX_AUTH_FAILURES,
|
|
163
|
+
});
|
|
164
|
+
if (decision.kind === 'auth') {
|
|
88
165
|
if (!pausedForAuth) {
|
|
89
|
-
|
|
166
|
+
// Surface the request id (from B1) so the user can paste it into a
|
|
167
|
+
// support thread and we can correlate against edge function logs.
|
|
168
|
+
const idSuffix = apiErr?.requestId ? kleur.dim(` (request id: ${apiErr.requestId})`) : '';
|
|
169
|
+
console.log(`${kleur.dim(timestamp())} ${kleur.red('✗')} Your key looks invalid or rotated.${idSuffix}`);
|
|
90
170
|
console.log(kleur.dim(' Fix: run `magicpixel login` (this watcher will keep retrying every 30s).'));
|
|
91
171
|
}
|
|
92
172
|
pausedForAuth = true;
|
|
93
|
-
|
|
173
|
+
consecutiveAuthFailures = decision.consecutiveAuthFailures;
|
|
174
|
+
backoffSec = decision.nextBackoffSec;
|
|
175
|
+
if (decision.giveUp) {
|
|
176
|
+
console.log(`${kleur.dim(timestamp())} ${kleur.red('✗')} Giving up after ${MAX_AUTH_FAILURES} consecutive auth failures.`);
|
|
177
|
+
console.log(kleur.dim(' Fix: run `magicpixel login` with a fresh key, then restart the watcher.'));
|
|
178
|
+
// Surface this to /admin/errors — persistent watcher auth failure
|
|
179
|
+
// means a key is mass-rejected (revoked, project deleted, edge
|
|
180
|
+
// misconfig) and we want visibility without waiting for a support
|
|
181
|
+
// ping. Awaited so the report flushes before exit.
|
|
182
|
+
const { reportAndExit } = await import('../util/telemetry.js');
|
|
183
|
+
await reportAndExit(err, 'sync (watch)', 2);
|
|
184
|
+
}
|
|
94
185
|
}
|
|
95
|
-
else if (
|
|
96
|
-
|
|
186
|
+
else if (decision.kind === 'network') {
|
|
187
|
+
consecutiveAuthFailures = 0;
|
|
188
|
+
if (decision.printMessage) {
|
|
189
|
+
const idSuffix = apiErr?.requestId ? kleur.dim(` (request id: ${apiErr.requestId})`) : '';
|
|
97
190
|
console.log(`${kleur.dim(timestamp())} ${kleur.yellow('!')} MagicPixel is offline or your internet is. ` +
|
|
98
|
-
`Sprites you already have still work. Retrying in ${
|
|
191
|
+
`Sprites you already have still work. Retrying in ${decision.nextBackoffSec}s.${idSuffix}`);
|
|
99
192
|
}
|
|
100
|
-
|
|
193
|
+
pausedForNetwork = true;
|
|
194
|
+
backoffSec = decision.nextBackoffSec;
|
|
101
195
|
}
|
|
102
196
|
else {
|
|
103
|
-
|
|
104
|
-
|
|
197
|
+
consecutiveAuthFailures = 0;
|
|
198
|
+
const idSuffix = apiErr?.requestId ? kleur.dim(` (request id: ${apiErr.requestId})`) : '';
|
|
199
|
+
console.log(`${kleur.dim(timestamp())} ${kleur.red('!')} ${firstLine}${idSuffix}`);
|
|
200
|
+
backoffSec = decision.nextBackoffSec;
|
|
105
201
|
}
|
|
106
202
|
}
|
|
107
203
|
finally {
|
|
108
204
|
inFlight = false;
|
|
109
|
-
resolveIdle?.();
|
|
110
|
-
resolveIdle = null;
|
|
111
205
|
}
|
|
112
206
|
};
|
|
113
207
|
await tick();
|
|
114
208
|
while (!stopping) {
|
|
115
|
-
|
|
209
|
+
// Race the sleep against a stop signal so Ctrl+C during a long error-
|
|
210
|
+
// backoff (up to 60s) exits within a tick instead of after the full wait.
|
|
211
|
+
await new Promise((resolve) => {
|
|
212
|
+
const timer = setTimeout(() => {
|
|
213
|
+
wakeStop = () => { };
|
|
214
|
+
resolve();
|
|
215
|
+
}, backoffSec * 1000);
|
|
216
|
+
wakeStop = () => {
|
|
217
|
+
clearTimeout(timer);
|
|
218
|
+
wakeStop = () => { };
|
|
219
|
+
resolve();
|
|
220
|
+
};
|
|
221
|
+
});
|
|
116
222
|
if (stopping)
|
|
117
223
|
break;
|
|
224
|
+
// `await tick()` only returns once its `finally` runs (clearing
|
|
225
|
+
// `inFlight`), so the loop naturally drains the in-flight sync on Ctrl+C
|
|
226
|
+
// before we exit — no separate idle-promise dance required.
|
|
118
227
|
await tick();
|
|
119
228
|
}
|
|
120
|
-
if (
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
console.log(kleur.dim('[watch] stopped.'));
|
|
126
|
-
process.exit(0);
|
|
229
|
+
if (!opts.quiet)
|
|
230
|
+
console.log(kleur.dim('[watch] stopped.'));
|
|
231
|
+
// Preserve any non-zero exit code set by a failed tick — don't mask a
|
|
232
|
+
// download failure with a clean exit just because the user pressed Ctrl+C.
|
|
233
|
+
process.exit(process.exitCode ?? 0);
|
|
127
234
|
}
|
|
128
235
|
function isNetworkError(err) {
|
|
129
236
|
// The api layer wraps fetch failures as `manifest: network error (...)`;
|
|
130
237
|
// bare ENOTFOUND/ETIMEDOUT/etc. also surface here from `fetchAssetBytes`.
|
|
238
|
+
// Include 502/504 strings — corporate proxies often surface upstream gateway
|
|
239
|
+
// failures as terse text rather than as ApiError (e.g. 502 from a TLS
|
|
240
|
+
// terminator before our edge function ever sees the request).
|
|
131
241
|
const msg = err.message;
|
|
132
242
|
return (/network error/i.test(msg) ||
|
|
133
|
-
/ENOTFOUND|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN/i.test(msg) ||
|
|
134
|
-
/fetch failed/i.test(msg)
|
|
243
|
+
/ENOTFOUND|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH/i.test(msg) ||
|
|
244
|
+
/fetch failed/i.test(msg) ||
|
|
245
|
+
/\b(502|504)\b.*\b(bad gateway|gateway timeout)\b/i.test(msg));
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Pure helper: given how many consecutive idle ticks we've seen and the
|
|
249
|
+
* configured poll interval, return the next backoff in seconds. Exported so
|
|
250
|
+
* the watch-mode regression test can guard the "ticks vs seconds" bug fixed
|
|
251
|
+
* in 0.4.0 without exercising the full watch loop.
|
|
252
|
+
*/
|
|
253
|
+
export function nextBackoffForIdle(consecutiveIdleTicks, intervalSec, thresholds = { softSec: 180, hardSec: 900 }) {
|
|
254
|
+
const idleSeconds = consecutiveIdleTicks * intervalSec;
|
|
255
|
+
if (idleSeconds >= thresholds.hardSec)
|
|
256
|
+
return Math.max(intervalSec, 10);
|
|
257
|
+
if (idleSeconds >= thresholds.softSec)
|
|
258
|
+
return Math.max(intervalSec, 5);
|
|
259
|
+
return intervalSec;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Pure helper: classify a per-tick error and compute the next backoff +
|
|
263
|
+
* counter updates. Extracted so the auth-failure watchdog and the
|
|
264
|
+
* network-recovery message can be unit-tested without spinning up signals or
|
|
265
|
+
* mocking timers.
|
|
266
|
+
*
|
|
267
|
+
* Caller is responsible for performing side effects (logging, `process.exit`,
|
|
268
|
+
* assigning the returned values back onto loop state).
|
|
269
|
+
*/
|
|
270
|
+
export function classifyTickError(err, state) {
|
|
271
|
+
const apiErr = err instanceof ApiError ? err : null;
|
|
272
|
+
if (apiErr && (apiErr.status === 401 || apiErr.status === 403)) {
|
|
273
|
+
const next = state.consecutiveAuthFailures + 1;
|
|
274
|
+
return {
|
|
275
|
+
kind: 'auth',
|
|
276
|
+
nextBackoffSec: 30,
|
|
277
|
+
consecutiveAuthFailures: next,
|
|
278
|
+
giveUp: next >= state.maxAuthFailures,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
if (isNetworkError(err)) {
|
|
282
|
+
const nextBackoffSec = Math.min(state.backoffSec * 2, 60);
|
|
283
|
+
// Print on the first offline tick, then again whenever backoff changes —
|
|
284
|
+
// capped at 60s so users who walked away still see periodic confirmation
|
|
285
|
+
// the watcher is alive and retrying.
|
|
286
|
+
const printMessage = !state.pausedForNetwork || nextBackoffSec !== state.backoffSec;
|
|
287
|
+
return { kind: 'network', nextBackoffSec, printMessage };
|
|
288
|
+
}
|
|
289
|
+
return { kind: 'other', nextBackoffSec: Math.min(state.backoffSec * 2, 60) };
|
|
135
290
|
}
|
|
136
291
|
async function runOnce(opts, runOpts = {}) {
|
|
137
292
|
const config = await loadConfig();
|
|
@@ -165,13 +320,28 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
165
320
|
renamed.push({ id: entry.id, oldKey: prevKey, newKey: entry.key });
|
|
166
321
|
}
|
|
167
322
|
}
|
|
168
|
-
//
|
|
323
|
+
// Compute the disk path once per entry and reuse it across the diff /
|
|
324
|
+
// orphan / download loops — saves three resolves + two security asserts
|
|
325
|
+
// per asset on large projects. Every entry in `manifest` is pre-seeded, so
|
|
326
|
+
// the lookup never misses; non-null assertion is safe.
|
|
327
|
+
const diskPathById = new Map();
|
|
328
|
+
for (const entry of manifest) {
|
|
329
|
+
diskPathById.set(entry.id, assetDiskPath(config.outDir, entry));
|
|
330
|
+
}
|
|
331
|
+
const pathFor = (entry) => diskPathById.get(entry.id);
|
|
332
|
+
// Diff against disk. SHA pre-check runs through the same concurrency pool
|
|
333
|
+
// used for downloads — on a 1k-asset project that's all-unchanged this is
|
|
334
|
+
// the dominant wall-clock cost of an incremental sync. We cache each result
|
|
335
|
+
// so the download loop can reuse it for `If-None-Match` ETags instead of
|
|
336
|
+
// re-hashing the same file moments later.
|
|
337
|
+
const diffLimit = createLimit(concurrency);
|
|
338
|
+
const shaByEntryId = new Map();
|
|
169
339
|
const toDownload = [];
|
|
170
340
|
let bytesSaved = 0;
|
|
171
341
|
let unchanged = 0;
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
|
|
342
|
+
await Promise.all(manifest.map((entry) => diffLimit(async () => {
|
|
343
|
+
const localSha = await fileSha256(pathFor(entry));
|
|
344
|
+
shaByEntryId.set(entry.id, localSha);
|
|
175
345
|
if (entry.sha256 && localSha && entry.sha256 === localSha) {
|
|
176
346
|
unchanged++;
|
|
177
347
|
if (entry.size_bytes)
|
|
@@ -180,14 +350,14 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
180
350
|
else {
|
|
181
351
|
toDownload.push(entry);
|
|
182
352
|
}
|
|
183
|
-
}
|
|
353
|
+
})));
|
|
184
354
|
// Orphan detection only when we have the full picture.
|
|
185
355
|
// (Renames also produce a stale path on disk — collected separately below.)
|
|
186
356
|
let orphans = [];
|
|
187
357
|
if (!since) {
|
|
188
|
-
const remoteDiskPaths = new Set(manifest.map((e) =>
|
|
189
|
-
|
|
190
|
-
|
|
358
|
+
const remoteDiskPaths = new Set(manifest.map((e) => pathFor(e)));
|
|
359
|
+
const localPngs = await walkOutDirPngs(config.outDir);
|
|
360
|
+
orphans = localPngs.map((a) => a.abs).filter((p) => !remoteDiskPaths.has(p));
|
|
191
361
|
}
|
|
192
362
|
// Stale paths from detected renames (always pruned, even in incremental mode —
|
|
193
363
|
// otherwise the old PNG silently lingers next to the renamed copy).
|
|
@@ -251,13 +421,25 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
251
421
|
const run = createLimit(concurrency);
|
|
252
422
|
let done = 0;
|
|
253
423
|
await Promise.all(toDownload.map((entry) => run(async () => {
|
|
254
|
-
const diskPath =
|
|
424
|
+
const diskPath = pathFor(entry);
|
|
255
425
|
const existedBefore = existsSync(diskPath);
|
|
256
426
|
try {
|
|
257
|
-
|
|
427
|
+
// Reuse the SHA computed during the diff pre-check — re-hashing the
|
|
428
|
+
// same file moments later was a measurable cost on large projects.
|
|
429
|
+
const localSha = shaByEntryId.get(entry.id) ?? null;
|
|
258
430
|
const bytes = await fetchAssetBytes(config, entry.key, localSha);
|
|
259
431
|
if (bytes === null) {
|
|
432
|
+
// Server returned 304 (ETag matched). Credit the asset's manifest
|
|
433
|
+
// size to bytesSaved so the end-of-sync summary reflects the
|
|
434
|
+
// bandwidth the conditional GET avoided.
|
|
435
|
+
//
|
|
436
|
+
// Invariant: we only reach here for entries that made it into
|
|
437
|
+
// `toDownload`, i.e. the disk-SHA pre-check above did NOT credit
|
|
438
|
+
// them (typically because `entry.sha256` was null in the manifest
|
|
439
|
+
// but the ETag still matched server-side). So no double-counting.
|
|
260
440
|
result.unchanged++;
|
|
441
|
+
if (entry.size_bytes)
|
|
442
|
+
result.bytesSaved += entry.size_bytes;
|
|
261
443
|
}
|
|
262
444
|
else {
|
|
263
445
|
if (entry.sha256) {
|
|
@@ -266,11 +448,22 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
266
448
|
throw new Error(`sha256 mismatch (expected ${entry.sha256.slice(0, 12)}…, got ${actual.slice(0, 12)}…)`);
|
|
267
449
|
}
|
|
268
450
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
451
|
+
// Wrap any of mkdir/writeFile/rename so the user sees a friendly
|
|
452
|
+
// multi-line diagnostic instead of a raw `EACCES: permission denied`.
|
|
453
|
+
try {
|
|
454
|
+
await mkdir(dirname(diskPath), { recursive: true });
|
|
455
|
+
const tmp = tmpPathFor(diskPath);
|
|
456
|
+
assertPathInsideRoot(tmp, resolve(process.cwd(), config.outDir), 'outDir');
|
|
457
|
+
await writeFile(tmp, bytes);
|
|
458
|
+
await rename(tmp, diskPath);
|
|
459
|
+
}
|
|
460
|
+
catch (fsErr) {
|
|
461
|
+
throw friendlyFsError(fsErr, {
|
|
462
|
+
operation: `Writing asset`,
|
|
463
|
+
path: diskPath,
|
|
464
|
+
hint: `Sync can't continue until outDir (${config.outDir}) is writable.`,
|
|
465
|
+
});
|
|
466
|
+
}
|
|
274
467
|
if (existedBefore)
|
|
275
468
|
result.modified.push(entry.key);
|
|
276
469
|
else
|
|
@@ -281,7 +474,17 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
281
474
|
catch (e) {
|
|
282
475
|
result.failed++;
|
|
283
476
|
progress?.stop();
|
|
284
|
-
|
|
477
|
+
// Multi-line messages come from friendlyFsError — print all lines
|
|
478
|
+
// so the user sees the fix hint. Single-line errors stay terse.
|
|
479
|
+
const msg = e.message ?? String(e);
|
|
480
|
+
if (msg.includes('\n')) {
|
|
481
|
+
console.log(` ${kleur.red('!')} ${entry.key}:`);
|
|
482
|
+
for (const line of msg.split('\n'))
|
|
483
|
+
console.log(` ${line}`);
|
|
484
|
+
}
|
|
485
|
+
else {
|
|
486
|
+
console.log(` ${kleur.red('!')} ${entry.key}: ${msg}`);
|
|
487
|
+
}
|
|
285
488
|
progress?.start();
|
|
286
489
|
}
|
|
287
490
|
finally {
|
|
@@ -339,7 +542,27 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
339
542
|
}
|
|
340
543
|
const nextState = { ...state, assets: nextAssets };
|
|
341
544
|
if (result.failed === 0) {
|
|
342
|
-
|
|
545
|
+
// Advance the cursor to the newest row we actually observed, NOT the
|
|
546
|
+
// wall-clock time the sync started. Using startedAt silently skipped
|
|
547
|
+
// rows whose `updated_at` lived in the gap between the manifest snapshot
|
|
548
|
+
// and the next poll (notably when only metadata changed — e.g. artboard
|
|
549
|
+
// renames — and the row updated between fetch start and save).
|
|
550
|
+
const maxUpdatedAt = maxIsoTimestamp(manifest.map((e) => e.updated_at));
|
|
551
|
+
if (maxUpdatedAt) {
|
|
552
|
+
// Take the later of: newest row we saw, or the prior cursor. Never
|
|
553
|
+
// rewind — an incremental sync that returned 0 rows must keep the
|
|
554
|
+
// existing lastSync (otherwise we'd re-download history next tick).
|
|
555
|
+
nextState.lastSync =
|
|
556
|
+
state.lastSync && state.lastSync > maxUpdatedAt ? state.lastSync : maxUpdatedAt;
|
|
557
|
+
}
|
|
558
|
+
else if (state.lastSync) {
|
|
559
|
+
nextState.lastSync = state.lastSync;
|
|
560
|
+
}
|
|
561
|
+
else {
|
|
562
|
+
// No prior cursor and an empty manifest (fresh project): fall back to
|
|
563
|
+
// startedAt so future incremental polls have a baseline.
|
|
564
|
+
nextState.lastSync = startedAt;
|
|
565
|
+
}
|
|
343
566
|
delete nextState.lastError;
|
|
344
567
|
}
|
|
345
568
|
else {
|
|
@@ -361,18 +584,20 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
361
584
|
const indexPath = await emitTypedIndex(config.outDir, diskEntries);
|
|
362
585
|
if (verbose)
|
|
363
586
|
console.log(kleur.dim(` index → ${relative(process.cwd(), indexPath)}`));
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
// Never let an AGENTS.md write failure break sync.
|
|
587
|
+
}
|
|
588
|
+
// AGENTS.md hint is always written — `public/` and `static/` users
|
|
589
|
+
// (emitIndex: false) need the absolute-URL snippet just as much as
|
|
590
|
+
// bundler-importable outDir users need the ES-import one.
|
|
591
|
+
// `ensureAgentsDoc` no-ops once our marker section is present.
|
|
592
|
+
try {
|
|
593
|
+
const agentsResult = await ensureAgentsDoc(config.outDir);
|
|
594
|
+
if (verbose && agentsResult !== 'unchanged') {
|
|
595
|
+
console.log(kleur.dim(` AGENTS.md ${agentsResult}`));
|
|
374
596
|
}
|
|
375
597
|
}
|
|
598
|
+
catch {
|
|
599
|
+
// Never let an AGENTS.md write failure break sync.
|
|
600
|
+
}
|
|
376
601
|
await saveState(nextState);
|
|
377
602
|
if (result.failed > 0 && verbose) {
|
|
378
603
|
console.log(kleur.yellow(`\n${result.failed} download${result.failed === 1 ? '' : 's'} failed — lastSync not advanced. Re-run to retry.`));
|
|
@@ -386,7 +611,10 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
386
611
|
(renamed.length ? `, renamed ${renamed.length}` : '') +
|
|
387
612
|
(result.failed ? `, failed ${result.failed}` : '');
|
|
388
613
|
console.log(result.failed ? kleur.yellow(`done with errors. ${summary}`) : kleur.green(`✓ done. ${summary}`));
|
|
389
|
-
|
|
614
|
+
// Per-file change list. Suppress renames here — printRenames below owns
|
|
615
|
+
// the rename block (it adds the import-update hints). Letting both fire
|
|
616
|
+
// would print each rename twice.
|
|
617
|
+
printChanges(result, ' ', { includeRenames: renamed.length === 0 });
|
|
390
618
|
if (renamed.length > 0)
|
|
391
619
|
printRenames(renamed, { withHints: true });
|
|
392
620
|
}
|
|
@@ -395,14 +623,16 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
395
623
|
return result;
|
|
396
624
|
}
|
|
397
625
|
const CHANGE_PRINT_CAP = 50;
|
|
398
|
-
function printChanges(r, indent) {
|
|
626
|
+
function printChanges(r, indent, opts = { includeRenames: true }) {
|
|
399
627
|
const lines = [];
|
|
400
628
|
for (const k of r.added)
|
|
401
629
|
lines.push(`${indent}${kleur.green('+')} ${k}`);
|
|
402
630
|
for (const k of r.modified)
|
|
403
631
|
lines.push(`${indent}${kleur.cyan('~')} ${k}`);
|
|
404
|
-
|
|
405
|
-
|
|
632
|
+
if (opts.includeRenames) {
|
|
633
|
+
for (const r2 of r.renamed)
|
|
634
|
+
lines.push(`${indent}${kleur.cyan('↪')} ${r2.oldKey} → ${r2.newKey}`);
|
|
635
|
+
}
|
|
406
636
|
for (const k of r.removed)
|
|
407
637
|
lines.push(`${indent}${kleur.red('-')} ${k}`);
|
|
408
638
|
if (lines.length === 0)
|
|
@@ -437,13 +667,6 @@ function progressText(done, total, bytes) {
|
|
|
437
667
|
const bar = '█'.repeat(filled) + '░'.repeat(barWidth - filled);
|
|
438
668
|
return `${bar} ${done}/${total} ${kleur.dim(formatBytes(bytes))}`;
|
|
439
669
|
}
|
|
440
|
-
function formatBytes(n) {
|
|
441
|
-
if (n < 1024)
|
|
442
|
-
return `${n} B`;
|
|
443
|
-
if (n < 1024 * 1024)
|
|
444
|
-
return `${(n / 1024).toFixed(1)} KB`;
|
|
445
|
-
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
446
|
-
}
|
|
447
670
|
function timestamp() {
|
|
448
671
|
const d = new Date();
|
|
449
672
|
return `[${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}:${String(d.getSeconds()).padStart(2, '0')}]`;
|
|
@@ -462,68 +685,7 @@ function printOrphans(orphans) {
|
|
|
462
685
|
for (const p of orphans) {
|
|
463
686
|
console.log(` ${kleur.dim('?')} ${relative(process.cwd(), p)}`);
|
|
464
687
|
}
|
|
465
|
-
console.log(kleur.dim('
|
|
466
|
-
}
|
|
467
|
-
async function findLocalPngs(dir) {
|
|
468
|
-
if (!existsSync(dir))
|
|
469
|
-
return [];
|
|
470
|
-
const root = resolve(dir);
|
|
471
|
-
const out = [];
|
|
472
|
-
async function walk(d) {
|
|
473
|
-
const entries = await readdir(d, { withFileTypes: true });
|
|
474
|
-
for (const ent of entries) {
|
|
475
|
-
if (ent.isSymbolicLink())
|
|
476
|
-
continue;
|
|
477
|
-
const full = resolve(d, ent.name);
|
|
478
|
-
try {
|
|
479
|
-
assertPathInsideRoot(full, root, 'outDir');
|
|
480
|
-
}
|
|
481
|
-
catch {
|
|
482
|
-
continue;
|
|
483
|
-
}
|
|
484
|
-
if (ent.isDirectory()) {
|
|
485
|
-
if (ent.name.startsWith('.'))
|
|
486
|
-
continue;
|
|
487
|
-
await walk(full);
|
|
488
|
-
}
|
|
489
|
-
else if (ent.isFile() && ent.name.toLowerCase().endsWith('.png')) {
|
|
490
|
-
out.push(full);
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
await walk(root);
|
|
495
|
-
return out;
|
|
496
|
-
}
|
|
497
|
-
async function pruneEmptyDirs(root) {
|
|
498
|
-
if (!existsSync(root))
|
|
499
|
-
return;
|
|
500
|
-
async function walk(d) {
|
|
501
|
-
const entries = await readdir(d, { withFileTypes: true });
|
|
502
|
-
let isEmpty = true;
|
|
503
|
-
for (const ent of entries) {
|
|
504
|
-
const full = resolve(d, ent.name);
|
|
505
|
-
if (ent.isDirectory()) {
|
|
506
|
-
const childEmpty = await walk(full);
|
|
507
|
-
if (childEmpty) {
|
|
508
|
-
await rm(full, { recursive: true, force: true });
|
|
509
|
-
}
|
|
510
|
-
else {
|
|
511
|
-
isEmpty = false;
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
else {
|
|
515
|
-
isEmpty = false;
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
return isEmpty;
|
|
519
|
-
}
|
|
520
|
-
const entries = await readdir(root, { withFileTypes: true });
|
|
521
|
-
for (const ent of entries) {
|
|
522
|
-
if (ent.isDirectory()) {
|
|
523
|
-
const full = resolve(root, ent.name);
|
|
524
|
-
const empty = await walk(full);
|
|
525
|
-
if (empty)
|
|
526
|
-
await rm(full, { recursive: true, force: true });
|
|
527
|
-
}
|
|
528
|
-
}
|
|
688
|
+
console.log(kleur.dim(' You passed --no-prune, so these were kept. Remove the flag to delete them.'));
|
|
529
689
|
}
|
|
690
|
+
// Note: orphan scan delegates to `walkOutDirPngs` in util/paths.ts (shared
|
|
691
|
+
// with the typed-index emitter). `maxIsoTimestamp` lives in util/iso.ts.
|