@magicpixelart/cli 0.5.13 → 0.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -0
- package/README.md +9 -8
- package/dist/api.js +124 -8
- package/dist/commands/connect.js +10 -11
- package/dist/commands/init.js +3 -0
- package/dist/commands/push.js +89 -47
- package/dist/commands/start.js +2 -3
- package/dist/commands/sync.js +331 -84
- package/dist/index.js +3 -3
- package/dist/util/engineConnect.js +32 -0
- package/dist/util/gameScan.js +12 -1
- package/dist/util/localEdit.js +29 -0
- package/dist/util/pullDecision.js +45 -0
- package/dist/util/reconcile.js +23 -0
- package/dist/util/security.js +3 -1
- package/dist/util/syncPath.js +31 -0
- package/dist/util/unityFilter.js +54 -2
- package/dist/util/watchCopy.js +1 -1
- package/package.json +1 -1
package/dist/commands/sync.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import kleur from 'kleur';
|
|
2
2
|
import ora from 'ora';
|
|
3
|
-
import { createHash } from 'node:crypto';
|
|
4
3
|
import { mkdir, unlink, readdir, rm, writeFile } from 'node:fs/promises';
|
|
5
4
|
import { existsSync } from 'node:fs';
|
|
6
5
|
import { dirname, relative, resolve } from 'node:path';
|
|
7
6
|
import { loadConfig, loadState, saveState } from '../config.js';
|
|
8
|
-
import {
|
|
7
|
+
import { ensureEngineConnect } from '../util/engineConnect.js';
|
|
8
|
+
import { fetchManifestSnapshot, fetchAssetBytes, ApiError, getLastProjectInfo, primeManifestEtags, manifestEtagsSnapshot } from '../api.js';
|
|
9
9
|
import { fileSha256 } from '../util/hash.js';
|
|
10
10
|
import { pruneEmptyDirs, walkOutDirPngs } from '../util/paths.js';
|
|
11
11
|
import { createLimit } from '../util/limit.js';
|
|
@@ -13,9 +13,9 @@ import { emitTypedIndex, ensureAgentsDoc, scanDiskAssets } from '../util/emitInd
|
|
|
13
13
|
import { assertPathInsideRoot, assertSafeIoPath } from '../util/security.js';
|
|
14
14
|
import { detectProjectKind, isEngineKind } from '../util/framework.js';
|
|
15
15
|
import { indexGamePngs, matchConnectGlobs, GAME_INDEX_CAP_HINT, connectCapMessage, countingSpritesText } from '../util/gameScan.js';
|
|
16
|
-
import { collectSourceRelMap, isPathInside, syncDiskPathFromKey } from '../util/syncPath.js';
|
|
16
|
+
import { aliasCollisionKeys, collectSourceRelMap, isPathInside, syncDiskPathFromKey } from '../util/syncPath.js';
|
|
17
17
|
import { DEFAULT_UNITY_PPU, writeMissingUnityMetas } from '../util/unityMeta.js';
|
|
18
|
-
import {
|
|
18
|
+
import { applyUnityPullPolicy, isWorkingSetEntry, workingSetPullKeys } from '../util/unityFilter.js';
|
|
19
19
|
import { selectFullSyncOrphans } from '../util/prunePolicy.js';
|
|
20
20
|
import { runTmpJanitor } from '../util/tmpJanitor.js';
|
|
21
21
|
import { friendlyFsError } from '../util/errors.js';
|
|
@@ -24,7 +24,28 @@ import { formatBytes } from '../util/format.js';
|
|
|
24
24
|
import { computePreviousKeyOrphans } from '../util/previousKeyOrphans.js';
|
|
25
25
|
import { cmd } from '../util/invoke.js';
|
|
26
26
|
import { formatWatchSpriteLine } from '../util/watchCopy.js';
|
|
27
|
+
import { decidePull } from '../util/pullDecision.js';
|
|
28
|
+
import { hasUnpushedLocalEdit } from '../util/localEdit.js';
|
|
29
|
+
import { shouldReconcile } from '../util/reconcile.js';
|
|
27
30
|
import { runPush } from './push.js';
|
|
31
|
+
/** Did the push half of a run actually move bytes into MagicPixel? */
|
|
32
|
+
export function pushWasActive(pushed) {
|
|
33
|
+
if (!pushed)
|
|
34
|
+
return false;
|
|
35
|
+
return pushed.created > 0 || pushed.updated > 0 || pushed.imported > 0;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Two consumers need the local PNG listing: the full-sync orphan sweep (no
|
|
39
|
+
* `since`) and the sha-based rename fallback (incremental, and only when
|
|
40
|
+
* something is download-bound). On an idle incremental tick — the common case
|
|
41
|
+
* now that unchanged manifests come back as a bodyless 304 — neither runs, so
|
|
42
|
+
* walking the whole output tree would be pure waste.
|
|
43
|
+
*
|
|
44
|
+
* Full syncs must ALWAYS walk: skipping there would silently disable pruning.
|
|
45
|
+
*/
|
|
46
|
+
export function needsLocalPngWalk(since, toDownloadCount) {
|
|
47
|
+
return !since || toDownloadCount > 0;
|
|
48
|
+
}
|
|
28
49
|
export async function syncCommand(opts) {
|
|
29
50
|
// Sweep any leaked `<file>.<pid>.<hex>.tmp` files left behind by a prior
|
|
30
51
|
// crashed/killed CLI run before they pile up. Runs once per CLI invocation
|
|
@@ -32,10 +53,16 @@ export async function syncCommand(opts) {
|
|
|
32
53
|
// never race a concurrent in-flight write. See util/tmpJanitor.ts.
|
|
33
54
|
try {
|
|
34
55
|
const cfg = await loadConfig();
|
|
35
|
-
|
|
56
|
+
try {
|
|
57
|
+
await runTmpJanitor(cfg.outDir);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
/* janitor is best-effort — never let it block a sync */
|
|
61
|
+
}
|
|
62
|
+
await ensureEngineConnect(cfg, process.cwd(), !opts.dryRun);
|
|
36
63
|
}
|
|
37
64
|
catch {
|
|
38
|
-
/*
|
|
65
|
+
/* missing/invalid config is reported by runOnce / watchLoop */
|
|
39
66
|
}
|
|
40
67
|
if (opts.watch) {
|
|
41
68
|
await watchLoop(opts);
|
|
@@ -94,17 +121,24 @@ async function watchLoop(opts) {
|
|
|
94
121
|
// genuinely broken (revoked key) rather than transiently blipped.
|
|
95
122
|
const MAX_AUTH_FAILURES = 5;
|
|
96
123
|
let consecutiveAuthFailures = 0;
|
|
97
|
-
// Adaptive idle backoff:
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
|
|
107
|
-
|
|
124
|
+
// Adaptive idle backoff: the fast interval is a "hot window" around actual
|
|
125
|
+
// work, not a steady state. After ~1 min of nothing-to-do we slow the poll
|
|
126
|
+
// from intervalSec → 5s, and after ~5 min → 10s, so a dev who walked away
|
|
127
|
+
// isn't polling the manifest hundreds of times an hour. ANY change (pulled
|
|
128
|
+
// OR pushed) or error resets this, so the "edit → see it" promise stays
|
|
129
|
+
// intact the moment the user comes back. Error backoff (2→60s) is separate
|
|
130
|
+
// and continues to win.
|
|
131
|
+
//
|
|
132
|
+
// Idle is measured in wall-clock seconds since the last change, not in
|
|
133
|
+
// ticks: once we've backed off, a "tick" is 5s or 10s long, so counting
|
|
134
|
+
// ticks × intervalSec would undercount elapsed time and push the 5-minute
|
|
135
|
+
// step far past 5 minutes.
|
|
136
|
+
let lastChangeAt = Date.now();
|
|
137
|
+
// Re-walk the game tree on this cadence so new PNGs are ingested without
|
|
138
|
+
// listing thousands of folders on every 2s tick.
|
|
139
|
+
const GAME_INDEX_TTL_MS = 30_000;
|
|
140
|
+
let gameIndexCache = header.gameIndex;
|
|
141
|
+
let gameIndexAt = Date.now();
|
|
108
142
|
// Cancellable idle sleep — `onStopSignal` calls `wakeStop()` so the next
|
|
109
143
|
// tick exits immediately instead of waiting out the full backoff (which can
|
|
110
144
|
// be 60s at the error ceiling). Initialised to a no-op so the first
|
|
@@ -119,9 +153,11 @@ async function watchLoop(opts) {
|
|
|
119
153
|
stopping = true;
|
|
120
154
|
wakeStop();
|
|
121
155
|
process.stdout.write('\x1b[2K\r');
|
|
156
|
+
const sig = signal === 'SIGTERM' ? 'SIGTERM' : 'SIGINT';
|
|
122
157
|
if (inFlight) {
|
|
123
|
-
|
|
124
|
-
|
|
158
|
+
const again = sig === 'SIGINT' ? 'Ctrl+C' : 'SIGTERM';
|
|
159
|
+
console.log(kleur.dim(`Still finishing this check… (${again} again to stop now)`));
|
|
160
|
+
process.once(sig, () => process.exit(sig === 'SIGINT' ? 130 : 143));
|
|
125
161
|
}
|
|
126
162
|
else {
|
|
127
163
|
if (!opts.quiet)
|
|
@@ -143,7 +179,18 @@ async function watchLoop(opts) {
|
|
|
143
179
|
};
|
|
144
180
|
try {
|
|
145
181
|
onStatus('Checking MagicPixel…');
|
|
146
|
-
const
|
|
182
|
+
const reuseIndex = gameIndexCache && Date.now() - gameIndexAt < GAME_INDEX_TTL_MS
|
|
183
|
+
? gameIndexCache
|
|
184
|
+
: undefined;
|
|
185
|
+
const r = await runOnce({ ...opts, watch: false }, {
|
|
186
|
+
watchMode: true,
|
|
187
|
+
onStatus,
|
|
188
|
+
gameIndex: reuseIndex,
|
|
189
|
+
onGameIndex: (idx) => {
|
|
190
|
+
gameIndexCache = idx;
|
|
191
|
+
gameIndexAt = Date.now();
|
|
192
|
+
},
|
|
193
|
+
});
|
|
147
194
|
// Reset backoff on any successful tick. Resume message fires once when
|
|
148
195
|
// we recover from an auth pause — `getApiKey()` re-reads
|
|
149
196
|
// .magicpixel/credentials on every call, so a `magicpixel login` in
|
|
@@ -163,17 +210,16 @@ async function watchLoop(opts) {
|
|
|
163
210
|
console.log(`${kleur.dim(timestamp())} ${kleur.green('✓')} Back online — resuming.`);
|
|
164
211
|
}
|
|
165
212
|
const changedCount = r.added.length + r.modified.length + r.removed.length + r.renamed.length;
|
|
166
|
-
|
|
213
|
+
// Pushing a locally edited sprite is activity too — it just isn't part of
|
|
214
|
+
// the printed pull list. Without it, an artist working only in Unity
|
|
215
|
+
// would slide into the 10s idle poll while actively saving files.
|
|
216
|
+
if (changedCount > 0 || pushWasActive(r.pushed)) {
|
|
167
217
|
// Snap back to the fast interval the moment anything changes.
|
|
168
|
-
|
|
218
|
+
lastChangeAt = Date.now();
|
|
169
219
|
backoffSec = intervalSec;
|
|
170
220
|
}
|
|
171
221
|
else {
|
|
172
|
-
|
|
173
|
-
backoffSec = nextBackoffForIdle(consecutiveIdleTicks, intervalSec, {
|
|
174
|
-
softSec: IDLE_SOFT_BACKOFF_SECONDS,
|
|
175
|
-
hardSec: IDLE_HARD_BACKOFF_SECONDS,
|
|
176
|
-
});
|
|
222
|
+
backoffSec = nextBackoffForIdle((Date.now() - lastChangeAt) / 1000, intervalSec);
|
|
177
223
|
}
|
|
178
224
|
if (opts.quiet)
|
|
179
225
|
return;
|
|
@@ -194,7 +240,7 @@ async function watchLoop(opts) {
|
|
|
194
240
|
const firstLine = err.message.split('\n')[0];
|
|
195
241
|
process.stdout.write('\x1b[2K\r');
|
|
196
242
|
// Any error breaks the idle streak so we come back fast once it clears.
|
|
197
|
-
|
|
243
|
+
lastChangeAt = Date.now();
|
|
198
244
|
const decision = classifyTickError(err, {
|
|
199
245
|
backoffSec,
|
|
200
246
|
pausedForAuth,
|
|
@@ -242,7 +288,6 @@ async function watchLoop(opts) {
|
|
|
242
288
|
}
|
|
243
289
|
}
|
|
244
290
|
finally {
|
|
245
|
-
header.gameIndex = undefined;
|
|
246
291
|
inFlight = false;
|
|
247
292
|
}
|
|
248
293
|
};
|
|
@@ -287,13 +332,20 @@ function isNetworkError(err) {
|
|
|
287
332
|
/\b(502|504)\b.*\b(bad gateway|gateway timeout)\b/i.test(msg));
|
|
288
333
|
}
|
|
289
334
|
/**
|
|
290
|
-
*
|
|
291
|
-
* configured
|
|
292
|
-
* the watch
|
|
293
|
-
|
|
335
|
+
* Idle thresholds (seconds of consecutive idleness) at which the watcher steps
|
|
336
|
+
* down from the configured interval to 5s, then to 10s. Single source of truth
|
|
337
|
+
* for both the watch loop and its regression tests.
|
|
338
|
+
*/
|
|
339
|
+
export const IDLE_BACKOFF_THRESHOLDS = { softSec: 60, hardSec: 300 };
|
|
340
|
+
/**
|
|
341
|
+
* Pure helper: given how long we've been idle (wall-clock seconds since the
|
|
342
|
+
* last pulled or pushed change) and the configured poll interval, return the
|
|
343
|
+
* next backoff in seconds. Exported so the watch-mode regression test can guard
|
|
344
|
+
* the "ticks vs seconds" bug fixed in 0.4.0 without exercising the full watch
|
|
345
|
+
* loop. Callers pass elapsed time, not tick counts: once the watcher has backed
|
|
346
|
+
* off, ticks are longer than `intervalSec`.
|
|
294
347
|
*/
|
|
295
|
-
export function nextBackoffForIdle(
|
|
296
|
-
const idleSeconds = consecutiveIdleTicks * intervalSec;
|
|
348
|
+
export function nextBackoffForIdle(idleSeconds, intervalSec, thresholds = IDLE_BACKOFF_THRESHOLDS) {
|
|
297
349
|
if (idleSeconds >= thresholds.hardSec)
|
|
298
350
|
return Math.max(intervalSec, 10);
|
|
299
351
|
if (idleSeconds >= thresholds.softSec)
|
|
@@ -354,6 +406,11 @@ async function loadWatchHeader(onProgress) {
|
|
|
354
406
|
async function runOnce(opts, runOpts = {}) {
|
|
355
407
|
const config = await loadConfig();
|
|
356
408
|
const state = await loadState();
|
|
409
|
+
// Seed manifest validators persisted by the previous run so a one-shot sync
|
|
410
|
+
// can answer with a 304 on its very first poll (watchers already keep them
|
|
411
|
+
// in memory). A `--full` run intentionally sends no `since`, which is a
|
|
412
|
+
// different validator key, so it can't be short-circuited by a stale entry.
|
|
413
|
+
primeManifestEtags(state.manifestEtags);
|
|
357
414
|
const startedAt = new Date().toISOString();
|
|
358
415
|
const concurrency = Math.max(1, Math.min(opts.concurrency ?? 6, 16));
|
|
359
416
|
const verbose = !opts.quiet && !runOpts.watchMode;
|
|
@@ -363,7 +420,13 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
363
420
|
if (verbose && config.endpoint) {
|
|
364
421
|
console.log(kleur.yellow(`! using custom endpoint: ${config.endpoint}`));
|
|
365
422
|
}
|
|
366
|
-
|
|
423
|
+
// Incremental unless asked for a full run — or unless a full pass is due:
|
|
424
|
+
// permanently-deleted rows can't be reported incrementally (see reconcile.ts).
|
|
425
|
+
const reconcileDue = !opts.full && !!state.lastSync && shouldReconcile(state.lastReconcile, Date.now());
|
|
426
|
+
let since = opts.full || reconcileDue ? undefined : state.lastSync;
|
|
427
|
+
if (reconcileDue && verbose) {
|
|
428
|
+
console.log(kleur.dim('Running a full reconcile (periodic) to clear anything deleted in MagicPixel.'));
|
|
429
|
+
}
|
|
367
430
|
const previousAssets = state.assets ?? {}; // id → key from prior sync
|
|
368
431
|
const spinner = verbose
|
|
369
432
|
? ora(since ? `Fetching manifest since ${humanTime(since)}…` : 'Fetching manifest…').start()
|
|
@@ -371,8 +434,27 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
371
434
|
if (!spinner)
|
|
372
435
|
onStatus?.(since ? 'Checking MagicPixel for edits…' : 'Fetching your sprites from MagicPixel…');
|
|
373
436
|
let manifest;
|
|
437
|
+
// Composite keys the cloud reports as gone (document trashed / componentized)
|
|
438
|
+
// since our cursor. Empty on --full, where the manifest itself is the truth.
|
|
439
|
+
let removedRemoteKeys = [];
|
|
440
|
+
// Capture before the Unity filter drops rows — lastSync must not jump past
|
|
441
|
+
// an editor save whose `unity` flag was omitted on this tick.
|
|
442
|
+
let observedUpdatedAt = null;
|
|
374
443
|
try {
|
|
375
|
-
|
|
444
|
+
let snapshot = await fetchManifestSnapshot(config, since);
|
|
445
|
+
if (snapshot.removalsTruncated && since) {
|
|
446
|
+
// The server couldn't list every removal in this window. Trusting the
|
|
447
|
+
// partial list and advancing the cursor would strand the rest forever, so
|
|
448
|
+
// re-read the whole manifest and let the orphan sweep decide.
|
|
449
|
+
if (verbose) {
|
|
450
|
+
console.log(kleur.dim('Many sprites were removed at once — re-reading the full manifest to stay in sync.'));
|
|
451
|
+
}
|
|
452
|
+
since = undefined;
|
|
453
|
+
snapshot = await fetchManifestSnapshot(config, undefined);
|
|
454
|
+
}
|
|
455
|
+
manifest = snapshot.entries;
|
|
456
|
+
removedRemoteKeys = snapshot.removedKeys;
|
|
457
|
+
observedUpdatedAt = maxIsoTimestamp(manifest.map((e) => e.updated_at));
|
|
376
458
|
const projectInfo = getLastProjectInfo();
|
|
377
459
|
const projectSuffix = projectInfo && verbose
|
|
378
460
|
? kleur.dim(` · project: ${projectInfo.name ?? projectInfo.id.slice(0, 8)}`)
|
|
@@ -421,8 +503,11 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
421
503
|
},
|
|
422
504
|
})
|
|
423
505
|
: { files: [], capped: false });
|
|
506
|
+
if (!runOpts.gameIndex)
|
|
507
|
+
runOpts.onGameIndex?.(gameIndex);
|
|
424
508
|
const connected = matchConnectGlobs(gameIndex, config.connect ?? []);
|
|
425
509
|
const sourceByKey = collectSourceRelMap(connected.entries, state.synced);
|
|
510
|
+
aliasCollisionKeys(sourceByKey, manifest.map((e) => e.key));
|
|
426
511
|
if (verbose && gameIndex.capped) {
|
|
427
512
|
console.log(kleur.yellow(`! ${GAME_INDEX_CAP_HINT}`));
|
|
428
513
|
}
|
|
@@ -434,6 +519,7 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
434
519
|
(connected.capped ? ` of ${connected.total} matched` : '')));
|
|
435
520
|
}
|
|
436
521
|
const pathForKey = (key) => syncDiskPathFromKey(config.outDir, key, sourceByKey);
|
|
522
|
+
const workingSet = workingSetPullKeys(sourceByKey, state.synced);
|
|
437
523
|
// Files whose artboard is no longer flagged for Unity. Deleted even in
|
|
438
524
|
// incremental mode: un-checking the box in the editor must actually remove
|
|
439
525
|
// the sprite from the game project, not leave a stale copy behind.
|
|
@@ -444,14 +530,25 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
444
530
|
// response must never delete art from someone's game project.
|
|
445
531
|
const unknownFlagPaths = new Set();
|
|
446
532
|
if (projectKind === 'Unity') {
|
|
447
|
-
const filtered =
|
|
533
|
+
const filtered = applyUnityPullPolicy(manifest, {
|
|
534
|
+
syncAll: config.unitySyncAll,
|
|
535
|
+
alwaysPull: (entry) => isWorkingSetEntry(entry, workingSet),
|
|
536
|
+
});
|
|
448
537
|
if (filtered.unknown.length > 0) {
|
|
449
|
-
|
|
450
|
-
|
|
538
|
+
const n = filtered.unknown.length;
|
|
539
|
+
const msg = `${n} artboard${n === 1 ? '' : 's'} came back without a "Sync to Unity" flag — skipped (existing files kept).`;
|
|
540
|
+
if (verbose) {
|
|
541
|
+
console.log(kleur.yellow(`! ${msg}`));
|
|
542
|
+
console.log(kleur.dim(` Fix: re-run \`${cmd('sync')}\` in a moment, or upgrade with \`npm i -D @magicpixelart/cli@latest\`.`));
|
|
543
|
+
}
|
|
544
|
+
else {
|
|
545
|
+
onStatus?.(msg);
|
|
546
|
+
}
|
|
451
547
|
}
|
|
452
548
|
const unknownSet = new Set(filtered.unknown);
|
|
549
|
+
const pullSet = new Set(filtered.entries);
|
|
453
550
|
for (const entry of manifest) {
|
|
454
|
-
if (
|
|
551
|
+
if (pullSet.has(entry))
|
|
455
552
|
continue;
|
|
456
553
|
const p = pathForKey(entry.key);
|
|
457
554
|
if (unknownSet.has(entry)) {
|
|
@@ -481,8 +578,20 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
481
578
|
bytesIn: 0,
|
|
482
579
|
bytesSaved: 0,
|
|
483
580
|
renamed: [],
|
|
581
|
+
conflicts: [],
|
|
484
582
|
};
|
|
485
|
-
|
|
583
|
+
// Persist validators even when the cursor didn't move, so the next
|
|
584
|
+
// one-shot run starts from a conditional GET.
|
|
585
|
+
const lastSync = observedUpdatedAt && !(state.lastSync && state.lastSync > observedUpdatedAt)
|
|
586
|
+
? observedUpdatedAt
|
|
587
|
+
: state.lastSync;
|
|
588
|
+
await saveState({
|
|
589
|
+
...state,
|
|
590
|
+
...(lastSync ? { lastSync } : {}),
|
|
591
|
+
...(since ? {} : { lastReconcile: startedAt }),
|
|
592
|
+
manifestEtags: manifestEtagsSnapshot(),
|
|
593
|
+
});
|
|
594
|
+
await maybePushLocalSprites(config.push, verbose, gameIndex, live && !!runOpts.watchMode, onStatus);
|
|
486
595
|
return empty;
|
|
487
596
|
}
|
|
488
597
|
const skipped = manifest.length - filtered.entries.length;
|
|
@@ -524,12 +633,61 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
524
633
|
const diffLimit = createLimit(concurrency);
|
|
525
634
|
const shaByEntryId = new Map();
|
|
526
635
|
const toDownload = [];
|
|
636
|
+
const conflicts = [];
|
|
527
637
|
let bytesSaved = 0;
|
|
528
638
|
let unchanged = 0;
|
|
639
|
+
const synced = state.synced ?? {};
|
|
640
|
+
const cloudShaByAssetId = new Map();
|
|
641
|
+
for (const s of Object.values(synced)) {
|
|
642
|
+
if (!s.assetId || !s.sha256)
|
|
643
|
+
continue;
|
|
644
|
+
const prev = cloudShaByAssetId.get(s.assetId);
|
|
645
|
+
if (prev && prev !== s.sha256)
|
|
646
|
+
cloudShaByAssetId.set(s.assetId, 'ambiguous');
|
|
647
|
+
else if (!prev)
|
|
648
|
+
cloudShaByAssetId.set(s.assetId, s.sha256);
|
|
649
|
+
}
|
|
650
|
+
const previousCloudSha = (entry) => {
|
|
651
|
+
const direct = synced[entry.key]?.sha256;
|
|
652
|
+
if (direct)
|
|
653
|
+
return direct;
|
|
654
|
+
for (const k of entry.previous_keys ?? []) {
|
|
655
|
+
const sha = synced[k]?.sha256;
|
|
656
|
+
if (sha)
|
|
657
|
+
return sha;
|
|
658
|
+
}
|
|
659
|
+
if (entry.asset_id) {
|
|
660
|
+
const sha = cloudShaByAssetId.get(entry.asset_id);
|
|
661
|
+
if (sha && sha !== 'ambiguous')
|
|
662
|
+
return sha;
|
|
663
|
+
}
|
|
664
|
+
return undefined;
|
|
665
|
+
};
|
|
529
666
|
await Promise.all(manifest.map((entry) => diffLimit(async () => {
|
|
667
|
+
const previousCloudSha256 = previousCloudSha(entry);
|
|
668
|
+
const inWorkingSet = isWorkingSetEntry(entry, workingSet);
|
|
669
|
+
// Unchanged cloud composite: skip hashing the original PNG (it almost
|
|
670
|
+
// never matches) unless the file is gone and we need to restore it.
|
|
671
|
+
if (entry.sha256 && previousCloudSha256 === entry.sha256 && existsSync(pathFor(entry))) {
|
|
672
|
+
shaByEntryId.set(entry.id, null);
|
|
673
|
+
unchanged++;
|
|
674
|
+
if (entry.size_bytes)
|
|
675
|
+
bytesSaved += entry.size_bytes;
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
530
678
|
const localSha = await fileSha256(pathFor(entry));
|
|
531
679
|
shaByEntryId.set(entry.id, localSha);
|
|
532
|
-
|
|
680
|
+
const decision = decidePull({
|
|
681
|
+
cloudSha256: entry.sha256,
|
|
682
|
+
localSha256: localSha,
|
|
683
|
+
previousCloudSha256,
|
|
684
|
+
lastPushedDiskSha256: synced[entry.key]?.diskSha256,
|
|
685
|
+
inWorkingSet,
|
|
686
|
+
});
|
|
687
|
+
if (decision === 'conflict') {
|
|
688
|
+
conflicts.push(entry.key);
|
|
689
|
+
}
|
|
690
|
+
else if (decision === 'skip') {
|
|
533
691
|
unchanged++;
|
|
534
692
|
if (entry.size_bytes)
|
|
535
693
|
bytesSaved += entry.size_bytes;
|
|
@@ -542,30 +700,32 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
542
700
|
// (Renames also produce a stale path on disk — collected separately below.)
|
|
543
701
|
let orphans = [];
|
|
544
702
|
const remoteDiskPaths = new Set(manifest.map((e) => pathFor(e)));
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
703
|
+
const remoteKeySet = new Set(manifest.map((e) => e.key));
|
|
704
|
+
const previousKeySet = new Set(Object.values(previousAssets));
|
|
705
|
+
const localPngs = needsLocalPngWalk(since, toDownload.length)
|
|
706
|
+
? await walkOutDirPngs(config.outDir)
|
|
707
|
+
: [];
|
|
708
|
+
// Every key we have ever synced, indexed by the disk path it maps to. Used
|
|
709
|
+
// both for orphan tracking and to look up a prune candidate's edit baseline.
|
|
710
|
+
const keyByPath = new Map();
|
|
711
|
+
for (const key of Object.keys(state.synced ?? {}).concat(Object.values(previousAssets))) {
|
|
712
|
+
try {
|
|
713
|
+
const p = pathForKey(key);
|
|
714
|
+
if (!keyByPath.has(p))
|
|
715
|
+
keyByPath.set(p, key);
|
|
716
|
+
}
|
|
717
|
+
catch {
|
|
718
|
+
/* unusable key — nothing to track */
|
|
719
|
+
}
|
|
720
|
+
}
|
|
550
721
|
// Sprites created in Unity (or any PNG we never pulled) are pending a
|
|
551
722
|
// push into MagicPixel, not orphans — see prunePolicy.ts.
|
|
552
723
|
if (!since) {
|
|
553
|
-
const trackedPaths = new Set(Object.keys(state.synced ?? {})
|
|
554
|
-
.concat(Object.values(previousAssets))
|
|
555
|
-
.map((key) => {
|
|
556
|
-
try {
|
|
557
|
-
return pathForKey(key);
|
|
558
|
-
}
|
|
559
|
-
catch {
|
|
560
|
-
return '';
|
|
561
|
-
}
|
|
562
|
-
})
|
|
563
|
-
.filter(Boolean));
|
|
564
724
|
const policy = selectFullSyncOrphans({
|
|
565
725
|
localPaths: localPngs.map((a) => a.abs),
|
|
566
726
|
remoteDiskPaths,
|
|
567
727
|
protectedPaths: unknownFlagPaths,
|
|
568
|
-
isTracked: (p) =>
|
|
728
|
+
isTracked: (p) => keyByPath.has(p),
|
|
569
729
|
});
|
|
570
730
|
orphans = policy.orphans;
|
|
571
731
|
}
|
|
@@ -625,15 +785,20 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
625
785
|
}
|
|
626
786
|
})
|
|
627
787
|
.filter((p) => p && existsSync(p) && isPathInside(p, outRoot));
|
|
628
|
-
//
|
|
788
|
+
// Every prune candidate goes through `addOrphan`, so no source can bypass the
|
|
789
|
+
// local-edit guard applied below. De-duplicates vs the full-sync orphan list.
|
|
629
790
|
const orphanSet = new Set(orphans);
|
|
791
|
+
const addOrphan = (p) => {
|
|
792
|
+
if (existsSync(p) && isPathInside(p, outRoot))
|
|
793
|
+
orphanSet.add(p);
|
|
794
|
+
};
|
|
630
795
|
for (const p of renameStalePaths)
|
|
631
|
-
|
|
796
|
+
addOrphan(p);
|
|
632
797
|
// Un-checked "Sync to Unity" artboards: their PNG (and .meta) leaves the
|
|
633
798
|
// Unity project on the next sync, incremental or not. Working-set originals
|
|
634
799
|
// are never in this list.
|
|
635
800
|
for (const p of deselectedPaths)
|
|
636
|
-
|
|
801
|
+
addOrphan(p);
|
|
637
802
|
// Server-side rename history. Each manifest entry carries the composite
|
|
638
803
|
// keys this row was previously emitted under (populated by the editor when
|
|
639
804
|
// a doc/artboard is renamed). For each prior key whose file is still on
|
|
@@ -646,16 +811,58 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
646
811
|
resolveDiskPath: (key) => pathForKey(key),
|
|
647
812
|
fileExists: existsSync,
|
|
648
813
|
});
|
|
649
|
-
for (const p of prevKeyResult.orphanPaths)
|
|
650
|
-
|
|
651
|
-
orphanSet.add(p);
|
|
652
|
-
}
|
|
814
|
+
for (const p of prevKeyResult.orphanPaths)
|
|
815
|
+
addOrphan(p);
|
|
653
816
|
for (const r of prevKeyResult.renames) {
|
|
654
817
|
if (!renamed.some((x) => x.id === r.id && x.oldKey === r.oldKey)) {
|
|
655
818
|
renamed.push(r);
|
|
656
819
|
}
|
|
657
820
|
}
|
|
658
|
-
|
|
821
|
+
// Cloud-side removals (document trashed, or converted into a component
|
|
822
|
+
// master). These rows leave the manifest entirely, so an incremental run has
|
|
823
|
+
// no other way to learn about them — without this the PNG would live on in
|
|
824
|
+
// the game project forever. Only keys we pulled ourselves are eligible:
|
|
825
|
+
// a same-named sprite authored in Unity is pending push, not an orphan.
|
|
826
|
+
for (const key of removedRemoteKeys) {
|
|
827
|
+
if (remoteKeySet.has(key))
|
|
828
|
+
continue; // still live under another row
|
|
829
|
+
if (!synced[key] && !previousKeySet.has(key))
|
|
830
|
+
continue; // never ours
|
|
831
|
+
try {
|
|
832
|
+
addOrphan(pathForKey(key));
|
|
833
|
+
}
|
|
834
|
+
catch {
|
|
835
|
+
continue;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
// A locally-edited file is never deleted, whatever put it in the prune set
|
|
839
|
+
// (deselected "Sync to Unity", trashed document, rename fallback,
|
|
840
|
+
// previous-key sweep): that edit is the only copy of the work until the user
|
|
841
|
+
// pushes it or discards it. Manifest-resident conflicts were already detected
|
|
842
|
+
// during the diff, so they skip the re-hash.
|
|
843
|
+
const conflictPaths = new Set();
|
|
844
|
+
for (const key of conflicts) {
|
|
845
|
+
try {
|
|
846
|
+
conflictPaths.add(pathForKey(key));
|
|
847
|
+
}
|
|
848
|
+
catch {
|
|
849
|
+
/* unusable key — nothing to protect */
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
orphans = [];
|
|
853
|
+
const keptFromPrune = [];
|
|
854
|
+
for (const p of orphanSet) {
|
|
855
|
+
if (conflictPaths.has(p))
|
|
856
|
+
continue;
|
|
857
|
+
const key = keyByPath.get(p);
|
|
858
|
+
if (key &&
|
|
859
|
+
(await hasUnpushedLocalEdit({ absPath: p, baselineDiskSha256: synced[key]?.diskSha256 }))) {
|
|
860
|
+
keptFromPrune.push(key);
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
orphans.push(p);
|
|
864
|
+
}
|
|
865
|
+
conflicts.push(...keptFromPrune);
|
|
659
866
|
// Legacy-suffix folder sweep — always runs, even in incremental mode.
|
|
660
867
|
//
|
|
661
868
|
// Background: server-side slug uniqueness rules have changed over time
|
|
@@ -708,6 +915,12 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
708
915
|
}
|
|
709
916
|
console.log();
|
|
710
917
|
}
|
|
918
|
+
if (conflicts.length > 0) {
|
|
919
|
+
if (live) {
|
|
920
|
+
printConflicts(conflicts, new Set(keptFromPrune));
|
|
921
|
+
}
|
|
922
|
+
onStatus?.(`${conflicts.length} sprite${conflicts.length === 1 ? '' : 's'} changed in both places — kept your local copy.`);
|
|
923
|
+
}
|
|
711
924
|
if (opts.dryRun) {
|
|
712
925
|
if (verbose) {
|
|
713
926
|
console.log(kleur.dim('--dry-run: no files written.'));
|
|
@@ -727,6 +940,7 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
727
940
|
failed: 0,
|
|
728
941
|
bytesIn: 0,
|
|
729
942
|
bytesSaved,
|
|
943
|
+
conflicts,
|
|
730
944
|
};
|
|
731
945
|
}
|
|
732
946
|
const result = {
|
|
@@ -738,6 +952,7 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
738
952
|
failed: 0,
|
|
739
953
|
bytesIn: 0,
|
|
740
954
|
bytesSaved,
|
|
955
|
+
conflicts,
|
|
741
956
|
};
|
|
742
957
|
let progress = null;
|
|
743
958
|
if (live && toDownload.length > 0) {
|
|
@@ -768,14 +983,9 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
768
983
|
result.bytesSaved += entry.size_bytes;
|
|
769
984
|
}
|
|
770
985
|
else {
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
throw new Error(`sha256 mismatch (expected ${entry.sha256.slice(0, 12)}…, got ${actual.slice(0, 12)}…)`);
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
// Wrap any of mkdir/writeFile so the user sees a friendly
|
|
778
|
-
// multi-line diagnostic instead of a raw `EACCES: permission denied`.
|
|
986
|
+
// Do not require bytes to match `entry.sha256`. Single-artboard
|
|
987
|
+
// downloads serve storage_path (so 2048/4096 Unity tiles do not
|
|
988
|
+
// 546 the isolate); that PNG can differ from a cached composite hash.
|
|
779
989
|
//
|
|
780
990
|
// NOTE: Asset PNGs deliberately use a direct writeFile rather than
|
|
781
991
|
// atomicWrite. Vite's chokidar watcher only suppresses atomic-rename
|
|
@@ -944,9 +1154,15 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
944
1154
|
for (const key of result.removed)
|
|
945
1155
|
delete nextSynced[key];
|
|
946
1156
|
const writtenKeys = new Set([...result.added, ...result.modified]);
|
|
1157
|
+
const conflictSet = new Set(conflicts);
|
|
947
1158
|
for (const entry of manifest) {
|
|
948
1159
|
if (!entry.asset_id)
|
|
949
1160
|
continue;
|
|
1161
|
+
// Conflicted keys keep their old baselines: recording the new cloud sha
|
|
1162
|
+
// (with the locally-edited disk sha) would make both halves believe they
|
|
1163
|
+
// are in sync and freeze the divergence forever.
|
|
1164
|
+
if (conflictSet.has(entry.key))
|
|
1165
|
+
continue;
|
|
950
1166
|
const sha = entry.sha256 ?? (await fileSha256(pathFor(entry)));
|
|
951
1167
|
if (!sha)
|
|
952
1168
|
continue;
|
|
@@ -971,14 +1187,23 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
971
1187
|
...(sourceRel ? { sourceRel } : {}),
|
|
972
1188
|
};
|
|
973
1189
|
}
|
|
974
|
-
const nextState = {
|
|
975
|
-
|
|
1190
|
+
const nextState = {
|
|
1191
|
+
...state,
|
|
1192
|
+
assets: nextAssets,
|
|
1193
|
+
synced: nextSynced,
|
|
1194
|
+
manifestEtags: manifestEtagsSnapshot(),
|
|
1195
|
+
// A completed full pass is the only run that can prove nothing is orphaned.
|
|
1196
|
+
...(!since && result.failed === 0 ? { lastReconcile: startedAt } : {}),
|
|
1197
|
+
};
|
|
1198
|
+
// Unresolved conflicts hold the cursor, exactly like a failed download: the
|
|
1199
|
+
// next sync must see the same rows again and re-report them.
|
|
1200
|
+
if (result.failed === 0 && conflicts.length === 0) {
|
|
976
1201
|
// Advance the cursor to the newest row we actually observed, NOT the
|
|
977
1202
|
// wall-clock time the sync started. Using startedAt silently skipped
|
|
978
1203
|
// rows whose `updated_at` lived in the gap between the manifest snapshot
|
|
979
1204
|
// and the next poll (notably when only metadata changed — e.g. artboard
|
|
980
1205
|
// renames — and the row updated between fetch start and save).
|
|
981
|
-
const maxUpdatedAt = maxIsoTimestamp(manifest.map((e) => e.updated_at));
|
|
1206
|
+
const maxUpdatedAt = observedUpdatedAt ?? maxIsoTimestamp(manifest.map((e) => e.updated_at));
|
|
982
1207
|
if (maxUpdatedAt) {
|
|
983
1208
|
// Take the later of: newest row we saw, or the prior cursor. Never
|
|
984
1209
|
// rewind — an incremental sync that returned 0 rows must keep the
|
|
@@ -996,9 +1221,14 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
996
1221
|
}
|
|
997
1222
|
delete nextState.lastError;
|
|
998
1223
|
}
|
|
999
|
-
else {
|
|
1224
|
+
else if (result.failed > 0) {
|
|
1000
1225
|
nextState.lastError = `${result.failed} download${result.failed === 1 ? '' : 's'} failed at ${startedAt}`;
|
|
1001
1226
|
}
|
|
1227
|
+
else {
|
|
1228
|
+
nextState.lastError =
|
|
1229
|
+
`${conflicts.length} sprite${conflicts.length === 1 ? '' : 's'} changed both locally and in MagicPixel at ${startedAt}` +
|
|
1230
|
+
` (${conflicts.slice(0, 3).join(', ')}${conflicts.length > 3 ? ', …' : ''})`;
|
|
1231
|
+
}
|
|
1002
1232
|
// Emit typed index from the filesystem — never from the manifest. This
|
|
1003
1233
|
// guarantees the barrel can never disagree with what bundlers see on disk,
|
|
1004
1234
|
// and removes the silent-failure mode where a flaky manifest call left
|
|
@@ -1050,16 +1280,16 @@ async function runOnce(opts, runOpts = {}) {
|
|
|
1050
1280
|
printRenames(renamed, { withHints: true });
|
|
1051
1281
|
}
|
|
1052
1282
|
onStatus?.('Checking your game files…');
|
|
1053
|
-
await maybePushLocalSprites(config.push, verbose, gameIndex, live && !!runOpts.watchMode);
|
|
1283
|
+
const pushed = await maybePushLocalSprites(config.push, verbose, gameIndex, live && !!runOpts.watchMode, onStatus);
|
|
1054
1284
|
if (result.failed)
|
|
1055
1285
|
process.exitCode = 1;
|
|
1056
|
-
return result;
|
|
1286
|
+
return { ...result, pushed };
|
|
1057
1287
|
}
|
|
1058
|
-
async function maybePushLocalSprites(pushEnabled, verbose, gameIndex, announceErrors = verbose) {
|
|
1288
|
+
async function maybePushLocalSprites(pushEnabled, verbose, gameIndex, announceErrors = verbose, onStatus) {
|
|
1059
1289
|
if (pushEnabled === false)
|
|
1060
1290
|
return null;
|
|
1061
1291
|
try {
|
|
1062
|
-
return await runPush({ quiet: !verbose, gameIndex });
|
|
1292
|
+
return await runPush({ quiet: !verbose, gameIndex, onStatus });
|
|
1063
1293
|
}
|
|
1064
1294
|
catch (e) {
|
|
1065
1295
|
if (announceErrors) {
|
|
@@ -1127,6 +1357,23 @@ function humanTime(iso) {
|
|
|
1127
1357
|
return iso;
|
|
1128
1358
|
}
|
|
1129
1359
|
}
|
|
1360
|
+
/**
|
|
1361
|
+
* Both-sides-changed report. Printed on every run that has conflicts (also in
|
|
1362
|
+
* watch mode) because the alternative — staying silent — is what makes a lost
|
|
1363
|
+
* edit feel like a sync bug.
|
|
1364
|
+
*/
|
|
1365
|
+
function printConflicts(keys, removedRemotely = new Set()) {
|
|
1366
|
+
console.log();
|
|
1367
|
+
console.log(kleur.yellow(`${keys.length} sprite${keys.length === 1 ? '' : 's'} changed in both places — not overwritten:`));
|
|
1368
|
+
for (const key of keys) {
|
|
1369
|
+
const note = removedRemotely.has(key) ? kleur.dim(' (no longer synced from MagicPixel)') : '';
|
|
1370
|
+
console.log(` ${kleur.yellow('~')} ${key}${note}`);
|
|
1371
|
+
}
|
|
1372
|
+
console.log(kleur.dim(` Keep the local edit: run \`${cmd('push')}\` (it will report the cloud change), or delete the PNG to accept MagicPixel's copy.`));
|
|
1373
|
+
if (removedRemotely.size > 0) {
|
|
1374
|
+
console.log(kleur.dim(` Marked (no longer synced): re-check "Sync to Unity" (or restore the document) to push the edit up, or delete the PNG to accept the removal.`));
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1130
1377
|
function printOrphans(orphans) {
|
|
1131
1378
|
console.log();
|
|
1132
1379
|
console.log(kleur.yellow(`Orphaned local files (not in manifest):`));
|