@iksdev/shard-cli 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/shard.js +278 -3
- package/package.json +1 -1
package/bin/shard.js
CHANGED
|
@@ -4,6 +4,8 @@ const fs = require('fs');
|
|
|
4
4
|
const fsp = require('fs/promises');
|
|
5
5
|
const os = require('os');
|
|
6
6
|
const path = require('path');
|
|
7
|
+
const { Readable } = require('stream');
|
|
8
|
+
const { pipeline } = require('stream/promises');
|
|
7
9
|
|
|
8
10
|
const CONFIG_DIR = path.join(os.homedir(), '.shard-cli');
|
|
9
11
|
const CONFIG_PATH = path.join(CONFIG_DIR, 'config.json');
|
|
@@ -17,7 +19,7 @@ function printHelp() {
|
|
|
17
19
|
Usage:
|
|
18
20
|
shard login --username <name> --password <pass> [--server <url>]
|
|
19
21
|
shard whoami [--server <url>]
|
|
20
|
-
shard sync <folder> [--server <url>] [--dry-run] [--force]
|
|
22
|
+
shard sync <folder> [--server <url>] [--dry-run] [--force] [--once] [--interval-ms <n>]
|
|
21
23
|
shard logout
|
|
22
24
|
shard config show
|
|
23
25
|
shard config set-server <url>
|
|
@@ -25,6 +27,7 @@ Usage:
|
|
|
25
27
|
Examples:
|
|
26
28
|
shard login --server http://localhost:3000 --username admin --password secret
|
|
27
29
|
shard sync ./MonDossier
|
|
30
|
+
shard sync ./MonDossier --once
|
|
28
31
|
shard sync ./MonDossier --dry-run
|
|
29
32
|
`);
|
|
30
33
|
}
|
|
@@ -285,10 +288,270 @@ async function uploadOneFile(server, token, file) {
|
|
|
285
288
|
});
|
|
286
289
|
}
|
|
287
290
|
|
|
291
|
+
function fileListToMap(files) {
|
|
292
|
+
const map = new Map();
|
|
293
|
+
for (const file of files) {
|
|
294
|
+
map.set(file.relPath, file);
|
|
295
|
+
}
|
|
296
|
+
return map;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function makeFingerprint(file) {
|
|
300
|
+
return `${file.size}:${file.mtimeMs}`;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function normalizeRemoteRelPath(name) {
|
|
304
|
+
return String(name || '').replace(/\\/g, '/');
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
async function fetchAllRemoteFiles(server, token) {
|
|
308
|
+
const out = [];
|
|
309
|
+
const limit = 100;
|
|
310
|
+
let offset = 0;
|
|
311
|
+
|
|
312
|
+
while (true) {
|
|
313
|
+
const data = await httpJson(`${server}/api/files?limit=${limit}&offset=${offset}&sort=created_at&order=asc`, {
|
|
314
|
+
method: 'GET',
|
|
315
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
316
|
+
});
|
|
317
|
+
const rows = Array.isArray(data.files) ? data.files : [];
|
|
318
|
+
for (const row of rows) {
|
|
319
|
+
out.push({
|
|
320
|
+
id: row.id,
|
|
321
|
+
relPath: normalizeRemoteRelPath(row.original_name),
|
|
322
|
+
size: Number(row.file_size || 0)
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
if (!data.pagination?.hasMore) break;
|
|
326
|
+
offset += limit;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
return out;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
async function downloadRemoteFile(server, token, remoteFile, destPath) {
|
|
333
|
+
const res = await fetch(`${server}/api/files/${remoteFile.id}/download`, {
|
|
334
|
+
method: 'GET',
|
|
335
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
if (!res.ok) {
|
|
339
|
+
let detail = `HTTP ${res.status}`;
|
|
340
|
+
try {
|
|
341
|
+
const data = await res.json();
|
|
342
|
+
detail = data.message || data.error || detail;
|
|
343
|
+
} catch {
|
|
344
|
+
// ignore JSON parse failures
|
|
345
|
+
}
|
|
346
|
+
throw new Error(detail);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
if (!res.body) {
|
|
350
|
+
throw new Error('Reponse de telechargement vide');
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
await fsp.mkdir(path.dirname(destPath), { recursive: true });
|
|
354
|
+
const tempPath = `${destPath}.part`;
|
|
355
|
+
const writable = fs.createWriteStream(tempPath);
|
|
356
|
+
await pipeline(Readable.fromWeb(res.body), writable);
|
|
357
|
+
await fsp.rename(tempPath, destPath);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
async function pullMissingRemoteFiles(server, token, rootDir, state, localFiles, remoteFiles, dryRun) {
|
|
361
|
+
const localMap = fileListToMap(localFiles);
|
|
362
|
+
const missingRemote = remoteFiles.filter((rf) => !localMap.has(rf.relPath));
|
|
363
|
+
|
|
364
|
+
if (missingRemote.length === 0) return { downloaded: 0, errors: 0 };
|
|
365
|
+
|
|
366
|
+
console.log(`A telecharger: ${missingRemote.length}`);
|
|
367
|
+
if (dryRun) {
|
|
368
|
+
for (const rf of missingRemote.slice(0, 50)) console.log(`- ${rf.relPath} (remote)`); // eslint-disable-line no-console
|
|
369
|
+
if (missingRemote.length > 50) console.log(`... +${missingRemote.length - 50} autres`);
|
|
370
|
+
return { downloaded: 0, errors: 0 };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
let downloaded = 0;
|
|
374
|
+
let errors = 0;
|
|
375
|
+
for (let i = 0; i < missingRemote.length; i += 1) {
|
|
376
|
+
const rf = missingRemote[i];
|
|
377
|
+
const label = `[DOWN ${i + 1}/${missingRemote.length}]`;
|
|
378
|
+
try {
|
|
379
|
+
const absPath = path.join(rootDir, ...rf.relPath.split('/'));
|
|
380
|
+
console.log(`${label} GET ${rf.relPath} (${formatBytes(rf.size)})`);
|
|
381
|
+
await downloadRemoteFile(server, token, rf, absPath);
|
|
382
|
+
const st = await fsp.stat(absPath);
|
|
383
|
+
state.files[rf.relPath] = {
|
|
384
|
+
size: st.size,
|
|
385
|
+
mtimeMs: Math.round(st.mtimeMs),
|
|
386
|
+
uploadedAt: new Date().toISOString(),
|
|
387
|
+
remoteId: rf.id
|
|
388
|
+
};
|
|
389
|
+
downloaded += 1;
|
|
390
|
+
console.log(`${label} OK ${rf.relPath}`);
|
|
391
|
+
} catch (error) {
|
|
392
|
+
errors += 1;
|
|
393
|
+
console.error(`${label} FAIL ${rf.relPath} -> ${error.message}`);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
return { downloaded, errors };
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function seedStateFromRemote(state, localFiles, remoteFiles) {
|
|
401
|
+
const remoteByPath = new Map(remoteFiles.map((rf) => [rf.relPath, rf]));
|
|
402
|
+
for (const lf of localFiles) {
|
|
403
|
+
if (state.files[lf.relPath]) continue;
|
|
404
|
+
const remote = remoteByPath.get(lf.relPath);
|
|
405
|
+
if (!remote) continue;
|
|
406
|
+
if (Number(remote.size) !== Number(lf.size)) continue;
|
|
407
|
+
state.files[lf.relPath] = {
|
|
408
|
+
size: lf.size,
|
|
409
|
+
mtimeMs: lf.mtimeMs,
|
|
410
|
+
uploadedAt: new Date().toISOString(),
|
|
411
|
+
remoteId: remote.id
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async function deleteRemoteFile(server, token, remoteId) {
|
|
417
|
+
return httpJson(`${server}/api/files/${remoteId}`, {
|
|
418
|
+
method: 'DELETE',
|
|
419
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async function renameRemoteFile(server, token, remoteId, newName) {
|
|
424
|
+
return httpJson(`${server}/api/files/${remoteId}/rename`, {
|
|
425
|
+
method: 'PATCH',
|
|
426
|
+
headers: {
|
|
427
|
+
Authorization: `Bearer ${token}`,
|
|
428
|
+
'Content-Type': 'application/json'
|
|
429
|
+
},
|
|
430
|
+
body: JSON.stringify({ newName })
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function sleep(ms) {
|
|
435
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
async function mirrorRealtime(server, token, rootDir, state, intervalMs) {
|
|
439
|
+
console.log(`Mode temps reel actif (${intervalMs} ms). Ctrl+C pour quitter.`);
|
|
440
|
+
|
|
441
|
+
let stopped = false;
|
|
442
|
+
const onStop = () => { stopped = true; };
|
|
443
|
+
process.on('SIGINT', onStop);
|
|
444
|
+
process.on('SIGTERM', onStop);
|
|
445
|
+
|
|
446
|
+
let previousFiles = await listFilesRecursive(rootDir);
|
|
447
|
+
let previousMap = fileListToMap(previousFiles);
|
|
448
|
+
|
|
449
|
+
while (!stopped) {
|
|
450
|
+
await sleep(intervalMs);
|
|
451
|
+
if (stopped) break;
|
|
452
|
+
|
|
453
|
+
let currentFiles;
|
|
454
|
+
try {
|
|
455
|
+
currentFiles = await listFilesRecursive(rootDir);
|
|
456
|
+
} catch (error) {
|
|
457
|
+
console.error(`Watcher error: ${error.message}`);
|
|
458
|
+
continue;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const currentMap = fileListToMap(currentFiles);
|
|
462
|
+
const prevKeys = new Set(previousMap.keys());
|
|
463
|
+
const currKeys = new Set(currentMap.keys());
|
|
464
|
+
const removed = [...prevKeys].filter((k) => !currKeys.has(k)).map((k) => previousMap.get(k));
|
|
465
|
+
const added = [...currKeys].filter((k) => !prevKeys.has(k)).map((k) => currentMap.get(k));
|
|
466
|
+
const changed = [...currKeys]
|
|
467
|
+
.filter((k) => prevKeys.has(k))
|
|
468
|
+
.map((k) => ({ prev: previousMap.get(k), cur: currentMap.get(k) }))
|
|
469
|
+
.filter((p) => p.prev.size !== p.cur.size || p.prev.mtimeMs !== p.cur.mtimeMs)
|
|
470
|
+
.map((p) => p.cur);
|
|
471
|
+
|
|
472
|
+
// Detecter les renames simples (meme taille + mtime) pour appeler PATCH rename.
|
|
473
|
+
const addedByFinger = new Map();
|
|
474
|
+
for (const a of added) {
|
|
475
|
+
const key = makeFingerprint(a);
|
|
476
|
+
if (!addedByFinger.has(key)) addedByFinger.set(key, []);
|
|
477
|
+
addedByFinger.get(key).push(a);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
const renamedPairs = [];
|
|
481
|
+
const trulyRemoved = [];
|
|
482
|
+
for (const r of removed) {
|
|
483
|
+
const matches = addedByFinger.get(makeFingerprint(r)) || [];
|
|
484
|
+
const remoteId = state.files[r.relPath]?.remoteId || null;
|
|
485
|
+
if (remoteId && matches.length > 0) {
|
|
486
|
+
const to = matches.shift();
|
|
487
|
+
renamedPairs.push({ from: r, to, remoteId });
|
|
488
|
+
} else {
|
|
489
|
+
trulyRemoved.push(r);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
const renamedTargets = new Set(renamedPairs.map((p) => p.to.relPath));
|
|
493
|
+
const trulyAdded = added.filter((a) => !renamedTargets.has(a.relPath));
|
|
494
|
+
|
|
495
|
+
for (const pair of renamedPairs) {
|
|
496
|
+
try {
|
|
497
|
+
await renameRemoteFile(server, token, pair.remoteId, pair.to.relPath);
|
|
498
|
+
delete state.files[pair.from.relPath];
|
|
499
|
+
state.files[pair.to.relPath] = {
|
|
500
|
+
size: pair.to.size,
|
|
501
|
+
mtimeMs: pair.to.mtimeMs,
|
|
502
|
+
uploadedAt: new Date().toISOString(),
|
|
503
|
+
remoteId: pair.remoteId
|
|
504
|
+
};
|
|
505
|
+
console.log(`[WATCH] RENAME ${pair.from.relPath} -> ${pair.to.relPath}`);
|
|
506
|
+
} catch (error) {
|
|
507
|
+
console.error(`[WATCH] RENAME FAIL ${pair.from.relPath} -> ${pair.to.relPath}: ${error.message}`);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
for (const file of trulyRemoved) {
|
|
512
|
+
const remoteId = state.files[file.relPath]?.remoteId || null;
|
|
513
|
+
if (!remoteId) {
|
|
514
|
+
delete state.files[file.relPath];
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
517
|
+
try {
|
|
518
|
+
await deleteRemoteFile(server, token, remoteId);
|
|
519
|
+
delete state.files[file.relPath];
|
|
520
|
+
console.log(`[WATCH] DELETE ${file.relPath}`);
|
|
521
|
+
} catch (error) {
|
|
522
|
+
console.error(`[WATCH] DELETE FAIL ${file.relPath}: ${error.message}`);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
const uploads = [...trulyAdded, ...changed];
|
|
527
|
+
for (const file of uploads) {
|
|
528
|
+
try {
|
|
529
|
+
const result = await uploadOneFile(server, token, file);
|
|
530
|
+
state.files[file.relPath] = {
|
|
531
|
+
size: file.size,
|
|
532
|
+
mtimeMs: file.mtimeMs,
|
|
533
|
+
uploadedAt: new Date().toISOString(),
|
|
534
|
+
remoteId: result?.file?.id || state.files[file.relPath]?.remoteId || null
|
|
535
|
+
};
|
|
536
|
+
console.log(`[WATCH] UPLOAD ${file.relPath} (${formatBytes(file.size)})`);
|
|
537
|
+
} catch (error) {
|
|
538
|
+
console.error(`[WATCH] UPLOAD FAIL ${file.relPath}: ${error.message}`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
await writeState(rootDir, state);
|
|
543
|
+
previousFiles = currentFiles;
|
|
544
|
+
previousMap = currentMap;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
process.off('SIGINT', onStop);
|
|
548
|
+
process.off('SIGTERM', onStop);
|
|
549
|
+
}
|
|
550
|
+
|
|
288
551
|
async function syncFolder(positionals, flags) {
|
|
289
552
|
const target = positionals[0];
|
|
290
553
|
if (!target) {
|
|
291
|
-
throw new Error('Usage: shard sync <folder> [--server <url>] [--dry-run] [--force]');
|
|
554
|
+
throw new Error('Usage: shard sync <folder> [--server <url>] [--dry-run] [--force] [--once] [--interval-ms <n>]');
|
|
292
555
|
}
|
|
293
556
|
|
|
294
557
|
const rootDir = path.resolve(process.cwd(), target);
|
|
@@ -312,10 +575,18 @@ async function syncFolder(positionals, flags) {
|
|
|
312
575
|
headers: { Authorization: `Bearer ${token}` }
|
|
313
576
|
});
|
|
314
577
|
|
|
315
|
-
const files = await listFilesRecursive(rootDir);
|
|
316
578
|
const state = await readState(rootDir);
|
|
317
579
|
const force = Boolean(flags.force);
|
|
318
580
|
const dryRun = Boolean(flags['dry-run']);
|
|
581
|
+
const once = Boolean(flags.once);
|
|
582
|
+
const intervalMs = Math.max(parseInt(flags['interval-ms'] || '2000', 10) || 2000, 500);
|
|
583
|
+
|
|
584
|
+
const remoteFiles = await fetchAllRemoteFiles(server, token);
|
|
585
|
+
const initialLocalFiles = await listFilesRecursive(rootDir);
|
|
586
|
+
await pullMissingRemoteFiles(server, token, rootDir, state, initialLocalFiles, remoteFiles, dryRun);
|
|
587
|
+
|
|
588
|
+
const files = await listFilesRecursive(rootDir);
|
|
589
|
+
seedStateFromRemote(state, files, remoteFiles);
|
|
319
590
|
|
|
320
591
|
const changed = files.filter((f) => {
|
|
321
592
|
if (force) return true;
|
|
@@ -328,6 +599,7 @@ async function syncFolder(positionals, flags) {
|
|
|
328
599
|
console.log(`Server: ${server}`);
|
|
329
600
|
console.log(`Dossier: ${rootDir}`);
|
|
330
601
|
console.log(`Total: ${files.length} fichier(s)`);
|
|
602
|
+
console.log(`Distants: ${remoteFiles.length}`);
|
|
331
603
|
console.log(`A uploader: ${changed.length}`);
|
|
332
604
|
console.log(`Inchanges: ${unchanged}`);
|
|
333
605
|
|
|
@@ -370,6 +642,9 @@ async function syncFolder(positionals, flags) {
|
|
|
370
642
|
|
|
371
643
|
console.log(`Termine. Uploades: ${success}, erreurs: ${failed}`);
|
|
372
644
|
if (failed > 0) process.exitCode = 2;
|
|
645
|
+
if (!once && !dryRun) {
|
|
646
|
+
await mirrorRealtime(server, token, rootDir, state, intervalMs);
|
|
647
|
+
}
|
|
373
648
|
}
|
|
374
649
|
|
|
375
650
|
async function main() {
|