appwrite-utils-cli 1.2.1 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -53,6 +53,7 @@ export declare class ComprehensiveTransfer {
53
53
  private results;
54
54
  private startTime;
55
55
  private tempDir;
56
+ private cachedMaxFileSize?;
56
57
  constructor(options: ComprehensiveTransferOptions);
57
58
  execute(): Promise<TransferResults>;
58
59
  private transferAllUsers;
@@ -66,6 +67,7 @@ export declare class ComprehensiveTransfer {
66
67
  */
67
68
  private transferDatabaseDocuments;
68
69
  private transferAllBuckets;
70
+ private createBucketWithFallback;
69
71
  private transferBucketFiles;
70
72
  private validateAndDownloadFile;
71
73
  private transferAllFunctions;
@@ -74,6 +76,10 @@ export declare class ComprehensiveTransfer {
74
76
  * Helper method to fetch all collections from a database
75
77
  */
76
78
  private fetchAllCollections;
79
+ /**
80
+ * Helper method to fetch all buckets with pagination
81
+ */
82
+ private fetchAllBuckets;
77
83
  /**
78
84
  * Helper method to parse attribute objects (simplified version of parseAttribute)
79
85
  */
@@ -29,6 +29,7 @@ export class ComprehensiveTransfer {
29
29
  results;
30
30
  startTime;
31
31
  tempDir;
32
+ cachedMaxFileSize; // Cache successful maximumFileSize for subsequent buckets
32
33
  constructor(options) {
33
34
  this.options = options;
34
35
  this.sourceClient = getClient(options.sourceEndpoint, options.sourceProject, options.sourceKey);
@@ -272,24 +273,25 @@ export class ComprehensiveTransfer {
272
273
  async transferAllBuckets() {
273
274
  MessageFormatter.info("Starting bucket transfer phase", { prefix: "Transfer" });
274
275
  try {
275
- const sourceBuckets = await this.sourceStorage.listBuckets();
276
- const targetBuckets = await this.targetStorage.listBuckets();
276
+ // Get all buckets from source with pagination
277
+ const allSourceBuckets = await this.fetchAllBuckets(this.sourceStorage);
278
+ const allTargetBuckets = await this.fetchAllBuckets(this.targetStorage);
277
279
  if (this.options.dryRun) {
278
280
  let totalFiles = 0;
279
- for (const bucket of sourceBuckets.buckets) {
281
+ for (const bucket of allSourceBuckets) {
280
282
  const files = await this.sourceStorage.listFiles(bucket.$id, [Query.limit(1)]);
281
283
  totalFiles += files.total;
282
284
  }
283
- MessageFormatter.info(`DRY RUN: Would transfer ${sourceBuckets.buckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
285
+ MessageFormatter.info(`DRY RUN: Would transfer ${allSourceBuckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
284
286
  return;
285
287
  }
286
- const transferTasks = sourceBuckets.buckets.map(bucket => this.limit(async () => {
288
+ const transferTasks = allSourceBuckets.map(bucket => this.limit(async () => {
287
289
  try {
288
290
  // Check if bucket exists in target
289
- const existingBucket = targetBuckets.buckets.find(tb => tb.$id === bucket.$id);
291
+ const existingBucket = allTargetBuckets.find(tb => tb.$id === bucket.$id);
290
292
  if (!existingBucket) {
291
- // Create bucket in target
292
- await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, bucket.maximumFileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
293
+ // Create bucket with fallback strategy for maximumFileSize
294
+ await this.createBucketWithFallback(bucket);
293
295
  MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
294
296
  }
295
297
  // Transfer bucket files with enhanced validation
@@ -309,6 +311,95 @@ export class ComprehensiveTransfer {
309
311
  MessageFormatter.error("Bucket transfer phase failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
310
312
  }
311
313
  }
314
+ async createBucketWithFallback(bucket) {
315
+ // Determine the optimal size to try first
316
+ let sizeToTry;
317
+ if (this.cachedMaxFileSize) {
318
+ // Use cached size if it's smaller than or equal to the bucket's original size
319
+ if (bucket.maximumFileSize >= this.cachedMaxFileSize) {
320
+ sizeToTry = this.cachedMaxFileSize;
321
+ MessageFormatter.info(`Bucket ${bucket.name}: Using cached maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
322
+ }
323
+ else {
324
+ // Original size is smaller than cached size, try original first
325
+ sizeToTry = bucket.maximumFileSize;
326
+ }
327
+ }
328
+ else {
329
+ // No cached size yet, try original size first
330
+ sizeToTry = bucket.maximumFileSize;
331
+ }
332
+ // Try the optimal size first
333
+ try {
334
+ await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, sizeToTry, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
335
+ // Success - cache this size if it's not already cached or is smaller than cached
336
+ if (!this.cachedMaxFileSize || sizeToTry < this.cachedMaxFileSize) {
337
+ this.cachedMaxFileSize = sizeToTry;
338
+ MessageFormatter.info(`Bucket ${bucket.name}: Cached successful maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
339
+ }
340
+ // Log if we used a different size than original
341
+ if (sizeToTry !== bucket.maximumFileSize) {
342
+ MessageFormatter.warning(`Bucket ${bucket.name}: maximumFileSize used ${sizeToTry} instead of original ${bucket.maximumFileSize} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
343
+ }
344
+ return; // Success, exit the function
345
+ }
346
+ catch (error) {
347
+ const err = error instanceof Error ? error : new Error(String(error));
348
+ // Check if the error is related to maximumFileSize validation
349
+ if (err.message.includes('maximumFileSize') || err.message.includes('valid range')) {
350
+ MessageFormatter.warning(`Bucket ${bucket.name}: Failed with maximumFileSize ${sizeToTry}, falling back to smaller sizes...`, { prefix: "Transfer" });
351
+ // Continue to fallback logic below
352
+ }
353
+ else {
354
+ // Different error, don't retry
355
+ throw err;
356
+ }
357
+ }
358
+ // Fallback to progressively smaller sizes
359
+ const fallbackSizes = [
360
+ 5_000_000_000, // 5GB
361
+ 2_500_000_000, // 2.5GB
362
+ 2_000_000_000, // 2GB
363
+ 1_000_000_000, // 1GB
364
+ 500_000_000, // 500MB
365
+ 100_000_000 // 100MB
366
+ ];
367
+ // Remove sizes that are larger than or equal to the already-tried size
368
+ const validSizes = fallbackSizes
369
+ .filter(size => size < sizeToTry)
370
+ .sort((a, b) => b - a); // Sort descending
371
+ let lastError = null;
372
+ for (const fileSize of validSizes) {
373
+ try {
374
+ await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, fileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
375
+ // Success - cache this size if it's not already cached or is smaller than cached
376
+ if (!this.cachedMaxFileSize || fileSize < this.cachedMaxFileSize) {
377
+ this.cachedMaxFileSize = fileSize;
378
+ MessageFormatter.info(`Bucket ${bucket.name}: Cached successful maximumFileSize ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
379
+ }
380
+ // Log if we had to reduce the file size
381
+ if (fileSize !== bucket.maximumFileSize) {
382
+ MessageFormatter.warning(`Bucket ${bucket.name}: maximumFileSize reduced from ${bucket.maximumFileSize} to ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
383
+ }
384
+ return; // Success, exit the function
385
+ }
386
+ catch (error) {
387
+ lastError = error instanceof Error ? error : new Error(String(error));
388
+ // Check if the error is related to maximumFileSize validation
389
+ if (lastError.message.includes('maximumFileSize') || lastError.message.includes('valid range')) {
390
+ MessageFormatter.warning(`Bucket ${bucket.name}: Failed with maximumFileSize ${fileSize}, trying smaller size...`, { prefix: "Transfer" });
391
+ continue; // Try next smaller size
392
+ }
393
+ else {
394
+ // Different error, don't retry
395
+ throw lastError;
396
+ }
397
+ }
398
+ }
399
+ // If we get here, all fallback sizes failed
400
+ MessageFormatter.error(`Bucket ${bucket.name}: All fallback file sizes failed. Last error: ${lastError?.message}`, lastError || undefined, { prefix: "Transfer" });
401
+ throw lastError || new Error('All fallback file sizes failed');
402
+ }
312
403
  async transferBucketFiles(sourceBucketId, targetBucketId) {
313
404
  let lastFileId;
314
405
  let transferredFiles = 0;
@@ -480,6 +571,29 @@ export class ComprehensiveTransfer {
480
571
  }
481
572
  return collections;
482
573
  }
574
+ /**
575
+ * Helper method to fetch all buckets with pagination
576
+ */
577
+ async fetchAllBuckets(storage) {
578
+ const buckets = [];
579
+ let lastId;
580
+ while (true) {
581
+ const queries = [Query.limit(100)];
582
+ if (lastId) {
583
+ queries.push(Query.cursorAfter(lastId));
584
+ }
585
+ const result = await tryAwaitWithRetry(async () => storage.listBuckets(queries));
586
+ if (result.buckets.length === 0) {
587
+ break;
588
+ }
589
+ buckets.push(...result.buckets);
590
+ if (result.buckets.length < 100) {
591
+ break;
592
+ }
593
+ lastId = result.buckets[result.buckets.length - 1].$id;
594
+ }
595
+ return buckets;
596
+ }
483
597
  /**
484
598
  * Helper method to parse attribute objects (simplified version of parseAttribute)
485
599
  */
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "appwrite-utils-cli",
3
3
  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
4
- "version": "1.2.1",
4
+ "version": "1.2.3",
5
5
  "main": "src/main.ts",
6
6
  "type": "module",
7
7
  "repository": {
@@ -68,6 +68,7 @@ export class ComprehensiveTransfer {
68
68
  private results: TransferResults;
69
69
  private startTime: number;
70
70
  private tempDir: string;
71
+ private cachedMaxFileSize?: number; // Cache successful maximumFileSize for subsequent buckets
71
72
 
72
73
  constructor(private options: ComprehensiveTransferOptions) {
73
74
  this.sourceClient = getClient(
@@ -408,39 +409,29 @@ export class ComprehensiveTransfer {
408
409
  MessageFormatter.info("Starting bucket transfer phase", { prefix: "Transfer" });
409
410
 
410
411
  try {
411
- const sourceBuckets = await this.sourceStorage.listBuckets();
412
- const targetBuckets = await this.targetStorage.listBuckets();
412
+ // Get all buckets from source with pagination
413
+ const allSourceBuckets = await this.fetchAllBuckets(this.sourceStorage);
414
+ const allTargetBuckets = await this.fetchAllBuckets(this.targetStorage);
413
415
 
414
416
  if (this.options.dryRun) {
415
417
  let totalFiles = 0;
416
- for (const bucket of sourceBuckets.buckets) {
418
+ for (const bucket of allSourceBuckets) {
417
419
  const files = await this.sourceStorage.listFiles(bucket.$id, [Query.limit(1)]);
418
420
  totalFiles += files.total;
419
421
  }
420
- MessageFormatter.info(`DRY RUN: Would transfer ${sourceBuckets.buckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
422
+ MessageFormatter.info(`DRY RUN: Would transfer ${allSourceBuckets.length} buckets with ${totalFiles} files`, { prefix: "Transfer" });
421
423
  return;
422
424
  }
423
425
 
424
- const transferTasks = sourceBuckets.buckets.map(bucket =>
426
+ const transferTasks = allSourceBuckets.map(bucket =>
425
427
  this.limit(async () => {
426
428
  try {
427
429
  // Check if bucket exists in target
428
- const existingBucket = targetBuckets.buckets.find(tb => tb.$id === bucket.$id);
430
+ const existingBucket = allTargetBuckets.find(tb => tb.$id === bucket.$id);
429
431
 
430
432
  if (!existingBucket) {
431
- // Create bucket in target
432
- await this.targetStorage.createBucket(
433
- bucket.$id,
434
- bucket.name,
435
- bucket.$permissions,
436
- bucket.fileSecurity,
437
- bucket.enabled,
438
- bucket.maximumFileSize,
439
- bucket.allowedFileExtensions,
440
- bucket.compression as any,
441
- bucket.encryption,
442
- bucket.antivirus
443
- );
433
+ // Create bucket with fallback strategy for maximumFileSize
434
+ await this.createBucketWithFallback(bucket);
444
435
  MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
445
436
  }
446
437
 
@@ -463,6 +454,152 @@ export class ComprehensiveTransfer {
463
454
  }
464
455
  }
465
456
 
457
+ private async createBucketWithFallback(bucket: Models.Bucket): Promise<void> {
458
+ // Determine the optimal size to try first
459
+ let sizeToTry: number;
460
+
461
+ if (this.cachedMaxFileSize) {
462
+ // Use cached size if it's smaller than or equal to the bucket's original size
463
+ if (bucket.maximumFileSize >= this.cachedMaxFileSize) {
464
+ sizeToTry = this.cachedMaxFileSize;
465
+ MessageFormatter.info(
466
+ `Bucket ${bucket.name}: Using cached maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`,
467
+ { prefix: "Transfer" }
468
+ );
469
+ } else {
470
+ // Original size is smaller than cached size, try original first
471
+ sizeToTry = bucket.maximumFileSize;
472
+ }
473
+ } else {
474
+ // No cached size yet, try original size first
475
+ sizeToTry = bucket.maximumFileSize;
476
+ }
477
+
478
+ // Try the optimal size first
479
+ try {
480
+ await this.targetStorage.createBucket(
481
+ bucket.$id,
482
+ bucket.name,
483
+ bucket.$permissions,
484
+ bucket.fileSecurity,
485
+ bucket.enabled,
486
+ sizeToTry,
487
+ bucket.allowedFileExtensions,
488
+ bucket.compression as any,
489
+ bucket.encryption,
490
+ bucket.antivirus
491
+ );
492
+
493
+ // Success - cache this size if it's not already cached or is smaller than cached
494
+ if (!this.cachedMaxFileSize || sizeToTry < this.cachedMaxFileSize) {
495
+ this.cachedMaxFileSize = sizeToTry;
496
+ MessageFormatter.info(
497
+ `Bucket ${bucket.name}: Cached successful maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`,
498
+ { prefix: "Transfer" }
499
+ );
500
+ }
501
+
502
+ // Log if we used a different size than original
503
+ if (sizeToTry !== bucket.maximumFileSize) {
504
+ MessageFormatter.warning(
505
+ `Bucket ${bucket.name}: maximumFileSize used ${sizeToTry} instead of original ${bucket.maximumFileSize} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`,
506
+ { prefix: "Transfer" }
507
+ );
508
+ }
509
+
510
+ return; // Success, exit the function
511
+ } catch (error) {
512
+ const err = error instanceof Error ? error : new Error(String(error));
513
+
514
+ // Check if the error is related to maximumFileSize validation
515
+ if (err.message.includes('maximumFileSize') || err.message.includes('valid range')) {
516
+ MessageFormatter.warning(
517
+ `Bucket ${bucket.name}: Failed with maximumFileSize ${sizeToTry}, falling back to smaller sizes...`,
518
+ { prefix: "Transfer" }
519
+ );
520
+ // Continue to fallback logic below
521
+ } else {
522
+ // Different error, don't retry
523
+ throw err;
524
+ }
525
+ }
526
+
527
+ // Fallback to progressively smaller sizes
528
+ const fallbackSizes = [
529
+ 5_000_000_000, // 5GB
530
+ 2_500_000_000, // 2.5GB
531
+ 2_000_000_000, // 2GB
532
+ 1_000_000_000, // 1GB
533
+ 500_000_000, // 500MB
534
+ 100_000_000 // 100MB
535
+ ];
536
+
537
+ // Remove sizes that are larger than or equal to the already-tried size
538
+ const validSizes = fallbackSizes
539
+ .filter(size => size < sizeToTry)
540
+ .sort((a, b) => b - a); // Sort descending
541
+
542
+ let lastError: Error | null = null;
543
+
544
+ for (const fileSize of validSizes) {
545
+ try {
546
+ await this.targetStorage.createBucket(
547
+ bucket.$id,
548
+ bucket.name,
549
+ bucket.$permissions,
550
+ bucket.fileSecurity,
551
+ bucket.enabled,
552
+ fileSize,
553
+ bucket.allowedFileExtensions,
554
+ bucket.compression as any,
555
+ bucket.encryption,
556
+ bucket.antivirus
557
+ );
558
+
559
+ // Success - cache this size if it's not already cached or is smaller than cached
560
+ if (!this.cachedMaxFileSize || fileSize < this.cachedMaxFileSize) {
561
+ this.cachedMaxFileSize = fileSize;
562
+ MessageFormatter.info(
563
+ `Bucket ${bucket.name}: Cached successful maximumFileSize ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`,
564
+ { prefix: "Transfer" }
565
+ );
566
+ }
567
+
568
+ // Log if we had to reduce the file size
569
+ if (fileSize !== bucket.maximumFileSize) {
570
+ MessageFormatter.warning(
571
+ `Bucket ${bucket.name}: maximumFileSize reduced from ${bucket.maximumFileSize} to ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`,
572
+ { prefix: "Transfer" }
573
+ );
574
+ }
575
+
576
+ return; // Success, exit the function
577
+ } catch (error) {
578
+ lastError = error instanceof Error ? error : new Error(String(error));
579
+
580
+ // Check if the error is related to maximumFileSize validation
581
+ if (lastError.message.includes('maximumFileSize') || lastError.message.includes('valid range')) {
582
+ MessageFormatter.warning(
583
+ `Bucket ${bucket.name}: Failed with maximumFileSize ${fileSize}, trying smaller size...`,
584
+ { prefix: "Transfer" }
585
+ );
586
+ continue; // Try next smaller size
587
+ } else {
588
+ // Different error, don't retry
589
+ throw lastError;
590
+ }
591
+ }
592
+ }
593
+
594
+ // If we get here, all fallback sizes failed
595
+ MessageFormatter.error(
596
+ `Bucket ${bucket.name}: All fallback file sizes failed. Last error: ${lastError?.message}`,
597
+ lastError || undefined,
598
+ { prefix: "Transfer" }
599
+ );
600
+ throw lastError || new Error('All fallback file sizes failed');
601
+ }
602
+
466
603
  private async transferBucketFiles(sourceBucketId: string, targetBucketId: string): Promise<void> {
467
604
  let lastFileId: string | undefined;
468
605
  let transferredFiles = 0;
@@ -676,6 +813,37 @@ export class ComprehensiveTransfer {
676
813
  return collections;
677
814
  }
678
815
 
816
+ /**
817
+ * Helper method to fetch all buckets with pagination
818
+ */
819
+ private async fetchAllBuckets(storage: Storage): Promise<Models.Bucket[]> {
820
+ const buckets: Models.Bucket[] = [];
821
+ let lastId: string | undefined;
822
+
823
+ while (true) {
824
+ const queries = [Query.limit(100)];
825
+ if (lastId) {
826
+ queries.push(Query.cursorAfter(lastId));
827
+ }
828
+
829
+ const result = await tryAwaitWithRetry(async () => storage.listBuckets(queries));
830
+
831
+ if (result.buckets.length === 0) {
832
+ break;
833
+ }
834
+
835
+ buckets.push(...result.buckets);
836
+
837
+ if (result.buckets.length < 100) {
838
+ break;
839
+ }
840
+
841
+ lastId = result.buckets[result.buckets.length - 1].$id;
842
+ }
843
+
844
+ return buckets;
845
+ }
846
+
679
847
  /**
680
848
  * Helper method to parse attribute objects (simplified version of parseAttribute)
681
849
  */