appwrite-utils-cli 1.2.2 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -53,6 +53,7 @@ export declare class ComprehensiveTransfer {
53
53
  private results;
54
54
  private startTime;
55
55
  private tempDir;
56
+ private cachedMaxFileSize?;
56
57
  constructor(options: ComprehensiveTransferOptions);
57
58
  execute(): Promise<TransferResults>;
58
59
  private transferAllUsers;
@@ -66,6 +67,7 @@ export declare class ComprehensiveTransfer {
66
67
  */
67
68
  private transferDatabaseDocuments;
68
69
  private transferAllBuckets;
70
+ private createBucketWithFallback;
69
71
  private transferBucketFiles;
70
72
  private validateAndDownloadFile;
71
73
  private transferAllFunctions;
@@ -29,6 +29,7 @@ export class ComprehensiveTransfer {
29
29
  results;
30
30
  startTime;
31
31
  tempDir;
32
+ cachedMaxFileSize; // Cache successful maximumFileSize for subsequent buckets
32
33
  constructor(options) {
33
34
  this.options = options;
34
35
  this.sourceClient = getClient(options.sourceEndpoint, options.sourceProject, options.sourceKey);
@@ -289,8 +290,8 @@ export class ComprehensiveTransfer {
289
290
  // Check if bucket exists in target
290
291
  const existingBucket = allTargetBuckets.find(tb => tb.$id === bucket.$id);
291
292
  if (!existingBucket) {
292
- // Create bucket in target
293
- await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, bucket.maximumFileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
293
+ // Create bucket with fallback strategy for maximumFileSize
294
+ await this.createBucketWithFallback(bucket);
294
295
  MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
295
296
  }
296
297
  // Transfer bucket files with enhanced validation
@@ -310,6 +311,95 @@ export class ComprehensiveTransfer {
310
311
  MessageFormatter.error("Bucket transfer phase failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
311
312
  }
312
313
  }
314
+ async createBucketWithFallback(bucket) {
315
+ // Determine the optimal size to try first
316
+ let sizeToTry;
317
+ if (this.cachedMaxFileSize) {
318
+ // Use cached size if it's smaller than or equal to the bucket's original size
319
+ if (bucket.maximumFileSize >= this.cachedMaxFileSize) {
320
+ sizeToTry = this.cachedMaxFileSize;
321
+ MessageFormatter.info(`Bucket ${bucket.name}: Using cached maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
322
+ }
323
+ else {
324
+ // Original size is smaller than cached size, try original first
325
+ sizeToTry = bucket.maximumFileSize;
326
+ }
327
+ }
328
+ else {
329
+ // No cached size yet, try original size first
330
+ sizeToTry = bucket.maximumFileSize;
331
+ }
332
+ // Try the optimal size first
333
+ try {
334
+ await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, sizeToTry, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
335
+ // Success - cache this size if it's not already cached or is smaller than cached
336
+ if (!this.cachedMaxFileSize || sizeToTry < this.cachedMaxFileSize) {
337
+ this.cachedMaxFileSize = sizeToTry;
338
+ MessageFormatter.info(`Bucket ${bucket.name}: Cached successful maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
339
+ }
340
+ // Log if we used a different size than original
341
+ if (sizeToTry !== bucket.maximumFileSize) {
342
+ MessageFormatter.warning(`Bucket ${bucket.name}: maximumFileSize used ${sizeToTry} instead of original ${bucket.maximumFileSize} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
343
+ }
344
+ return; // Success, exit the function
345
+ }
346
+ catch (error) {
347
+ const err = error instanceof Error ? error : new Error(String(error));
348
+ // Check if the error is related to maximumFileSize validation
349
+ if (err.message.includes('maximumFileSize') || err.message.includes('valid range')) {
350
+ MessageFormatter.warning(`Bucket ${bucket.name}: Failed with maximumFileSize ${sizeToTry}, falling back to smaller sizes...`, { prefix: "Transfer" });
351
+ // Continue to fallback logic below
352
+ }
353
+ else {
354
+ // Different error, don't retry
355
+ throw err;
356
+ }
357
+ }
358
+ // Fallback to progressively smaller sizes
359
+ const fallbackSizes = [
360
+ 5_000_000_000, // 5GB
361
+ 2_500_000_000, // 2.5GB
362
+ 2_000_000_000, // 2GB
363
+ 1_000_000_000, // 1GB
364
+ 500_000_000, // 500MB
365
+ 100_000_000 // 100MB
366
+ ];
367
+ // Remove sizes that are larger than or equal to the already-tried size
368
+ const validSizes = fallbackSizes
369
+ .filter(size => size < sizeToTry)
370
+ .sort((a, b) => b - a); // Sort descending
371
+ let lastError = null;
372
+ for (const fileSize of validSizes) {
373
+ try {
374
+ await this.targetStorage.createBucket(bucket.$id, bucket.name, bucket.$permissions, bucket.fileSecurity, bucket.enabled, fileSize, bucket.allowedFileExtensions, bucket.compression, bucket.encryption, bucket.antivirus);
375
+ // Success - cache this size if it's not already cached or is smaller than cached
376
+ if (!this.cachedMaxFileSize || fileSize < this.cachedMaxFileSize) {
377
+ this.cachedMaxFileSize = fileSize;
378
+ MessageFormatter.info(`Bucket ${bucket.name}: Cached successful maximumFileSize ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
379
+ }
380
+ // Log if we had to reduce the file size
381
+ if (fileSize !== bucket.maximumFileSize) {
382
+ MessageFormatter.warning(`Bucket ${bucket.name}: maximumFileSize reduced from ${bucket.maximumFileSize} to ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`, { prefix: "Transfer" });
383
+ }
384
+ return; // Success, exit the function
385
+ }
386
+ catch (error) {
387
+ lastError = error instanceof Error ? error : new Error(String(error));
388
+ // Check if the error is related to maximumFileSize validation
389
+ if (lastError.message.includes('maximumFileSize') || lastError.message.includes('valid range')) {
390
+ MessageFormatter.warning(`Bucket ${bucket.name}: Failed with maximumFileSize ${fileSize}, trying smaller size...`, { prefix: "Transfer" });
391
+ continue; // Try next smaller size
392
+ }
393
+ else {
394
+ // Different error, don't retry
395
+ throw lastError;
396
+ }
397
+ }
398
+ }
399
+ // If we get here, all fallback sizes failed
400
+ MessageFormatter.error(`Bucket ${bucket.name}: All fallback file sizes failed. Last error: ${lastError?.message}`, lastError || undefined, { prefix: "Transfer" });
401
+ throw lastError || new Error('All fallback file sizes failed');
402
+ }
313
403
  async transferBucketFiles(sourceBucketId, targetBucketId) {
314
404
  let lastFileId;
315
405
  let transferredFiles = 0;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "appwrite-utils-cli",
3
3
  "description": "Appwrite Utility Functions to help with database management, data conversion, data import, migrations, and much more. Meant to be used as a CLI tool, I do not recommend installing this in frontend environments.",
4
- "version": "1.2.2",
4
+ "version": "1.2.3",
5
5
  "main": "src/main.ts",
6
6
  "type": "module",
7
7
  "repository": {
@@ -68,6 +68,7 @@ export class ComprehensiveTransfer {
68
68
  private results: TransferResults;
69
69
  private startTime: number;
70
70
  private tempDir: string;
71
+ private cachedMaxFileSize?: number; // Cache successful maximumFileSize for subsequent buckets
71
72
 
72
73
  constructor(private options: ComprehensiveTransferOptions) {
73
74
  this.sourceClient = getClient(
@@ -429,19 +430,8 @@ export class ComprehensiveTransfer {
429
430
  const existingBucket = allTargetBuckets.find(tb => tb.$id === bucket.$id);
430
431
 
431
432
  if (!existingBucket) {
432
- // Create bucket in target
433
- await this.targetStorage.createBucket(
434
- bucket.$id,
435
- bucket.name,
436
- bucket.$permissions,
437
- bucket.fileSecurity,
438
- bucket.enabled,
439
- bucket.maximumFileSize,
440
- bucket.allowedFileExtensions,
441
- bucket.compression as any,
442
- bucket.encryption,
443
- bucket.antivirus
444
- );
433
+ // Create bucket with fallback strategy for maximumFileSize
434
+ await this.createBucketWithFallback(bucket);
445
435
  MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
446
436
  }
447
437
 
@@ -464,6 +454,152 @@ export class ComprehensiveTransfer {
464
454
  }
465
455
  }
466
456
 
457
+ private async createBucketWithFallback(bucket: Models.Bucket): Promise<void> {
458
+ // Determine the optimal size to try first
459
+ let sizeToTry: number;
460
+
461
+ if (this.cachedMaxFileSize) {
462
+ // Use cached size if it's smaller than or equal to the bucket's original size
463
+ if (bucket.maximumFileSize >= this.cachedMaxFileSize) {
464
+ sizeToTry = this.cachedMaxFileSize;
465
+ MessageFormatter.info(
466
+ `Bucket ${bucket.name}: Using cached maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`,
467
+ { prefix: "Transfer" }
468
+ );
469
+ } else {
470
+ // Original size is smaller than cached size, try original first
471
+ sizeToTry = bucket.maximumFileSize;
472
+ }
473
+ } else {
474
+ // No cached size yet, try original size first
475
+ sizeToTry = bucket.maximumFileSize;
476
+ }
477
+
478
+ // Try the optimal size first
479
+ try {
480
+ await this.targetStorage.createBucket(
481
+ bucket.$id,
482
+ bucket.name,
483
+ bucket.$permissions,
484
+ bucket.fileSecurity,
485
+ bucket.enabled,
486
+ sizeToTry,
487
+ bucket.allowedFileExtensions,
488
+ bucket.compression as any,
489
+ bucket.encryption,
490
+ bucket.antivirus
491
+ );
492
+
493
+ // Success - cache this size if it's not already cached or is smaller than cached
494
+ if (!this.cachedMaxFileSize || sizeToTry < this.cachedMaxFileSize) {
495
+ this.cachedMaxFileSize = sizeToTry;
496
+ MessageFormatter.info(
497
+ `Bucket ${bucket.name}: Cached successful maximumFileSize ${sizeToTry} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`,
498
+ { prefix: "Transfer" }
499
+ );
500
+ }
501
+
502
+ // Log if we used a different size than original
503
+ if (sizeToTry !== bucket.maximumFileSize) {
504
+ MessageFormatter.warning(
505
+ `Bucket ${bucket.name}: maximumFileSize used ${sizeToTry} instead of original ${bucket.maximumFileSize} (${(sizeToTry / 1_000_000_000).toFixed(1)}GB)`,
506
+ { prefix: "Transfer" }
507
+ );
508
+ }
509
+
510
+ return; // Success, exit the function
511
+ } catch (error) {
512
+ const err = error instanceof Error ? error : new Error(String(error));
513
+
514
+ // Check if the error is related to maximumFileSize validation
515
+ if (err.message.includes('maximumFileSize') || err.message.includes('valid range')) {
516
+ MessageFormatter.warning(
517
+ `Bucket ${bucket.name}: Failed with maximumFileSize ${sizeToTry}, falling back to smaller sizes...`,
518
+ { prefix: "Transfer" }
519
+ );
520
+ // Continue to fallback logic below
521
+ } else {
522
+ // Different error, don't retry
523
+ throw err;
524
+ }
525
+ }
526
+
527
+ // Fallback to progressively smaller sizes
528
+ const fallbackSizes = [
529
+ 5_000_000_000, // 5GB
530
+ 2_500_000_000, // 2.5GB
531
+ 2_000_000_000, // 2GB
532
+ 1_000_000_000, // 1GB
533
+ 500_000_000, // 500MB
534
+ 100_000_000 // 100MB
535
+ ];
536
+
537
+ // Remove sizes that are larger than or equal to the already-tried size
538
+ const validSizes = fallbackSizes
539
+ .filter(size => size < sizeToTry)
540
+ .sort((a, b) => b - a); // Sort descending
541
+
542
+ let lastError: Error | null = null;
543
+
544
+ for (const fileSize of validSizes) {
545
+ try {
546
+ await this.targetStorage.createBucket(
547
+ bucket.$id,
548
+ bucket.name,
549
+ bucket.$permissions,
550
+ bucket.fileSecurity,
551
+ bucket.enabled,
552
+ fileSize,
553
+ bucket.allowedFileExtensions,
554
+ bucket.compression as any,
555
+ bucket.encryption,
556
+ bucket.antivirus
557
+ );
558
+
559
+ // Success - cache this size if it's not already cached or is smaller than cached
560
+ if (!this.cachedMaxFileSize || fileSize < this.cachedMaxFileSize) {
561
+ this.cachedMaxFileSize = fileSize;
562
+ MessageFormatter.info(
563
+ `Bucket ${bucket.name}: Cached successful maximumFileSize ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`,
564
+ { prefix: "Transfer" }
565
+ );
566
+ }
567
+
568
+ // Log if we had to reduce the file size
569
+ if (fileSize !== bucket.maximumFileSize) {
570
+ MessageFormatter.warning(
571
+ `Bucket ${bucket.name}: maximumFileSize reduced from ${bucket.maximumFileSize} to ${fileSize} (${(fileSize / 1_000_000_000).toFixed(1)}GB)`,
572
+ { prefix: "Transfer" }
573
+ );
574
+ }
575
+
576
+ return; // Success, exit the function
577
+ } catch (error) {
578
+ lastError = error instanceof Error ? error : new Error(String(error));
579
+
580
+ // Check if the error is related to maximumFileSize validation
581
+ if (lastError.message.includes('maximumFileSize') || lastError.message.includes('valid range')) {
582
+ MessageFormatter.warning(
583
+ `Bucket ${bucket.name}: Failed with maximumFileSize ${fileSize}, trying smaller size...`,
584
+ { prefix: "Transfer" }
585
+ );
586
+ continue; // Try next smaller size
587
+ } else {
588
+ // Different error, don't retry
589
+ throw lastError;
590
+ }
591
+ }
592
+ }
593
+
594
+ // If we get here, all fallback sizes failed
595
+ MessageFormatter.error(
596
+ `Bucket ${bucket.name}: All fallback file sizes failed. Last error: ${lastError?.message}`,
597
+ lastError || undefined,
598
+ { prefix: "Transfer" }
599
+ );
600
+ throw lastError || new Error('All fallback file sizes failed');
601
+ }
602
+
467
603
  private async transferBucketFiles(sourceBucketId: string, targetBucketId: string): Promise<void> {
468
604
  let lastFileId: string | undefined;
469
605
  let transferredFiles = 0;