appydave-tools 0.77.3 → 0.77.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/lib/appydave/tools/dam/s3_downloader.rb +111 -0
- data/lib/appydave/tools/dam/s3_operations.rb +1 -96
- data/lib/appydave/tools/version.rb +1 -1
- data/lib/appydave/tools.rb +1 -0
- data/package.json +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b37b062c3680912049eb80e1cb4c40a213429d80e4a80fd5a9ea687e68bda536
|
|
4
|
+
data.tar.gz: a8ee199b0664434908e0178cc21956620a87455540f459ee6e1f85560cbe02f8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1f7e6164d4e947be359dda0e27b61507b35bbe65e10073760e765b12ea755bfcb66d38784567e1404fe8474745dc34782fe61684c322e091697b4f7df94fccab
|
|
7
|
+
data.tar.gz: f12c43fe505fd799f991c6f158bc30a82df78b3676aceb6a77208d24bc443293f3e5ee32172fe18f70034d03f9ab6d4c0f5dbee6319e29e62569a6ac7847e4e2
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
## [0.77.3](https://github.com/appydave/appydave-tools/compare/v0.77.2...v0.77.3) (2026-03-20)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* extract S3Uploader from S3Operations; upload delegates to focused class ([9e0dea9](https://github.com/appydave/appydave-tools/commit/9e0dea9e1437ed2b08ed64f016fe939edc164bc6))
|
|
7
|
+
|
|
1
8
|
## [0.77.2](https://github.com/appydave/appydave-tools/compare/v0.77.1...v0.77.2) (2026-03-20)
|
|
2
9
|
|
|
3
10
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Appydave
|
|
4
|
+
module Tools
|
|
5
|
+
module Dam
|
|
6
|
+
# Handles S3 download operations.
|
|
7
|
+
# Inherits shared infrastructure and helpers from S3Base.
|
|
8
|
+
class S3Downloader < S3Base
|
|
9
|
+
def download(dry_run: false)
|
|
10
|
+
project_dir = project_directory_path
|
|
11
|
+
staging_dir = File.join(project_dir, 's3-staging')
|
|
12
|
+
|
|
13
|
+
# Ensure project directory exists before download
|
|
14
|
+
unless Dir.exist?(project_dir)
|
|
15
|
+
puts "📁 Creating project directory: #{project_id}"
|
|
16
|
+
FileUtils.mkdir_p(project_dir) unless dry_run
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
s3_files = list_s3_files
|
|
20
|
+
|
|
21
|
+
if s3_files.empty?
|
|
22
|
+
puts "❌ No files found in S3 for #{brand}/#{project_id}"
|
|
23
|
+
return
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
total_size = s3_files.sum { |f| f['Size'] || 0 }
|
|
27
|
+
puts "📦 Downloading #{s3_files.size} file(s) (#{file_size_human(total_size)}) from S3 to #{project_id}/s3-staging/..."
|
|
28
|
+
puts ''
|
|
29
|
+
|
|
30
|
+
downloaded = 0
|
|
31
|
+
skipped = 0
|
|
32
|
+
failed = 0
|
|
33
|
+
|
|
34
|
+
# rubocop:disable Metrics/BlockLength
|
|
35
|
+
s3_files.each do |s3_file|
|
|
36
|
+
key = s3_file['Key']
|
|
37
|
+
relative_path = extract_relative_path(key)
|
|
38
|
+
local_file = File.join(staging_dir, relative_path)
|
|
39
|
+
|
|
40
|
+
# Check if file already exists and compare
|
|
41
|
+
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
42
|
+
s3_size = s3_file['Size']
|
|
43
|
+
|
|
44
|
+
if File.exist?(local_file)
|
|
45
|
+
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
46
|
+
|
|
47
|
+
if match_status == :synced
|
|
48
|
+
comparison_method = multipart_etag?(s3_etag) ? 'size match' : 'unchanged'
|
|
49
|
+
puts " ⏭️ Skipped: #{relative_path} (#{comparison_method})"
|
|
50
|
+
skipped += 1
|
|
51
|
+
next
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# File exists but content differs - warn before overwriting
|
|
55
|
+
puts " ⚠️ Warning: #{relative_path} exists locally with different content"
|
|
56
|
+
puts ' (multipart upload detected - comparing by size)' if multipart_etag?(s3_etag)
|
|
57
|
+
|
|
58
|
+
if s3_file['LastModified']
|
|
59
|
+
s3_time = s3_file['LastModified']
|
|
60
|
+
local_time = File.mtime(local_file)
|
|
61
|
+
puts " S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
|
|
62
|
+
|
|
63
|
+
puts ' ⚠️ Local file is NEWER than S3 - you may be overwriting recent changes!' if local_time > s3_time
|
|
64
|
+
end
|
|
65
|
+
puts ' Downloading will overwrite local version...'
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
if download_file(key, local_file, dry_run: dry_run)
|
|
69
|
+
downloaded += 1
|
|
70
|
+
else
|
|
71
|
+
failed += 1
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
# rubocop:enable Metrics/BlockLength
|
|
75
|
+
puts ''
|
|
76
|
+
puts '✅ Download complete!'
|
|
77
|
+
puts " Downloaded: #{downloaded}, Skipped: #{skipped}, Failed: #{failed}"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def download_file(s3_key, local_file, dry_run: false)
|
|
83
|
+
if dry_run
|
|
84
|
+
puts " [DRY-RUN] Would download: s3://#{brand_info.aws.s3_bucket}/#{s3_key} → #{local_file}"
|
|
85
|
+
return true
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
FileUtils.mkdir_p(File.dirname(local_file))
|
|
89
|
+
|
|
90
|
+
start_time = Time.now
|
|
91
|
+
|
|
92
|
+
s3_client.get_object(
|
|
93
|
+
bucket: brand_info.aws.s3_bucket,
|
|
94
|
+
key: s3_key,
|
|
95
|
+
response_target: local_file
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
elapsed = Time.now - start_time
|
|
99
|
+
elapsed_str = format_duration(elapsed)
|
|
100
|
+
file_size = File.size(local_file)
|
|
101
|
+
puts " ✓ Downloaded: #{File.basename(local_file)} (#{file_size_human(file_size)}) in #{elapsed_str}"
|
|
102
|
+
true
|
|
103
|
+
rescue Aws::S3::Errors::ServiceError => e
|
|
104
|
+
puts " ✗ Failed: #{File.basename(local_file)}"
|
|
105
|
+
puts " Error: #{e.message}"
|
|
106
|
+
false
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -14,74 +14,7 @@ module Appydave
|
|
|
14
14
|
|
|
15
15
|
# Download files from S3 to s3-staging/
|
|
16
16
|
def download(dry_run: false)
|
|
17
|
-
|
|
18
|
-
staging_dir = File.join(project_dir, 's3-staging')
|
|
19
|
-
|
|
20
|
-
# Ensure project directory exists before download
|
|
21
|
-
unless Dir.exist?(project_dir)
|
|
22
|
-
puts "📁 Creating project directory: #{project_id}"
|
|
23
|
-
FileUtils.mkdir_p(project_dir) unless dry_run
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
s3_files = list_s3_files
|
|
27
|
-
|
|
28
|
-
if s3_files.empty?
|
|
29
|
-
puts "❌ No files found in S3 for #{brand}/#{project_id}"
|
|
30
|
-
return
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
total_size = s3_files.sum { |f| f['Size'] || 0 }
|
|
34
|
-
puts "📦 Downloading #{s3_files.size} file(s) (#{file_size_human(total_size)}) from S3 to #{project_id}/s3-staging/..."
|
|
35
|
-
puts ''
|
|
36
|
-
|
|
37
|
-
downloaded = 0
|
|
38
|
-
skipped = 0
|
|
39
|
-
failed = 0
|
|
40
|
-
|
|
41
|
-
# rubocop:disable Metrics/BlockLength
|
|
42
|
-
s3_files.each do |s3_file|
|
|
43
|
-
key = s3_file['Key']
|
|
44
|
-
relative_path = extract_relative_path(key)
|
|
45
|
-
local_file = File.join(staging_dir, relative_path)
|
|
46
|
-
|
|
47
|
-
# Check if file already exists and compare
|
|
48
|
-
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
49
|
-
s3_size = s3_file['Size']
|
|
50
|
-
|
|
51
|
-
if File.exist?(local_file)
|
|
52
|
-
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
53
|
-
|
|
54
|
-
if match_status == :synced
|
|
55
|
-
comparison_method = multipart_etag?(s3_etag) ? 'size match' : 'unchanged'
|
|
56
|
-
puts " ⏭️ Skipped: #{relative_path} (#{comparison_method})"
|
|
57
|
-
skipped += 1
|
|
58
|
-
next
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# File exists but content differs - warn before overwriting
|
|
62
|
-
puts " ⚠️ Warning: #{relative_path} exists locally with different content"
|
|
63
|
-
puts ' (multipart upload detected - comparing by size)' if multipart_etag?(s3_etag)
|
|
64
|
-
|
|
65
|
-
if s3_file['LastModified']
|
|
66
|
-
s3_time = s3_file['LastModified']
|
|
67
|
-
local_time = File.mtime(local_file)
|
|
68
|
-
puts " S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
|
|
69
|
-
|
|
70
|
-
puts ' ⚠️ Local file is NEWER than S3 - you may be overwriting recent changes!' if local_time > s3_time
|
|
71
|
-
end
|
|
72
|
-
puts ' Downloading will overwrite local version...'
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
if download_file(key, local_file, dry_run: dry_run)
|
|
76
|
-
downloaded += 1
|
|
77
|
-
else
|
|
78
|
-
failed += 1
|
|
79
|
-
end
|
|
80
|
-
end
|
|
81
|
-
# rubocop:enable Metrics/BlockLength
|
|
82
|
-
puts ''
|
|
83
|
-
puts '✅ Download complete!'
|
|
84
|
-
puts " Downloaded: #{downloaded}, Skipped: #{skipped}, Failed: #{failed}"
|
|
17
|
+
S3Downloader.new(brand, project_id, **delegated_opts).download(dry_run: dry_run)
|
|
85
18
|
end
|
|
86
19
|
|
|
87
20
|
# Show sync status
|
|
@@ -418,34 +351,6 @@ module Appydave
|
|
|
418
351
|
{ brand_info: brand_info, brand_path: brand_path, s3_client: @s3_client_override }
|
|
419
352
|
end
|
|
420
353
|
|
|
421
|
-
# Download file from S3
|
|
422
|
-
def download_file(s3_key, local_file, dry_run: false)
|
|
423
|
-
if dry_run
|
|
424
|
-
puts " [DRY-RUN] Would download: s3://#{brand_info.aws.s3_bucket}/#{s3_key} → #{local_file}"
|
|
425
|
-
return true
|
|
426
|
-
end
|
|
427
|
-
|
|
428
|
-
FileUtils.mkdir_p(File.dirname(local_file))
|
|
429
|
-
|
|
430
|
-
start_time = Time.now
|
|
431
|
-
|
|
432
|
-
s3_client.get_object(
|
|
433
|
-
bucket: brand_info.aws.s3_bucket,
|
|
434
|
-
key: s3_key,
|
|
435
|
-
response_target: local_file
|
|
436
|
-
)
|
|
437
|
-
|
|
438
|
-
elapsed = Time.now - start_time
|
|
439
|
-
elapsed_str = format_duration(elapsed)
|
|
440
|
-
file_size = File.size(local_file)
|
|
441
|
-
puts " ✓ Downloaded: #{File.basename(local_file)} (#{file_size_human(file_size)}) in #{elapsed_str}"
|
|
442
|
-
true
|
|
443
|
-
rescue Aws::S3::Errors::ServiceError => e
|
|
444
|
-
puts " ✗ Failed: #{File.basename(local_file)}"
|
|
445
|
-
puts " Error: #{e.message}"
|
|
446
|
-
false
|
|
447
|
-
end
|
|
448
|
-
|
|
449
354
|
# Delete file from S3
|
|
450
355
|
def delete_s3_file(s3_key, dry_run: false)
|
|
451
356
|
if dry_run
|
data/lib/appydave/tools.rb
CHANGED
|
@@ -69,6 +69,7 @@ require 'appydave/tools/dam/project_resolver'
|
|
|
69
69
|
require 'appydave/tools/dam/config_loader'
|
|
70
70
|
require 'appydave/tools/dam/s3_base'
|
|
71
71
|
require 'appydave/tools/dam/s3_uploader'
|
|
72
|
+
require 'appydave/tools/dam/s3_downloader'
|
|
72
73
|
require 'appydave/tools/dam/s3_operations'
|
|
73
74
|
require 'appydave/tools/dam/s3_scanner'
|
|
74
75
|
require 'appydave/tools/dam/share_operations'
|
data/package.json
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: appydave-tools
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.77.
|
|
4
|
+
version: 0.77.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David Cruwys
|
|
@@ -375,6 +375,7 @@ files:
|
|
|
375
375
|
- lib/appydave/tools/dam/repo_sync.rb
|
|
376
376
|
- lib/appydave/tools/dam/s3_arg_parser.rb
|
|
377
377
|
- lib/appydave/tools/dam/s3_base.rb
|
|
378
|
+
- lib/appydave/tools/dam/s3_downloader.rb
|
|
378
379
|
- lib/appydave/tools/dam/s3_operations.rb
|
|
379
380
|
- lib/appydave/tools/dam/s3_scan_command.rb
|
|
380
381
|
- lib/appydave/tools/dam/s3_scanner.rb
|