appydave-tools 0.77.3 → 0.77.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/lib/appydave/tools/dam/s3_downloader.rb +111 -0
- data/lib/appydave/tools/dam/s3_operations.rb +4 -272
- data/lib/appydave/tools/dam/s3_status_checker.rb +178 -0
- data/lib/appydave/tools/version.rb +1 -1
- data/lib/appydave/tools.rb +2 -0
- data/package.json +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a66f70a8ea85fce726ea3da189166f623c7fb1ca007f70b1b5fb75f3fd828077
|
|
4
|
+
data.tar.gz: 423a8d4309fb166084940efe6bcb7fbd3ab9d620094b006bf95194aef671719a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d89bf4f8992c1ab8630a51af56dc97d9fbc651605593be1b70510cc5dd4837472dfc55514d12d9ff0542b439ec04d0c47752a3799136b7ca4cada342d49aa023
|
|
7
|
+
data.tar.gz: 6fc875fe4006e621674af60652e5ff1241d5404f4d4bdd6e3374314e1460212bce2ec8cfc4129feab3a55ec18ae01fb33f08dc55679ad55d1efde3ed209a5bb0
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
## [0.77.4](https://github.com/appydave/appydave-tools/compare/v0.77.3...v0.77.4) (2026-03-20)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* extract S3Downloader from S3Operations; download delegates to focused class ([4d97cdf](https://github.com/appydave/appydave-tools/commit/4d97cdf18ea56ca9b658c8858efd9041dcb7ca5b))
|
|
7
|
+
|
|
8
|
+
## [0.77.3](https://github.com/appydave/appydave-tools/compare/v0.77.2...v0.77.3) (2026-03-20)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* extract S3Uploader from S3Operations; upload delegates to focused class ([9e0dea9](https://github.com/appydave/appydave-tools/commit/9e0dea9e1437ed2b08ed64f016fe939edc164bc6))
|
|
14
|
+
|
|
1
15
|
## [0.77.2](https://github.com/appydave/appydave-tools/compare/v0.77.1...v0.77.2) (2026-03-20)
|
|
2
16
|
|
|
3
17
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Appydave
|
|
4
|
+
module Tools
|
|
5
|
+
module Dam
|
|
6
|
+
# Handles S3 download operations.
|
|
7
|
+
# Inherits shared infrastructure and helpers from S3Base.
|
|
8
|
+
class S3Downloader < S3Base
|
|
9
|
+
def download(dry_run: false)
|
|
10
|
+
project_dir = project_directory_path
|
|
11
|
+
staging_dir = File.join(project_dir, 's3-staging')
|
|
12
|
+
|
|
13
|
+
# Ensure project directory exists before download
|
|
14
|
+
unless Dir.exist?(project_dir)
|
|
15
|
+
puts "📁 Creating project directory: #{project_id}"
|
|
16
|
+
FileUtils.mkdir_p(project_dir) unless dry_run
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
s3_files = list_s3_files
|
|
20
|
+
|
|
21
|
+
if s3_files.empty?
|
|
22
|
+
puts "❌ No files found in S3 for #{brand}/#{project_id}"
|
|
23
|
+
return
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
total_size = s3_files.sum { |f| f['Size'] || 0 }
|
|
27
|
+
puts "📦 Downloading #{s3_files.size} file(s) (#{file_size_human(total_size)}) from S3 to #{project_id}/s3-staging/..."
|
|
28
|
+
puts ''
|
|
29
|
+
|
|
30
|
+
downloaded = 0
|
|
31
|
+
skipped = 0
|
|
32
|
+
failed = 0
|
|
33
|
+
|
|
34
|
+
# rubocop:disable Metrics/BlockLength
|
|
35
|
+
s3_files.each do |s3_file|
|
|
36
|
+
key = s3_file['Key']
|
|
37
|
+
relative_path = extract_relative_path(key)
|
|
38
|
+
local_file = File.join(staging_dir, relative_path)
|
|
39
|
+
|
|
40
|
+
# Check if file already exists and compare
|
|
41
|
+
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
42
|
+
s3_size = s3_file['Size']
|
|
43
|
+
|
|
44
|
+
if File.exist?(local_file)
|
|
45
|
+
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
46
|
+
|
|
47
|
+
if match_status == :synced
|
|
48
|
+
comparison_method = multipart_etag?(s3_etag) ? 'size match' : 'unchanged'
|
|
49
|
+
puts " ⏭️ Skipped: #{relative_path} (#{comparison_method})"
|
|
50
|
+
skipped += 1
|
|
51
|
+
next
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# File exists but content differs - warn before overwriting
|
|
55
|
+
puts " ⚠️ Warning: #{relative_path} exists locally with different content"
|
|
56
|
+
puts ' (multipart upload detected - comparing by size)' if multipart_etag?(s3_etag)
|
|
57
|
+
|
|
58
|
+
if s3_file['LastModified']
|
|
59
|
+
s3_time = s3_file['LastModified']
|
|
60
|
+
local_time = File.mtime(local_file)
|
|
61
|
+
puts " S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
|
|
62
|
+
|
|
63
|
+
puts ' ⚠️ Local file is NEWER than S3 - you may be overwriting recent changes!' if local_time > s3_time
|
|
64
|
+
end
|
|
65
|
+
puts ' Downloading will overwrite local version...'
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
if download_file(key, local_file, dry_run: dry_run)
|
|
69
|
+
downloaded += 1
|
|
70
|
+
else
|
|
71
|
+
failed += 1
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
# rubocop:enable Metrics/BlockLength
|
|
75
|
+
puts ''
|
|
76
|
+
puts '✅ Download complete!'
|
|
77
|
+
puts " Downloaded: #{downloaded}, Skipped: #{skipped}, Failed: #{failed}"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def download_file(s3_key, local_file, dry_run: false)
|
|
83
|
+
if dry_run
|
|
84
|
+
puts " [DRY-RUN] Would download: s3://#{brand_info.aws.s3_bucket}/#{s3_key} → #{local_file}"
|
|
85
|
+
return true
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
FileUtils.mkdir_p(File.dirname(local_file))
|
|
89
|
+
|
|
90
|
+
start_time = Time.now
|
|
91
|
+
|
|
92
|
+
s3_client.get_object(
|
|
93
|
+
bucket: brand_info.aws.s3_bucket,
|
|
94
|
+
key: s3_key,
|
|
95
|
+
response_target: local_file
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
elapsed = Time.now - start_time
|
|
99
|
+
elapsed_str = format_duration(elapsed)
|
|
100
|
+
file_size = File.size(local_file)
|
|
101
|
+
puts " ✓ Downloaded: #{File.basename(local_file)} (#{file_size_human(file_size)}) in #{elapsed_str}"
|
|
102
|
+
true
|
|
103
|
+
rescue Aws::S3::Errors::ServiceError => e
|
|
104
|
+
puts " ✗ Failed: #{File.basename(local_file)}"
|
|
105
|
+
puts " Error: #{e.message}"
|
|
106
|
+
false
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -14,168 +14,12 @@ module Appydave
|
|
|
14
14
|
|
|
15
15
|
# Download files from S3 to s3-staging/
|
|
16
16
|
def download(dry_run: false)
|
|
17
|
-
|
|
18
|
-
staging_dir = File.join(project_dir, 's3-staging')
|
|
19
|
-
|
|
20
|
-
# Ensure project directory exists before download
|
|
21
|
-
unless Dir.exist?(project_dir)
|
|
22
|
-
puts "📁 Creating project directory: #{project_id}"
|
|
23
|
-
FileUtils.mkdir_p(project_dir) unless dry_run
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
s3_files = list_s3_files
|
|
27
|
-
|
|
28
|
-
if s3_files.empty?
|
|
29
|
-
puts "❌ No files found in S3 for #{brand}/#{project_id}"
|
|
30
|
-
return
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
total_size = s3_files.sum { |f| f['Size'] || 0 }
|
|
34
|
-
puts "📦 Downloading #{s3_files.size} file(s) (#{file_size_human(total_size)}) from S3 to #{project_id}/s3-staging/..."
|
|
35
|
-
puts ''
|
|
36
|
-
|
|
37
|
-
downloaded = 0
|
|
38
|
-
skipped = 0
|
|
39
|
-
failed = 0
|
|
40
|
-
|
|
41
|
-
# rubocop:disable Metrics/BlockLength
|
|
42
|
-
s3_files.each do |s3_file|
|
|
43
|
-
key = s3_file['Key']
|
|
44
|
-
relative_path = extract_relative_path(key)
|
|
45
|
-
local_file = File.join(staging_dir, relative_path)
|
|
46
|
-
|
|
47
|
-
# Check if file already exists and compare
|
|
48
|
-
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
49
|
-
s3_size = s3_file['Size']
|
|
50
|
-
|
|
51
|
-
if File.exist?(local_file)
|
|
52
|
-
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
53
|
-
|
|
54
|
-
if match_status == :synced
|
|
55
|
-
comparison_method = multipart_etag?(s3_etag) ? 'size match' : 'unchanged'
|
|
56
|
-
puts " ⏭️ Skipped: #{relative_path} (#{comparison_method})"
|
|
57
|
-
skipped += 1
|
|
58
|
-
next
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# File exists but content differs - warn before overwriting
|
|
62
|
-
puts " ⚠️ Warning: #{relative_path} exists locally with different content"
|
|
63
|
-
puts ' (multipart upload detected - comparing by size)' if multipart_etag?(s3_etag)
|
|
64
|
-
|
|
65
|
-
if s3_file['LastModified']
|
|
66
|
-
s3_time = s3_file['LastModified']
|
|
67
|
-
local_time = File.mtime(local_file)
|
|
68
|
-
puts " S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
|
|
69
|
-
|
|
70
|
-
puts ' ⚠️ Local file is NEWER than S3 - you may be overwriting recent changes!' if local_time > s3_time
|
|
71
|
-
end
|
|
72
|
-
puts ' Downloading will overwrite local version...'
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
if download_file(key, local_file, dry_run: dry_run)
|
|
76
|
-
downloaded += 1
|
|
77
|
-
else
|
|
78
|
-
failed += 1
|
|
79
|
-
end
|
|
80
|
-
end
|
|
81
|
-
# rubocop:enable Metrics/BlockLength
|
|
82
|
-
puts ''
|
|
83
|
-
puts '✅ Download complete!'
|
|
84
|
-
puts " Downloaded: #{downloaded}, Skipped: #{skipped}, Failed: #{failed}"
|
|
17
|
+
S3Downloader.new(brand, project_id, **delegated_opts).download(dry_run: dry_run)
|
|
85
18
|
end
|
|
86
19
|
|
|
87
20
|
# Show sync status
|
|
88
21
|
def status
|
|
89
|
-
|
|
90
|
-
staging_dir = File.join(project_dir, 's3-staging')
|
|
91
|
-
|
|
92
|
-
# Check if project directory exists
|
|
93
|
-
unless Dir.exist?(project_dir)
|
|
94
|
-
puts "❌ Project not found: #{brand}/#{project_id}"
|
|
95
|
-
puts ''
|
|
96
|
-
puts ' This project does not exist locally.'
|
|
97
|
-
puts ' Possible causes:'
|
|
98
|
-
puts ' - Project name might be misspelled'
|
|
99
|
-
puts ' - Project may not exist in this brand'
|
|
100
|
-
puts ''
|
|
101
|
-
puts " Try: dam list #{brand} # See all projects for this brand"
|
|
102
|
-
return
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
s3_files = list_s3_files
|
|
106
|
-
local_files = list_local_files(staging_dir)
|
|
107
|
-
|
|
108
|
-
# Build a map of S3 files for quick lookup
|
|
109
|
-
s3_files_map = s3_files.each_with_object({}) do |file, hash|
|
|
110
|
-
relative_path = extract_relative_path(file['Key'])
|
|
111
|
-
hash[relative_path] = file
|
|
112
|
-
end
|
|
113
|
-
|
|
114
|
-
if s3_files.empty? && local_files.empty?
|
|
115
|
-
puts "ℹ️ No files in S3 or s3-staging/ for #{brand}/#{project_id}"
|
|
116
|
-
puts ''
|
|
117
|
-
puts ' This project exists but has no heavy files ready for S3 sync.'
|
|
118
|
-
puts ''
|
|
119
|
-
puts ' Next steps:'
|
|
120
|
-
puts " 1. Add video files to: #{staging_dir}/"
|
|
121
|
-
puts " 2. Upload to S3: dam s3-up #{brand} #{project_id}"
|
|
122
|
-
return
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
puts "📊 S3 Sync Status for #{brand}/#{project_id}"
|
|
126
|
-
|
|
127
|
-
# Show last sync time
|
|
128
|
-
if s3_files.any?
|
|
129
|
-
most_recent = s3_files.map { |f| f['LastModified'] }.compact.max
|
|
130
|
-
if most_recent
|
|
131
|
-
time_ago = format_time_ago(Time.now - most_recent)
|
|
132
|
-
puts " Last synced: #{time_ago} ago (#{most_recent.strftime('%Y-%m-%d %H:%M')})"
|
|
133
|
-
end
|
|
134
|
-
end
|
|
135
|
-
puts ''
|
|
136
|
-
|
|
137
|
-
# Combine all file paths (S3 + local)
|
|
138
|
-
all_paths = (s3_files_map.keys + local_files.keys).uniq.sort
|
|
139
|
-
|
|
140
|
-
total_s3_size = 0
|
|
141
|
-
total_local_size = 0
|
|
142
|
-
|
|
143
|
-
all_paths.each do |relative_path|
|
|
144
|
-
s3_file = s3_files_map[relative_path]
|
|
145
|
-
local_file = File.join(staging_dir, relative_path)
|
|
146
|
-
|
|
147
|
-
if s3_file && File.exist?(local_file)
|
|
148
|
-
# File exists in both S3 and local
|
|
149
|
-
s3_size = s3_file['Size']
|
|
150
|
-
local_size = File.size(local_file)
|
|
151
|
-
total_s3_size += s3_size
|
|
152
|
-
total_local_size += local_size
|
|
153
|
-
|
|
154
|
-
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
155
|
-
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
156
|
-
|
|
157
|
-
if match_status == :synced
|
|
158
|
-
status_label = multipart_etag?(s3_etag) ? 'synced*' : 'synced'
|
|
159
|
-
puts " ✓ #{relative_path} (#{file_size_human(s3_size)}) [#{status_label}]"
|
|
160
|
-
else
|
|
161
|
-
puts " ⚠️ #{relative_path} (#{file_size_human(s3_size)}) [modified]"
|
|
162
|
-
end
|
|
163
|
-
elsif s3_file
|
|
164
|
-
# File only in S3
|
|
165
|
-
s3_size = s3_file['Size']
|
|
166
|
-
total_s3_size += s3_size
|
|
167
|
-
puts " ☁️ #{relative_path} (#{file_size_human(s3_size)}) [S3 only]"
|
|
168
|
-
else
|
|
169
|
-
# File only local
|
|
170
|
-
local_size = File.size(local_file)
|
|
171
|
-
total_local_size += local_size
|
|
172
|
-
puts " 📁 #{relative_path} (#{file_size_human(local_size)}) [local only]"
|
|
173
|
-
end
|
|
174
|
-
end
|
|
175
|
-
|
|
176
|
-
puts ''
|
|
177
|
-
puts "S3 files: #{s3_files.size}, Local files: #{local_files.size}"
|
|
178
|
-
puts "S3 size: #{file_size_human(total_s3_size)}, Local size: #{file_size_human(total_local_size)}"
|
|
22
|
+
S3StatusChecker.new(brand, project_id, **delegated_opts).status
|
|
179
23
|
end
|
|
180
24
|
|
|
181
25
|
# Cleanup S3 files
|
|
@@ -319,97 +163,13 @@ module Appydave
|
|
|
319
163
|
# Calculate 3-state S3 sync status
|
|
320
164
|
# @return [String] One of: '↑ upload', '↓ download', '✓ synced', 'none'
|
|
321
165
|
def calculate_sync_status
|
|
322
|
-
|
|
323
|
-
staging_dir = File.join(project_dir, 's3-staging')
|
|
324
|
-
|
|
325
|
-
# No s3-staging directory means no S3 intent
|
|
326
|
-
return 'none' unless Dir.exist?(staging_dir)
|
|
327
|
-
|
|
328
|
-
# Get S3 files (if S3 configured)
|
|
329
|
-
begin
|
|
330
|
-
s3_files = list_s3_files
|
|
331
|
-
rescue StandardError
|
|
332
|
-
# S3 not configured or not accessible
|
|
333
|
-
return 'none'
|
|
334
|
-
end
|
|
335
|
-
|
|
336
|
-
local_files = list_local_files(staging_dir)
|
|
337
|
-
|
|
338
|
-
# No files anywhere
|
|
339
|
-
return 'none' if s3_files.empty? && local_files.empty?
|
|
340
|
-
|
|
341
|
-
# Build S3 files map
|
|
342
|
-
s3_files_map = s3_files.each_with_object({}) do |file, hash|
|
|
343
|
-
relative_path = extract_relative_path(file['Key'])
|
|
344
|
-
hash[relative_path] = file
|
|
345
|
-
end
|
|
346
|
-
|
|
347
|
-
# Check for differences
|
|
348
|
-
needs_upload = false
|
|
349
|
-
needs_download = false
|
|
350
|
-
|
|
351
|
-
# Check all local files
|
|
352
|
-
local_files.each_key do |relative_path|
|
|
353
|
-
local_file = File.join(staging_dir, relative_path)
|
|
354
|
-
s3_file = s3_files_map[relative_path]
|
|
355
|
-
|
|
356
|
-
if s3_file
|
|
357
|
-
# Compare using multipart-aware comparison
|
|
358
|
-
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
359
|
-
s3_size = s3_file['Size']
|
|
360
|
-
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
361
|
-
needs_upload = true if match_status != :synced
|
|
362
|
-
else
|
|
363
|
-
# Local file not in S3
|
|
364
|
-
needs_upload = true
|
|
365
|
-
end
|
|
366
|
-
end
|
|
367
|
-
|
|
368
|
-
# Check for S3-only files
|
|
369
|
-
s3_files_map.each_key do |relative_path|
|
|
370
|
-
local_file = File.join(staging_dir, relative_path)
|
|
371
|
-
needs_download = true unless File.exist?(local_file)
|
|
372
|
-
end
|
|
373
|
-
|
|
374
|
-
# Return status based on what's needed
|
|
375
|
-
if needs_upload && needs_download
|
|
376
|
-
'⚠️ both'
|
|
377
|
-
elsif needs_upload
|
|
378
|
-
'↑ upload'
|
|
379
|
-
elsif needs_download
|
|
380
|
-
'↓ download'
|
|
381
|
-
else
|
|
382
|
-
'✓ synced'
|
|
383
|
-
end
|
|
166
|
+
S3StatusChecker.new(brand, project_id, **delegated_opts).calculate_sync_status
|
|
384
167
|
end
|
|
385
168
|
|
|
386
169
|
# Calculate S3 sync timestamps (last upload/download times)
|
|
387
170
|
# @return [Hash] { last_upload: Time|nil, last_download: Time|nil }
|
|
388
171
|
def sync_timestamps
|
|
389
|
-
|
|
390
|
-
staging_dir = File.join(project_dir, 's3-staging')
|
|
391
|
-
|
|
392
|
-
# No s3-staging directory means no S3 intent
|
|
393
|
-
return { last_upload: nil, last_download: nil } unless Dir.exist?(staging_dir)
|
|
394
|
-
|
|
395
|
-
# Get S3 files (if S3 configured)
|
|
396
|
-
begin
|
|
397
|
-
s3_files = list_s3_files
|
|
398
|
-
rescue StandardError
|
|
399
|
-
# S3 not configured or not accessible
|
|
400
|
-
return { last_upload: nil, last_download: nil }
|
|
401
|
-
end
|
|
402
|
-
|
|
403
|
-
# Last upload time = most recent S3 file LastModified
|
|
404
|
-
last_upload = s3_files.map { |f| f['LastModified'] }.compact.max if s3_files.any?
|
|
405
|
-
|
|
406
|
-
# Last download time = most recent local file mtime (in s3-staging)
|
|
407
|
-
last_download = if Dir.exist?(staging_dir)
|
|
408
|
-
local_files = Dir.glob(File.join(staging_dir, '**/*')).select { |f| File.file?(f) }
|
|
409
|
-
local_files.map { |f| File.mtime(f) }.max if local_files.any?
|
|
410
|
-
end
|
|
411
|
-
|
|
412
|
-
{ last_upload: last_upload, last_download: last_download }
|
|
172
|
+
S3StatusChecker.new(brand, project_id, **delegated_opts).sync_timestamps
|
|
413
173
|
end
|
|
414
174
|
|
|
415
175
|
private
|
|
@@ -418,34 +178,6 @@ module Appydave
|
|
|
418
178
|
{ brand_info: brand_info, brand_path: brand_path, s3_client: @s3_client_override }
|
|
419
179
|
end
|
|
420
180
|
|
|
421
|
-
# Download file from S3
|
|
422
|
-
def download_file(s3_key, local_file, dry_run: false)
|
|
423
|
-
if dry_run
|
|
424
|
-
puts " [DRY-RUN] Would download: s3://#{brand_info.aws.s3_bucket}/#{s3_key} → #{local_file}"
|
|
425
|
-
return true
|
|
426
|
-
end
|
|
427
|
-
|
|
428
|
-
FileUtils.mkdir_p(File.dirname(local_file))
|
|
429
|
-
|
|
430
|
-
start_time = Time.now
|
|
431
|
-
|
|
432
|
-
s3_client.get_object(
|
|
433
|
-
bucket: brand_info.aws.s3_bucket,
|
|
434
|
-
key: s3_key,
|
|
435
|
-
response_target: local_file
|
|
436
|
-
)
|
|
437
|
-
|
|
438
|
-
elapsed = Time.now - start_time
|
|
439
|
-
elapsed_str = format_duration(elapsed)
|
|
440
|
-
file_size = File.size(local_file)
|
|
441
|
-
puts " ✓ Downloaded: #{File.basename(local_file)} (#{file_size_human(file_size)}) in #{elapsed_str}"
|
|
442
|
-
true
|
|
443
|
-
rescue Aws::S3::Errors::ServiceError => e
|
|
444
|
-
puts " ✗ Failed: #{File.basename(local_file)}"
|
|
445
|
-
puts " Error: #{e.message}"
|
|
446
|
-
false
|
|
447
|
-
end
|
|
448
|
-
|
|
449
181
|
# Delete file from S3
|
|
450
182
|
def delete_s3_file(s3_key, dry_run: false)
|
|
451
183
|
if dry_run
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Appydave
|
|
4
|
+
module Tools
|
|
5
|
+
module Dam
|
|
6
|
+
# Handles S3 status and sync state operations.
|
|
7
|
+
# Inherits shared infrastructure and helpers from S3Base.
|
|
8
|
+
class S3StatusChecker < S3Base
|
|
9
|
+
# Show sync status
|
|
10
|
+
def status
|
|
11
|
+
project_dir = project_directory_path
|
|
12
|
+
staging_dir = File.join(project_dir, 's3-staging')
|
|
13
|
+
|
|
14
|
+
unless Dir.exist?(project_dir)
|
|
15
|
+
puts "❌ Project not found: #{brand}/#{project_id}"
|
|
16
|
+
puts ''
|
|
17
|
+
puts ' This project does not exist locally.'
|
|
18
|
+
puts ' Possible causes:'
|
|
19
|
+
puts ' - Project name might be misspelled'
|
|
20
|
+
puts ' - Project may not exist in this brand'
|
|
21
|
+
puts ''
|
|
22
|
+
puts " Try: dam list #{brand} # See all projects for this brand"
|
|
23
|
+
return
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
s3_files = list_s3_files
|
|
27
|
+
local_files = list_local_files(staging_dir)
|
|
28
|
+
|
|
29
|
+
s3_files_map = s3_files.each_with_object({}) do |file, hash|
|
|
30
|
+
relative_path = extract_relative_path(file['Key'])
|
|
31
|
+
hash[relative_path] = file
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
if s3_files.empty? && local_files.empty?
|
|
35
|
+
puts "ℹ️ No files in S3 or s3-staging/ for #{brand}/#{project_id}"
|
|
36
|
+
puts ''
|
|
37
|
+
puts ' This project exists but has no heavy files ready for S3 sync.'
|
|
38
|
+
puts ''
|
|
39
|
+
puts ' Next steps:'
|
|
40
|
+
puts " 1. Add video files to: #{staging_dir}/"
|
|
41
|
+
puts " 2. Upload to S3: dam s3-up #{brand} #{project_id}"
|
|
42
|
+
return
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
puts "📊 S3 Sync Status for #{brand}/#{project_id}"
|
|
46
|
+
|
|
47
|
+
if s3_files.any?
|
|
48
|
+
most_recent = s3_files.map { |f| f['LastModified'] }.compact.max
|
|
49
|
+
if most_recent
|
|
50
|
+
time_ago = format_time_ago(Time.now - most_recent)
|
|
51
|
+
puts " Last synced: #{time_ago} ago (#{most_recent.strftime('%Y-%m-%d %H:%M')})"
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
puts ''
|
|
55
|
+
|
|
56
|
+
all_paths = (s3_files_map.keys + local_files.keys).uniq.sort
|
|
57
|
+
|
|
58
|
+
total_s3_size = 0
|
|
59
|
+
total_local_size = 0
|
|
60
|
+
|
|
61
|
+
all_paths.each do |relative_path|
|
|
62
|
+
s3_file = s3_files_map[relative_path]
|
|
63
|
+
local_file = File.join(staging_dir, relative_path)
|
|
64
|
+
|
|
65
|
+
if s3_file && File.exist?(local_file)
|
|
66
|
+
s3_size = s3_file['Size']
|
|
67
|
+
local_size = File.size(local_file)
|
|
68
|
+
total_s3_size += s3_size
|
|
69
|
+
total_local_size += local_size
|
|
70
|
+
|
|
71
|
+
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
72
|
+
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
73
|
+
|
|
74
|
+
if match_status == :synced
|
|
75
|
+
status_label = multipart_etag?(s3_etag) ? 'synced*' : 'synced'
|
|
76
|
+
puts " ✓ #{relative_path} (#{file_size_human(s3_size)}) [#{status_label}]"
|
|
77
|
+
else
|
|
78
|
+
puts " ⚠️ #{relative_path} (#{file_size_human(s3_size)}) [modified]"
|
|
79
|
+
end
|
|
80
|
+
elsif s3_file
|
|
81
|
+
s3_size = s3_file['Size']
|
|
82
|
+
total_s3_size += s3_size
|
|
83
|
+
puts " ☁️ #{relative_path} (#{file_size_human(s3_size)}) [S3 only]"
|
|
84
|
+
else
|
|
85
|
+
local_size = File.size(local_file)
|
|
86
|
+
total_local_size += local_size
|
|
87
|
+
puts " 📁 #{relative_path} (#{file_size_human(local_size)}) [local only]"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
puts ''
|
|
92
|
+
puts "S3 files: #{s3_files.size}, Local files: #{local_files.size}"
|
|
93
|
+
puts "S3 size: #{file_size_human(total_s3_size)}, Local size: #{file_size_human(total_local_size)}"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Calculate 3-state S3 sync status
|
|
97
|
+
# @return [String] One of: '↑ upload', '↓ download', '✓ synced', 'none'
|
|
98
|
+
def calculate_sync_status
|
|
99
|
+
project_dir = project_directory_path
|
|
100
|
+
staging_dir = File.join(project_dir, 's3-staging')
|
|
101
|
+
|
|
102
|
+
return 'none' unless Dir.exist?(staging_dir)
|
|
103
|
+
|
|
104
|
+
begin
|
|
105
|
+
s3_files = list_s3_files
|
|
106
|
+
rescue StandardError
|
|
107
|
+
return 'none'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
local_files = list_local_files(staging_dir)
|
|
111
|
+
|
|
112
|
+
return 'none' if s3_files.empty? && local_files.empty?
|
|
113
|
+
|
|
114
|
+
s3_files_map = s3_files.each_with_object({}) do |file, hash|
|
|
115
|
+
relative_path = extract_relative_path(file['Key'])
|
|
116
|
+
hash[relative_path] = file
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
needs_upload = false
|
|
120
|
+
needs_download = false
|
|
121
|
+
|
|
122
|
+
local_files.each_key do |relative_path|
|
|
123
|
+
local_file = File.join(staging_dir, relative_path)
|
|
124
|
+
s3_file = s3_files_map[relative_path]
|
|
125
|
+
|
|
126
|
+
if s3_file
|
|
127
|
+
s3_etag = s3_file['ETag'].gsub('"', '')
|
|
128
|
+
s3_size = s3_file['Size']
|
|
129
|
+
match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
|
|
130
|
+
needs_upload = true if match_status != :synced
|
|
131
|
+
else
|
|
132
|
+
needs_upload = true
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
s3_files_map.each_key do |relative_path|
|
|
137
|
+
local_file = File.join(staging_dir, relative_path)
|
|
138
|
+
needs_download = true unless File.exist?(local_file)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
if needs_upload && needs_download
|
|
142
|
+
'⚠️ both'
|
|
143
|
+
elsif needs_upload
|
|
144
|
+
'↑ upload'
|
|
145
|
+
elsif needs_download
|
|
146
|
+
'↓ download'
|
|
147
|
+
else
|
|
148
|
+
'✓ synced'
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Calculate S3 sync timestamps (last upload/download times)
|
|
153
|
+
# @return [Hash] { last_upload: Time|nil, last_download: Time|nil }
|
|
154
|
+
def sync_timestamps
|
|
155
|
+
project_dir = project_directory_path
|
|
156
|
+
staging_dir = File.join(project_dir, 's3-staging')
|
|
157
|
+
|
|
158
|
+
return { last_upload: nil, last_download: nil } unless Dir.exist?(staging_dir)
|
|
159
|
+
|
|
160
|
+
begin
|
|
161
|
+
s3_files = list_s3_files
|
|
162
|
+
rescue StandardError
|
|
163
|
+
return { last_upload: nil, last_download: nil }
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
last_upload = s3_files.map { |f| f['LastModified'] }.compact.max if s3_files.any?
|
|
167
|
+
|
|
168
|
+
last_download = if Dir.exist?(staging_dir)
|
|
169
|
+
local_files = Dir.glob(File.join(staging_dir, '**/*')).select { |f| File.file?(f) }
|
|
170
|
+
local_files.map { |f| File.mtime(f) }.max if local_files.any?
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
{ last_upload: last_upload, last_download: last_download }
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
data/lib/appydave/tools.rb
CHANGED
|
@@ -69,6 +69,8 @@ require 'appydave/tools/dam/project_resolver'
|
|
|
69
69
|
require 'appydave/tools/dam/config_loader'
|
|
70
70
|
require 'appydave/tools/dam/s3_base'
|
|
71
71
|
require 'appydave/tools/dam/s3_uploader'
|
|
72
|
+
require 'appydave/tools/dam/s3_downloader'
|
|
73
|
+
require 'appydave/tools/dam/s3_status_checker'
|
|
72
74
|
require 'appydave/tools/dam/s3_operations'
|
|
73
75
|
require 'appydave/tools/dam/s3_scanner'
|
|
74
76
|
require 'appydave/tools/dam/share_operations'
|
data/package.json
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: appydave-tools
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.77.
|
|
4
|
+
version: 0.77.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David Cruwys
|
|
@@ -375,9 +375,11 @@ files:
|
|
|
375
375
|
- lib/appydave/tools/dam/repo_sync.rb
|
|
376
376
|
- lib/appydave/tools/dam/s3_arg_parser.rb
|
|
377
377
|
- lib/appydave/tools/dam/s3_base.rb
|
|
378
|
+
- lib/appydave/tools/dam/s3_downloader.rb
|
|
378
379
|
- lib/appydave/tools/dam/s3_operations.rb
|
|
379
380
|
- lib/appydave/tools/dam/s3_scan_command.rb
|
|
380
381
|
- lib/appydave/tools/dam/s3_scanner.rb
|
|
382
|
+
- lib/appydave/tools/dam/s3_status_checker.rb
|
|
381
383
|
- lib/appydave/tools/dam/s3_uploader.rb
|
|
382
384
|
- lib/appydave/tools/dam/share_operations.rb
|
|
383
385
|
- lib/appydave/tools/dam/ssd_status.rb
|