appydave-tools 0.69.0 → 0.70.0

data/lib/appydave/tools/dam/project_listing.rb

@@ -11,7 +11,7 @@ module Appydave
       class ProjectListing
         # List all brands with summary table
         # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        def self.list_brands_with_counts(detailed: false)
+        def self.list_brands_with_counts(detailed: false, s3: false)
           brands = Config.available_brands
 
           if brands.empty?
@@ -19,64 +19,113 @@ module Appydave
             return
           end
 
-          # Gather brand data
-          brand_data = brands.map { |brand| collect_brand_data(brand, detailed: detailed) }
+          # Gather brand data (skip S3 if not requested)
+          brand_data = brands.map { |brand| collect_brand_data(brand, detailed: detailed, s3: s3) }
 
           if detailed
             # Detailed view with additional columns
-            header = 'BRAND                              KEY          PROJECTS         SIZE        LAST MODIFIED    ' \
-                     'GIT              S3 SYNC      PATH                                      SSD BACKUP                       ' \
-                     'WORKFLOW     ACTIVE'
-            puts header
-            puts '-' * 200
+            if s3
+              header = 'BRAND                              KEY          PROJECTS         SIZE        LAST MODIFIED    ' \
+                       'GIT              S3 SYNC      PATH                                      SSD BACKUP                       ' \
+                       'WORKFLOW     ACTIVE'
+              puts header
+              puts '-' * 200
+            else
+              header = 'BRAND                              KEY          PROJECTS         SIZE        LAST MODIFIED    ' \
+                       'GIT              PATH                                      SSD BACKUP                       ' \
+                       'WORKFLOW     ACTIVE'
+              puts header
+              puts '-' * 189
+            end
 
             brand_data.each do |data|
               brand_display = "#{data[:shortcut]} - #{data[:name]}"
 
-              puts format(
-                '%-30s %-12s %10d %12s %20s    %-15s  %-10s  %-35s  %-30s  %-10s  %6d',
-                brand_display,
-                data[:key],
-                data[:count],
-                format_size(data[:size]),
-                format_date(data[:modified]),
-                data[:git_status],
-                data[:s3_sync],
-                shorten_path(data[:path]),
-                data[:ssd_backup] || 'N/A',
-                data[:workflow] || 'N/A',
-                data[:active_count] || 0
-              )
+              if s3
+                puts format(
+                  '%-30s %-12s %10d %12s %20s    %-15s  %-10s  %-35s  %-30s  %-10s  %6d',
+                  brand_display,
+                  data[:key],
+                  data[:count],
+                  format_size(data[:size]),
+                  format_date(data[:modified]),
+                  data[:git_status],
+                  data[:s3_sync],
+                  shorten_path(data[:path]),
+                  data[:ssd_backup] || 'N/A',
+                  data[:workflow] || 'N/A',
+                  data[:active_count] || 0
+                )
+              else
+                puts format(
+                  '%-30s %-12s %10d %12s %20s    %-15s  %-35s  %-30s  %-10s  %6d',
+                  brand_display,
+                  data[:key],
+                  data[:count],
+                  format_size(data[:size]),
+                  format_date(data[:modified]),
+                  data[:git_status],
+                  shorten_path(data[:path]),
+                  data[:ssd_backup] || 'N/A',
+                  data[:workflow] || 'N/A',
+                  data[:active_count] || 0
+                )
+              end
             end
           else
             # Default view - use same format for header and data
             # rubocop:disable Style/RedundantFormat
-            puts format(
-              '%-30s %-15s %10s %12s %20s    %-15s  %-10s',
-              'BRAND',
-              'KEY',
-              'PROJECTS',
-              'SIZE',
-              'LAST MODIFIED',
-              'GIT',
-              'S3 SYNC'
-            )
+            if s3
+              puts format(
+                '%-30s %-15s %10s %12s %20s    %-15s  %-10s',
+                'BRAND',
+                'KEY',
+                'PROJECTS',
+                'SIZE',
+                'LAST MODIFIED',
+                'GIT',
+                'S3 SYNC'
+              )
+              puts '-' * 133
+            else
+              puts format(
+                '%-30s %-15s %10s %12s %20s    %-15s',
+                'BRAND',
+                'KEY',
+                'PROJECTS',
+                'SIZE',
+                'LAST MODIFIED',
+                'GIT'
+              )
+              puts '-' * 122
+            end
             # rubocop:enable Style/RedundantFormat
-            puts '-' * 133
 
             brand_data.each do |data|
               brand_display = "#{data[:shortcut]} - #{data[:name]}"
 
-              puts format(
-                '%-30s %-15s %10d %12s %20s    %-15s  %-10s',
-                brand_display,
-                data[:key],
-                data[:count],
-                format_size(data[:size]),
-                format_date(data[:modified]),
-                data[:git_status],
-                data[:s3_sync]
-              )
+              if s3
+                puts format(
+                  '%-30s %-15s %10d %12s %20s    %-15s  %-10s',
+                  brand_display,
+                  data[:key],
+                  data[:count],
+                  format_size(data[:size]),
+                  format_date(data[:modified]),
+                  data[:git_status],
+                  data[:s3_sync]
+                )
+              else
+                puts format(
+                  '%-30s %-15s %10d %12s %20s    %-15s',
+                  brand_display,
+                  data[:key],
+                  data[:count],
+                  format_size(data[:size]),
+                  format_date(data[:modified]),
+                  data[:git_status]
+                )
+              end
             end
           end
 
@@ -93,7 +142,7 @@ module Appydave
 
         # List all projects for a specific brand (Mode 3)
         # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        def self.list_brand_projects(brand_arg, detailed: false)
+        def self.list_brand_projects(brand_arg, detailed: false, s3: false)
           # ProjectResolver expects the original brand key/shortcut, not the expanded v-* version
           projects = ProjectResolver.list_projects(brand_arg)
 
@@ -114,13 +163,13 @@ module Appydave
             return
           end
 
-          # Gather project data
+          # Gather project data (skip S3 if not requested)
           brand_path = Config.brand_path(brand_arg)
           brand_info = Appydave::Tools::Configuration::Config.brands.get_brand(brand_arg)
           is_git_repo = Dir.exist?(File.join(brand_path, '.git'))
 
           project_data = projects.map do |project|
-            collect_project_data(brand_arg, brand_path, brand_info, project, is_git_repo, detailed: detailed)
+            collect_project_data(brand_arg, brand_path, brand_info, project, is_git_repo, detailed: detailed, s3: s3)
           end
 
           # Print common header
@@ -132,67 +181,119 @@ module Appydave
           if detailed
             # Detailed view with additional columns - use same format for header and data
             # rubocop:disable Style/RedundantFormat
-            puts format(
-              '%-45s %12s %15s  %-15s  %-12s  %-65s  %-18s  %-18s  %-30s  %-15s  %-15s',
-              'PROJECT',
-              'SIZE',
-              'AGE',
-              'GIT',
-              'S3',
-              'PATH',
-              'HEAVY FILES',
-              'LIGHT FILES',
-              'SSD BACKUP',
-              'S3 ↑ UPLOAD',
-              'S3 ↓ DOWNLOAD'
-            )
+            if s3
+              puts format(
+                '%-45s %12s %15s  %-15s  %-12s  %-65s  %-18s  %-18s  %-30s  %-15s  %-15s',
+                'PROJECT',
+                'SIZE',
+                'AGE',
+                'GIT',
+                'S3',
+                'PATH',
+                'HEAVY FILES',
+                'LIGHT FILES',
+                'SSD BACKUP',
+                'S3 ↑ UPLOAD',
+                'S3 ↓ DOWNLOAD'
+              )
+              puts '-' * 280
+            else
+              puts format(
+                '%-45s %12s %15s  %-15s  %-65s  %-18s  %-18s  %-30s',
+                'PROJECT',
+                'SIZE',
+                'AGE',
+                'GIT',
+                'PATH',
+                'HEAVY FILES',
+                'LIGHT FILES',
+                'SSD BACKUP'
+              )
+              puts '-' * 239
+            end
             # rubocop:enable Style/RedundantFormat
-            puts '-' * 280
 
             project_data.each do |data|
               age_display = data[:stale] ? "#{data[:age]} ⚠️" : data[:age]
-              s3_upload = data[:s3_last_upload] ? format_age(data[:s3_last_upload]) : 'N/A'
-              s3_download = data[:s3_last_download] ? format_age(data[:s3_last_download]) : 'N/A'
 
-              puts format(
-                '%-45s %12s %15s  %-15s  %-12s  %-65s  %-18s  %-18s  %-30s  %-15s  %-15s',
-                data[:name],
-                format_size(data[:size]),
-                age_display,
-                data[:git_status],
-                data[:s3_sync],
-                shorten_path(data[:path]),
-                data[:heavy_files] || 'N/A',
-                data[:light_files] || 'N/A',
-                data[:ssd_backup] || 'N/A',
-                s3_upload,
-                s3_download
-              )
+              if s3
+                s3_upload = data[:s3_last_upload] ? format_age(data[:s3_last_upload]) : 'N/A'
+                s3_download = data[:s3_last_download] ? format_age(data[:s3_last_download]) : 'N/A'
+
+                puts format(
+                  '%-45s %12s %15s  %-15s  %-12s  %-65s  %-18s  %-18s  %-30s  %-15s  %-15s',
+                  data[:name],
+                  format_size(data[:size]),
+                  age_display,
+                  data[:git_status],
+                  data[:s3_sync],
+                  shorten_path(data[:path]),
+                  data[:heavy_files] || 'N/A',
+                  data[:light_files] || 'N/A',
+                  data[:ssd_backup] || 'N/A',
+                  s3_upload,
+                  s3_download
+                )
+              else
+                puts format(
+                  '%-45s %12s %15s  %-15s  %-65s  %-18s  %-18s  %-30s',
+                  data[:name],
+                  format_size(data[:size]),
+                  age_display,
+                  data[:git_status],
+                  shorten_path(data[:path]),
+                  data[:heavy_files] || 'N/A',
+                  data[:light_files] || 'N/A',
+                  data[:ssd_backup] || 'N/A'
+                )
+              end
             end
           else
             # Default view - use same format for header and data
             # rubocop:disable Style/RedundantFormat
-            puts format(
-              '%-45s %12s %15s  %-15s  %-12s',
-              'PROJECT',
-              'SIZE',
-              'AGE',
-              'GIT',
-              'S3'
-            )
+            if s3
+              puts format(
+                '%-45s %12s %15s  %-15s  %-12s',
+                'PROJECT',
+                'SIZE',
+                'AGE',
+                'GIT',
+                'S3'
+              )
+              puts '-' * 130
+            else
+              puts format(
+                '%-45s %12s %15s  %-15s',
+                'PROJECT',
+                'SIZE',
+                'AGE',
+                'GIT'
+              )
+              puts '-' * 117
+            end
             # rubocop:enable Style/RedundantFormat
-            puts '-' * 130
 
             project_data.each do |data|
               age_display = data[:stale] ? "#{data[:age]} ⚠️" : data[:age]
-              puts format(
-                '%-45s %12s %15s  %-15s  %-12s',
-                data[:name],
-                format_size(data[:size]),
-                age_display,
-                data[:git_status],
-                data[:s3_sync]
-              )
+
+              if s3
+                puts format(
+                  '%-45s %12s %15s  %-15s  %-12s',
+                  data[:name],
+                  format_size(data[:size]),
+                  age_display,
+                  data[:git_status],
+                  data[:s3_sync]
+                )
+              else
+                puts format(
+                  '%-45s %12s %15s  %-15s',
+                  data[:name],
+                  format_size(data[:size]),
+                  age_display,
+                  data[:git_status]
+                )
+              end
             end
           end
 
@@ -313,7 +414,7 @@ module Appydave
 
         # Collect brand data for display
         # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        def self.collect_brand_data(brand, detailed: false)
+        def self.collect_brand_data(brand, detailed: false, s3: false)
           Appydave::Tools::Configuration::Config.configure
           brand_info = Appydave::Tools::Configuration::Config.brands.get_brand(brand)
           brand_path = Config.brand_path(brand)
@@ -331,8 +432,8 @@ module Appydave
           # Get git status
           git_status = calculate_git_status(brand_path)
 
-          # Get S3 sync status (count of projects with s3-staging)
-          s3_sync_status = calculate_s3_sync_status(brand, projects)
+          # Get S3 sync status (count of projects with s3-staging) - only if requested
+          s3_sync_status = s3 ? calculate_s3_sync_status(brand, projects) : 'N/A'
 
           result = {
             shortcut: shortcut || key,
@@ -449,7 +550,7 @@ module Appydave
 
         # Collect project data for display
         # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/ParameterLists
-        def self.collect_project_data(brand_arg, brand_path, brand_info, project, is_git_repo, detailed: false)
+        def self.collect_project_data(brand_arg, brand_path, brand_info, project, is_git_repo, detailed: false, s3: false)
           project_path = Config.project_path(brand_arg, project)
           size = FileHelper.calculate_directory_size(project_path)
           modified = File.mtime(project_path)
@@ -461,8 +562,8 @@ module Appydave
                          'N/A'
                        end
 
-          # Calculate 3-state S3 sync status
-          s3_sync = calculate_project_s3_sync_status(brand_arg, brand_info, project)
+          # Calculate 3-state S3 sync status - only if requested (performance optimization)
+          s3_sync = s3 ? calculate_project_s3_sync_status(brand_arg, brand_info, project) : 'N/A'
 
           result = {
             name: project,
@@ -500,16 +601,23 @@ module Appydave
                          File.exist?(ssd_project_path) ? shorten_path(ssd_project_path) : nil
                        end
 
-            # S3 timestamps (last upload/download)
-            s3_timestamps = calculate_s3_timestamps(brand_arg, brand_info, project)
-
-            result.merge!(
-              heavy_files: "#{heavy_count} (#{format_size(heavy_size)})",
-              light_files: "#{light_count} (#{format_size(light_size)})",
-              ssd_backup: ssd_path,
-              s3_last_upload: s3_timestamps[:last_upload],
-              s3_last_download: s3_timestamps[:last_download]
-            )
+            # S3 timestamps (last upload/download) - only if requested (performance optimization)
+            if s3
+              s3_timestamps = calculate_s3_timestamps(brand_arg, brand_info, project)
+              result.merge!(
+                heavy_files: "#{heavy_count} (#{format_size(heavy_size)})",
+                light_files: "#{light_count} (#{format_size(light_size)})",
+                ssd_backup: ssd_path,
+                s3_last_upload: s3_timestamps[:last_upload],
+                s3_last_download: s3_timestamps[:last_download]
+              )
+            else
+              result.merge!(
+                heavy_files: "#{heavy_count} (#{format_size(heavy_size)})",
+                light_files: "#{light_count} (#{format_size(light_size)})",
+                ssd_backup: ssd_path
+              )
+            end
           end
 
           result

data/lib/appydave/tools/dam/s3_operations.rb

@@ -152,35 +152,37 @@ module Appydave
 
             s3_path = build_s3_key(relative_path)
 
-            # Check if file already exists with same MD5
-            local_md5 = file_md5(file)
-            s3_md5 = s3_file_md5(s3_path)
-
-            if local_md5 == s3_md5
-              puts "  ⏭️  Skipped: #{relative_path} (unchanged)"
-              skipped += 1
-            else
-              # Warn if we're about to overwrite an existing S3 file
-              if s3_md5 && s3_md5 != local_md5
-                puts "  ⚠️  Warning: #{relative_path} exists in S3 with different content"
-
-                # Try to get S3 timestamp for comparison
-                s3_file_info = get_s3_file_info(s3_path)
-                if s3_file_info && s3_file_info['LastModified']
-                  s3_time = s3_file_info['LastModified']
-                  local_time = File.mtime(file)
-                  puts "     S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
-
-                  puts '     ⚠️  S3 file is NEWER than local - you may be overwriting recent changes!' if s3_time > local_time
-                end
-                puts '     Uploading will overwrite S3 version...'
+            # Check if file already exists in S3 and compare
+            s3_info = get_s3_file_info(s3_path)
+
+            if s3_info
+              s3_etag = s3_info['ETag'].gsub('"', '')
+              s3_size = s3_info['Size']
+              match_status = compare_files(local_file: file, s3_etag: s3_etag, s3_size: s3_size)
+
+              if match_status == :synced
+                comparison_method = multipart_etag?(s3_etag) ? 'size match' : 'unchanged'
+                puts "  ⏭️  Skipped: #{relative_path} (#{comparison_method})"
+                skipped += 1
+                next
               end
 
-              if upload_file(file, s3_path, dry_run: dry_run)
-                uploaded += 1
-              else
-                failed += 1
-              end
+              # File exists but content differs - warn before overwriting
+              puts "  ⚠️  Warning: #{relative_path} exists in S3 with different content"
+              puts '     (multipart upload detected - comparing by size)' if multipart_etag?(s3_etag)
+
+              s3_time = s3_info['LastModified']
+              local_time = File.mtime(file)
+              puts "     S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
+
+              puts '     ⚠️  S3 file is NEWER than local - you may be overwriting recent changes!' if s3_time > local_time
+              puts '     Uploading will overwrite S3 version...'
+            end
+
+            if upload_file(file, s3_path, dry_run: dry_run)
+              uploaded += 1
+            else
+              failed += 1
             end
           end
           # rubocop:enable Metrics/BlockLength
@@ -215,41 +217,47 @@ module Appydave
           skipped = 0
           failed = 0
 
+          # rubocop:disable Metrics/BlockLength
           s3_files.each do |s3_file|
             key = s3_file['Key']
             relative_path = extract_relative_path(key)
             local_file = File.join(staging_dir, relative_path)
 
-            # Check if file already exists with same MD5
-            s3_md5 = s3_file['ETag'].gsub('"', '')
-            local_md5 = File.exist?(local_file) ? file_md5(local_file) : nil
+            # Check if file already exists and compare
+            s3_etag = s3_file['ETag'].gsub('"', '')
+            s3_size = s3_file['Size']
 
-            if local_md5 == s3_md5
-              puts "  ⏭️  Skipped: #{relative_path} (unchanged)"
-              skipped += 1
-            else
-              # Warn if we're about to overwrite an existing local file
-              if local_md5 && local_md5 != s3_md5
-                puts "  ⚠️  Warning: #{relative_path} exists locally with different content"
-
-                # Compare timestamps
-                if s3_file['LastModified'] && File.exist?(local_file)
-                  s3_time = s3_file['LastModified']
-                  local_time = File.mtime(local_file)
-                  puts "     S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
-
-                  puts '     ⚠️  Local file is NEWER than S3 - you may be overwriting recent changes!' if local_time > s3_time
-                end
-                puts '     Downloading will overwrite local version...'
+            if File.exist?(local_file)
+              match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
+
+              if match_status == :synced
+                comparison_method = multipart_etag?(s3_etag) ? 'size match' : 'unchanged'
+                puts "  ⏭️  Skipped: #{relative_path} (#{comparison_method})"
+                skipped += 1
+                next
               end
 
-              if download_file(key, local_file, dry_run: dry_run)
-                downloaded += 1
-              else
-                failed += 1
+              # File exists but content differs - warn before overwriting
+              puts "  ⚠️  Warning: #{relative_path} exists locally with different content"
+              puts '     (multipart upload detected - comparing by size)' if multipart_etag?(s3_etag)
+
+              if s3_file['LastModified']
+                s3_time = s3_file['LastModified']
+                local_time = File.mtime(local_file)
+                puts "     S3: #{s3_time.strftime('%Y-%m-%d %H:%M')} | Local: #{local_time.strftime('%Y-%m-%d %H:%M')}"
+
+                puts '     ⚠️  Local file is NEWER than S3 - you may be overwriting recent changes!' if local_time > s3_time
               end
+              puts '     Downloading will overwrite local version...'
+            end
+
+            if download_file(key, local_file, dry_run: dry_run)
+              downloaded += 1
+            else
+              failed += 1
             end
           end
+          # rubocop:enable Metrics/BlockLength
           puts ''
           puts '✅ Download complete!'
           puts "   Downloaded: #{downloaded}, Skipped: #{skipped}, Failed: #{failed}"
@@ -322,11 +330,12 @@ module Appydave
               total_s3_size += s3_size
               total_local_size += local_size
 
-              local_md5 = file_md5(local_file)
-              s3_md5 = s3_file['ETag'].gsub('"', '')
+              s3_etag = s3_file['ETag'].gsub('"', '')
+              match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
 
-              if local_md5 == s3_md5
-                puts "  ✓ #{relative_path} (#{file_size_human(s3_size)}) [synced]"
+              if match_status == :synced
+                status_label = multipart_etag?(s3_etag) ? 'synced*' : 'synced'
+                puts "  ✓ #{relative_path} (#{file_size_human(s3_size)}) [#{status_label}]"
               else
                 puts "  ⚠️  #{relative_path} (#{file_size_human(s3_size)}) [modified]"
               end
@@ -526,10 +535,11 @@ module Appydave
             s3_file = s3_files_map[relative_path]
 
             if s3_file
-              # Compare MD5
-              local_md5 = file_md5(local_file)
-              s3_md5 = s3_file['ETag'].gsub('"', '')
-              needs_upload = true if local_md5 != s3_md5
+              # Compare using multipart-aware comparison
+              s3_etag = s3_file['ETag'].gsub('"', '')
+              s3_size = s3_file['Size']
+              match_status = compare_files(local_file: local_file, s3_etag: s3_etag, s3_size: s3_size)
+              needs_upload = true if match_status != :synced
             else
               # Local file not in S3
               needs_upload = true
@@ -623,6 +633,47 @@ module Appydave
           nil
         end
 
+        # Check if an S3 ETag is from a multipart upload
+        # Multipart ETags have format: "hash-partcount" (e.g., "d41d8cd98f00b204e9800998ecf8427e-5")
+        def multipart_etag?(etag)
+          return false if etag.nil?
+
+          etag.include?('-')
+        end
+
+        # Compare local file with S3 file, handling multipart ETags
+        # Returns: :synced, :modified, or :unknown
+        # For multipart uploads, falls back to size comparison since MD5 won't match
+        def compare_files(local_file:, s3_etag:, s3_size:)
+          return :unknown unless File.exist?(local_file)
+          return :unknown if s3_etag.nil?
+
+          local_size = File.size(local_file)
+
+          if multipart_etag?(s3_etag)
+            # Multipart upload - MD5 comparison won't work, use size
+            # Size match is a reasonable proxy for "unchanged" in this context
+            local_size == s3_size ? :synced : :modified
+          else
+            # Standard upload - use MD5 comparison
+            local_md5 = file_md5(local_file)
+            return :unknown if local_md5.nil?
+
+            local_md5 == s3_etag ? :synced : :modified
+          end
+        end
+
+        # Get S3 file size from path (for upload comparison)
+        def s3_file_size(s3_path)
+          response = s3_client.head_object(
+            bucket: brand_info.aws.s3_bucket,
+            key: s3_path
+          )
+          response.content_length
+        rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::ServiceError
+          nil
+        end
+
         # Upload file to S3
         def upload_file(local_file, s3_path, dry_run: false)
           if dry_run

data/lib/appydave/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Appydave
   module Tools
-    VERSION = '0.69.0'
+    VERSION = '0.70.0'
   end
 end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appydave-tools",
-  "version": "0.69.0",
+  "version": "0.70.0",
   "description": "AppyDave YouTube Automation Tools",
   "scripts": {
     "release": "semantic-release"