appydave-tools 0.16.0 → 0.17.0

data/lib/appydave/tools/vat/s3_operations.rb

@@ -0,0 +1,560 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'digest'
+require 'aws-sdk-s3'
+
+module Appydave
+  module Tools
+    module Vat
+      # S3 operations for VAT (upload, download, status, cleanup)
+      class S3Operations
+        attr_reader :brand_info, :brand, :project_id, :brand_path, :s3_client
+
+        def initialize(brand, project_id, brand_info: nil, brand_path: nil, s3_client: nil)
+          @brand = brand
+          @project_id = project_id
+
+          # Use injected dependencies or load from configuration
+          @brand_path = brand_path || Config.brand_path(brand)
+          @brand_info = brand_info || load_brand_info(brand)
+          @s3_client = s3_client || create_s3_client(@brand_info)
+        end
+
+        private
+
+        def load_brand_info(brand)
+          Appydave::Tools::Configuration::Config.configure
+          Appydave::Tools::Configuration::Config.brands.get_brand(brand)
+        end
+
+        def create_s3_client(brand_info)
+          profile_name = brand_info.aws.profile
+          raise "AWS profile not configured for brand '#{brand}'" if profile_name.nil? || profile_name.empty?
+
+          credentials = Aws::SharedCredentials.new(profile_name: profile_name)
+          Aws::S3::Client.new(
+            credentials: credentials,
+            region: brand_info.aws.region,
+            http_wire_trace: false,
+            ssl_verify_peer: true,
+            ssl_ca_bundle: '/etc/ssl/cert.pem' # macOS system certificates
+          )
+        end
+
+        public
+
+        # Upload files from s3-staging/ to S3
+        def upload(dry_run: false)
+          project_dir = File.join(brand_path, project_id)
+          staging_dir = File.join(project_dir, 's3-staging')
+
+          unless Dir.exist?(staging_dir)
+            puts "❌ No s3-staging directory found: #{staging_dir}"
+            puts 'Nothing to upload.'
+            return
+          end
+
+          files = Dir.glob("#{staging_dir}/**/*").select { |f| File.file?(f) }
+
+          if files.empty?
+            puts '❌ No files found in s3-staging/'
+            return
+          end
+
+          puts "📦 Uploading #{files.size} file(s) from #{project_id}/s3-staging/ to S3..."
+          puts ''
+
+          uploaded = 0
+          skipped = 0
+          failed = 0
+
+          files.each do |file|
+            relative_path = file.sub("#{staging_dir}/", '')
+            s3_path = build_s3_key(relative_path)
+
+            # Check if file already exists with same MD5
+            local_md5 = file_md5(file)
+            s3_md5 = s3_file_md5(s3_path)
+
+            if local_md5 == s3_md5
+              puts "  ⏭️  Skipped: #{relative_path} (unchanged)"
+              skipped += 1
+            elsif upload_file(file, s3_path, dry_run: dry_run)
+              uploaded += 1
+            else
+              failed += 1
+            end
+          end
+
+          puts ''
+          puts '✅ Upload complete!'
+          puts "   Uploaded: #{uploaded}, Skipped: #{skipped}, Failed: #{failed}"
+        end
+
+        # Download files from S3 to s3-staging/
+        def download(dry_run: false)
+          project_dir = File.join(brand_path, project_id)
+          staging_dir = File.join(project_dir, 's3-staging')
+
+          s3_files = list_s3_files
+
+          if s3_files.empty?
+            puts "❌ No files found in S3 for #{brand}/#{project_id}"
+            return
+          end
+
+          puts "📦 Downloading #{s3_files.size} file(s) from S3 to #{project_id}/s3-staging/..."
+          puts ''
+
+          downloaded = 0
+          skipped = 0
+          failed = 0
+
+          s3_files.each do |s3_file|
+            key = s3_file['Key']
+            relative_path = extract_relative_path(key)
+            local_file = File.join(staging_dir, relative_path)
+
+            # Check if file already exists with same MD5
+            s3_md5 = s3_file['ETag'].gsub('"', '')
+            local_md5 = File.exist?(local_file) ? file_md5(local_file) : nil
+
+            if local_md5 == s3_md5
+              puts "  ⏭️  Skipped: #{relative_path} (unchanged)"
+              skipped += 1
+            elsif download_file(key, local_file, dry_run: dry_run)
+              downloaded += 1
+            else
+              failed += 1
+            end
+          end
+
+          puts ''
+          puts '✅ Download complete!'
+          puts "   Downloaded: #{downloaded}, Skipped: #{skipped}, Failed: #{failed}"
+        end
+
+        # Show sync status
+        def status
+          project_dir = File.join(brand_path, project_id)
+          staging_dir = File.join(project_dir, 's3-staging')
+
+          s3_files = list_s3_files
+          local_files = list_local_files(staging_dir)
+
+          # Build a map of S3 files for quick lookup
+          s3_files_map = s3_files.each_with_object({}) do |file, hash|
+            relative_path = extract_relative_path(file['Key'])
+            hash[relative_path] = file
+          end
+
+          if s3_files.empty? && local_files.empty?
+            puts "❌ No files found in S3 or locally for #{brand}/#{project_id}"
+            return
+          end
+
+          puts "📊 S3 Sync Status for #{brand}/#{project_id}"
+          puts ''
+
+          # Combine all file paths (S3 + local)
+          all_paths = (s3_files_map.keys + local_files.keys).uniq.sort
+
+          total_s3_size = 0
+          total_local_size = 0
+
+          all_paths.each do |relative_path|
+            s3_file = s3_files_map[relative_path]
+            local_file = File.join(staging_dir, relative_path)
+
+            if s3_file && File.exist?(local_file)
+              # File exists in both S3 and local
+              s3_size = s3_file['Size']
+              local_size = File.size(local_file)
+              total_s3_size += s3_size
+              total_local_size += local_size
+
+              local_md5 = file_md5(local_file)
+              s3_md5 = s3_file['ETag'].gsub('"', '')
+
+              if local_md5 == s3_md5
+                puts "  ✓ #{relative_path} (#{file_size_human(s3_size)}) [synced]"
+              else
+                puts "  ⚠️  #{relative_path} (#{file_size_human(s3_size)}) [modified]"
+              end
+            elsif s3_file
+              # File only in S3
+              s3_size = s3_file['Size']
+              total_s3_size += s3_size
+              puts "  ☁️  #{relative_path} (#{file_size_human(s3_size)}) [S3 only]"
+            else
+              # File only local
+              local_size = File.size(local_file)
+              total_local_size += local_size
+              puts "  📁 #{relative_path} (#{file_size_human(local_size)}) [local only]"
+            end
+          end
+
+          puts ''
+          puts "S3 files: #{s3_files.size}, Local files: #{local_files.size}"
+          puts "S3 size: #{file_size_human(total_s3_size)}, Local size: #{file_size_human(total_local_size)}"
+        end
+
+        # Cleanup S3 files
+        def cleanup(force: false, dry_run: false)
+          s3_files = list_s3_files
+
+          if s3_files.empty?
+            puts "❌ No files found in S3 for #{brand}/#{project_id}"
+            return
+          end
+
+          puts "🗑️  Found #{s3_files.size} file(s) in S3 for #{brand}/#{project_id}"
+          puts ''
+
+          unless force
+            puts '⚠️  This will DELETE all files from S3 for this project.'
+            puts 'Use --force to confirm deletion.'
+            return
+          end
+
+          deleted = 0
+          failed = 0
+
+          s3_files.each do |s3_file|
+            key = s3_file['Key']
+            relative_path = extract_relative_path(key)
+
+            if delete_s3_file(key, dry_run: dry_run)
+              puts "  ✓ Deleted: #{relative_path}"
+              deleted += 1
+            else
+              puts "  ✗ Failed: #{relative_path}"
+              failed += 1
+            end
+          end
+
+          puts ''
+          puts '✅ Cleanup complete!'
+          puts "   Deleted: #{deleted}, Failed: #{failed}"
+        end
+
+        # Cleanup local s3-staging files
+        def cleanup_local(force: false, dry_run: false)
+          project_dir = File.join(brand_path, project_id)
+          staging_dir = File.join(project_dir, 's3-staging')
+
+          unless Dir.exist?(staging_dir)
+            puts "❌ No s3-staging directory found: #{staging_dir}"
+            return
+          end
+
+          files = Dir.glob("#{staging_dir}/**/*").select { |f| File.file?(f) }
+
+          if files.empty?
+            puts '❌ No files found in s3-staging/'
+            return
+          end
+
+          puts "🗑️  Found #{files.size} file(s) in #{project_id}/s3-staging/"
+          puts ''
+
+          unless force
+            puts '⚠️  This will DELETE all local files in s3-staging/ for this project.'
+            puts 'Use --force to confirm deletion.'
+            return
+          end
+
+          deleted = 0
+          failed = 0
+
+          files.each do |file|
+            relative_path = file.sub("#{staging_dir}/", '')
+
+            if delete_local_file(file, dry_run: dry_run)
+              puts "  ✓ Deleted: #{relative_path}"
+              deleted += 1
+            else
+              puts "  ✗ Failed: #{relative_path}"
+              failed += 1
+            end
+          end
+
+          # Clean up empty directories
+          unless dry_run
+            Dir.glob("#{staging_dir}/**/").reverse_each do |dir|
+              Dir.rmdir(dir) if Dir.empty?(dir)
+            rescue SystemCallError
+              # Directory not empty, skip
+            end
+          end
+
+          puts ''
+          puts '✅ Local cleanup complete!'
+          puts "   Deleted: #{deleted}, Failed: #{failed}"
+        end
+
+        # Archive project to SSD
+        def archive(force: false, dry_run: false)
+          ssd_backup = brand_info.locations.ssd_backup
+
+          unless ssd_backup && !ssd_backup.empty?
+            puts "❌ SSD backup location not configured for brand '#{brand}'"
+            return
+          end
+
+          unless Dir.exist?(ssd_backup)
+            puts "❌ SSD not mounted at #{ssd_backup}"
+            puts '   Please connect the SSD before archiving.'
+            return
+          end
+
+          project_dir = File.join(brand_path, project_id)
+
+          unless Dir.exist?(project_dir)
+            puts "❌ Project not found: #{project_dir}"
+            return
+          end
+
+          # Determine SSD destination path
+          ssd_project_dir = File.join(ssd_backup, project_id)
+
+          puts "📦 Archive: #{brand}/#{project_id}"
+          puts ''
+
+          # Step 1: Copy to SSD
+          if copy_to_ssd(project_dir, ssd_project_dir, dry_run: dry_run)
+            # Step 2: Delete local project (if force is true)
+            if force
+              delete_local_project(project_dir, dry_run: dry_run)
+            else
+              puts ''
+              puts '⚠️  Project copied to SSD but NOT deleted locally.'
+              puts '   Use --force to delete local copy after archiving.'
+            end
+          end
+
+          puts ''
+          puts dry_run ? '✅ Archive dry-run complete!' : '✅ Archive complete!'
+        end
+
+        # Build S3 key for a file
+        def build_s3_key(relative_path)
+          "#{brand_info.aws.s3_prefix}#{project_id}/#{relative_path}"
+        end
+
+        # Extract relative path from S3 key
+        def extract_relative_path(s3_key)
+          s3_key.sub("#{brand_info.aws.s3_prefix}#{project_id}/", '')
+        end
+
+        # Calculate MD5 hash of a file
+        def file_md5(file_path)
+          Digest::MD5.file(file_path).hexdigest
+        end
+
+        # Get MD5 of file in S3 (from ETag)
+        def s3_file_md5(s3_path)
+          response = s3_client.head_object(
+            bucket: brand_info.aws.s3_bucket,
+            key: s3_path
+          )
+          response.etag.gsub('"', '')
+        rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::ServiceError
+          nil
+        end
+
+        # Upload file to S3
+        def upload_file(local_file, s3_path, dry_run: false)
+          if dry_run
+            puts "  [DRY-RUN] Would upload: #{local_file} → s3://#{brand_info.aws.s3_bucket}/#{s3_path}"
+            return true
+          end
+
+          File.open(local_file, 'rb') do |file|
+            s3_client.put_object(
+              bucket: brand_info.aws.s3_bucket,
+              key: s3_path,
+              body: file
+            )
+          end
+
+          puts "  ✓ Uploaded: #{File.basename(local_file)} (#{file_size_human(File.size(local_file))})"
+          true
+        rescue Aws::S3::Errors::ServiceError => e
+          puts "  ✗ Failed: #{File.basename(local_file)}"
+          puts "    Error: #{e.message}"
+          false
+        end
+
+        # Download file from S3
+        def download_file(s3_key, local_file, dry_run: false)
+          if dry_run
+            puts "  [DRY-RUN] Would download: s3://#{brand_info.aws.s3_bucket}/#{s3_key} → #{local_file}"
+            return true
+          end
+
+          FileUtils.mkdir_p(File.dirname(local_file))
+
+          s3_client.get_object(
+            bucket: brand_info.aws.s3_bucket,
+            key: s3_key,
+            response_target: local_file
+          )
+
+          puts "  ✓ Downloaded: #{File.basename(local_file)} (#{file_size_human(File.size(local_file))})"
+          true
+        rescue Aws::S3::Errors::ServiceError => e
+          puts "  ✗ Failed: #{File.basename(local_file)}"
+          puts "    Error: #{e.message}"
+          false
+        end
+
+        # Delete file from S3
+        def delete_s3_file(s3_key, dry_run: false)
+          if dry_run
+            puts "  [DRY-RUN] Would delete: s3://#{brand_info.aws.s3_bucket}/#{s3_key}"
+            return true
+          end
+
+          s3_client.delete_object(
+            bucket: brand_info.aws.s3_bucket,
+            key: s3_key
+          )
+
+          true
+        rescue Aws::S3::Errors::ServiceError => e
+          puts "    Error: #{e.message}"
+          false
+        end
+
+        # Delete local file
+        def delete_local_file(file_path, dry_run: false)
+          if dry_run
+            puts "  [DRY-RUN] Would delete: #{file_path}"
+            return true
+          end
+
+          File.delete(file_path)
+          true
+        rescue StandardError => e
+          puts "    Error: #{e.message}"
+          false
+        end
+
+        # List files in S3 for a project
+        def list_s3_files
+          prefix = build_s3_key('')
+
+          response = s3_client.list_objects_v2(
+            bucket: brand_info.aws.s3_bucket,
+            prefix: prefix
+          )
+
+          return [] unless response.contents
+
+          response.contents.map do |obj|
+            {
+              'Key' => obj.key,
+              'Size' => obj.size,
+              'ETag' => obj.etag
+            }
+          end
+        rescue Aws::S3::Errors::ServiceError
+          []
+        end
+
+        # List local files in staging directory
+        def list_local_files(staging_dir)
+          return {} unless Dir.exist?(staging_dir)
+
+          files = Dir.glob("#{staging_dir}/**/*").select { |f| File.file?(f) }
+
+          files.each_with_object({}) do |file, hash|
+            relative_path = file.sub("#{staging_dir}/", '')
+            hash[relative_path] = file
+          end
+        end
+
+        # Human-readable file size
+        def file_size_human(bytes)
+          if bytes < 1024
+            "#{bytes} B"
+          elsif bytes < 1024 * 1024
+            "#{(bytes / 1024.0).round(1)} KB"
+          elsif bytes < 1024 * 1024 * 1024
+            "#{(bytes / (1024.0 * 1024)).round(1)} MB"
+          else
+            "#{(bytes / (1024.0 * 1024 * 1024)).round(2)} GB"
+          end
+        end
+
+        # Copy project to SSD
+        def copy_to_ssd(source_dir, dest_dir, dry_run: false)
+          if Dir.exist?(dest_dir)
+            puts '⚠️  Already exists on SSD'
+            puts "   Path: #{dest_dir}"
+            puts '   Skipping copy step'
+            return true
+          end
+
+          size = calculate_directory_size(source_dir)
+          puts '📋 Copy to SSD:'
+          puts "   Source: #{source_dir}"
+          puts "   Dest:   #{dest_dir}"
+          puts "   Size:   #{file_size_human(size)}"
+          puts ''
+
+          if dry_run
+            puts '   [DRY-RUN] Would copy entire project to SSD'
+            return true
+          end
+
+          FileUtils.mkdir_p(File.dirname(dest_dir))
+          FileUtils.cp_r(source_dir, dest_dir, preserve: true)
+          puts '   ✅ Copied to SSD'
+
+          true
+        rescue StandardError => e
+          puts "   ✗ Failed to copy: #{e.message}"
+          false
+        end
+
+        # Delete local project directory
+        def delete_local_project(project_dir, dry_run: false)
+          size = calculate_directory_size(project_dir)
+
+          puts ''
+          puts '🗑️  Delete local project:'
+          puts "   Path: #{project_dir}"
+          puts "   Size: #{file_size_human(size)}"
+          puts ''
+
+          if dry_run
+            puts '   [DRY-RUN] Would delete entire local folder'
+            return true
+          end
+
+          FileUtils.rm_rf(project_dir)
+          puts '   ✅ Deleted local folder'
+          puts "   💾 Freed: #{file_size_human(size)}"
+
+          true
+        rescue StandardError => e
+          puts "   ✗ Failed to delete: #{e.message}"
+          false
+        end
+
+        # Calculate total size of a directory
+        def calculate_directory_size(dir_path)
+          total = 0
+          Dir.glob(File.join(dir_path, '**', '*'), File::FNM_DOTMATCH).each do |file|
+            total += File.size(file) if File.file?(file)
+          end
+          total
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Appydave
   module Tools
-    VERSION = '0.16.0'
+    VERSION = '0.17.0'
   end
 end

data/lib/appydave/tools.rb

@@ -42,6 +42,7 @@ require 'appydave/tools/configuration/configurable'
 require 'appydave/tools/configuration/config'
 require 'appydave/tools/configuration/models/config_base'
 require 'appydave/tools/configuration/models/settings_config'
+require 'appydave/tools/configuration/models/brands_config'
 require 'appydave/tools/configuration/models/channels_config'
 require 'appydave/tools/configuration/models/youtube_automation_config'
 require 'appydave/tools/name_manager/project_name'
@@ -51,6 +52,13 @@ require 'appydave/tools/prompt_tools/prompt_completion'
 require 'appydave/tools/subtitle_processor/clean'
 require 'appydave/tools/subtitle_processor/join'
 
+require 'appydave/tools/vat/config'
+require 'appydave/tools/vat/project_resolver'
+require 'appydave/tools/vat/config_loader'
+require 'appydave/tools/vat/s3_operations'
+require 'appydave/tools/vat/project_listing'
+require 'appydave/tools/vat/manifest_generator'
+
 require 'appydave/tools/youtube_automation/gpt_agent'
 
 require 'appydave/tools/youtube_manager/models/youtube_details'
@@ -65,7 +73,7 @@ require 'appydave/tools/youtube_manager/reports/video_content_report'
 Appydave::Tools::Configuration::Config.set_default do |config|
   config.config_path = File.expand_path('~/.config/appydave')
   config.register(:settings, Appydave::Tools::Configuration::Models::SettingsConfig)
-  # config.register(:bank_reconciliation, Appydave::Tools::Configuration::Models::BankReconciliationConfig)
+  config.register(:brands, Appydave::Tools::Configuration::Models::BrandsConfig)
   config.register(:channels, Appydave::Tools::Configuration::Models::ChannelsConfig)
   config.register(:youtube_automation, Appydave::Tools::Configuration::Models::YoutubeAutomationConfig)
 end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appydave-tools",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "AppyDave YouTube Automation Tools",
   "scripts": {
     "release": "semantic-release"