appydave-tools 0.17.1 → 0.18.0

data/lib/appydave/tools/{vat → dam}/s3_operations.rb

@@ -7,18 +7,18 @@ require 'aws-sdk-s3'
 
 module Appydave
   module Tools
-    module Vat
+    module Dam
       # S3 operations for VAT (upload, download, status, cleanup)
       class S3Operations
         attr_reader :brand_info, :brand, :project_id, :brand_path, :s3_client
 
         def initialize(brand, project_id, brand_info: nil, brand_path: nil, s3_client: nil)
-          @brand = brand
           @project_id = project_id
 
           # Use injected dependencies or load from configuration
-          @brand_path = brand_path || Config.brand_path(brand)
           @brand_info = brand_info || load_brand_info(brand)
+          @brand = @brand_info.key # Use resolved brand key, not original input
+          @brand_path = brand_path || Config.brand_path(@brand)
           @s3_client = s3_client || create_s3_client(@brand_info)
         end
 

data/lib/appydave/tools/dam/sync_from_ssd.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module Appydave
+  module Tools
+    module Dam
+      # Sync light files from SSD to local for a brand
+      # Only copies non-video files (subtitles, images, docs)
+      class SyncFromSsd
+        attr_reader :brand, :brand_info, :brand_path
+
+        # Light file patterns to include (everything except heavy video files)
+        LIGHT_FILE_PATTERNS = %w[
+          **/*.srt
+          **/*.vtt
+          **/*.txt
+          **/*.md
+          **/*.jpg
+          **/*.jpeg
+          **/*.png
+          **/*.webp
+          **/*.json
+          **/*.yml
+          **/*.yaml
+        ].freeze
+
+        # Heavy file patterns to exclude (video files)
+        HEAVY_FILE_PATTERNS = %w[
+          *.mp4
+          *.mov
+          *.avi
+          *.mkv
+          *.webm
+        ].freeze
+
+        def initialize(brand, brand_info: nil, brand_path: nil)
+          @brand_info = brand_info || load_brand_info(brand)
+          @brand = @brand_info.key # Use resolved brand key, not original input
+          @brand_path = brand_path || Config.brand_path(@brand)
+        end
+
+        # Sync light files from SSD for all projects in manifest
+        def sync(dry_run: false)
+          puts dry_run ? '🔍 DRY-RUN MODE - No files will be copied' : '📦 Syncing from SSD...'
+          puts ''
+
+          # Validate SSD is mounted
+          ssd_backup = brand_info.locations.ssd_backup
+          unless ssd_backup && !ssd_backup.empty?
+            puts "❌ SSD backup location not configured for brand '#{brand}'"
+            return
+          end
+
+          unless Dir.exist?(ssd_backup)
+            puts "❌ SSD not mounted at #{ssd_backup}"
+            return
+          end
+
+          # Load manifest
+          manifest_file = File.join(brand_path, 'projects.json')
+          unless File.exist?(manifest_file)
+            puts '❌ projects.json not found!'
+            puts "   Run: vat manifest #{brand}"
+            return
+          end
+
+          manifest = load_manifest(manifest_file)
+          puts "📋 Loaded manifest: #{manifest[:projects].size} projects"
+          puts "   Last updated: #{manifest[:config][:last_updated]}"
+          puts ''
+
+          # Filter projects to sync
+          projects_to_sync = manifest[:projects].select { |p| should_sync_project?(p) }
+
+          puts '🔍 Analysis:'
+          puts "   Total projects in manifest: #{manifest[:projects].size}"
+          puts "   Projects to sync: #{projects_to_sync.size}"
+          puts "   Skipped (already local): #{manifest[:projects].size - projects_to_sync.size}"
+          puts ''
+
+          if projects_to_sync.empty?
+            puts '✅ Nothing to sync - all projects either already local or not on SSD'
+            return
+          end
+
+          # Sync each project
+          total_stats = { files: 0, bytes: 0, skipped: 0 }
+
+          projects_to_sync.each do |project|
+            stats = sync_project(project, ssd_backup, dry_run: dry_run)
+
+            # Only show project if there are files to sync or a warning
+            if stats[:reason] || stats[:files]&.positive?
+              puts "📁 #{project[:id]}"
+
+              puts "  ⚠️  Skipped: #{stats[:reason]}" if stats[:reason]
+
+              puts "  #{stats[:files]} file(s), #{format_bytes(stats[:bytes])}" if stats[:files]&.positive?
+              puts ''
+            end
+
+            total_stats[:files] += stats[:files] || 0
+            total_stats[:bytes] += stats[:bytes] || 0
+            total_stats[:skipped] += stats[:skipped] || 0
+          end
+
+          puts ''
+          puts '=' * 60
+          puts 'Summary:'
+          puts "  Projects scanned: #{projects_to_sync.size}"
+          puts "  Files #{dry_run ? 'to copy' : 'copied'}: #{total_stats[:files]}"
+          puts "  Total size: #{format_bytes(total_stats[:bytes])}"
+          puts ''
+          puts '✅ Sync complete!'
+          puts '   Run without --dry-run to perform the sync' if dry_run
+        end
+
+        private
+
+        def load_brand_info(brand)
+          Appydave::Tools::Configuration::Config.configure
+          Appydave::Tools::Configuration::Config.brands.get_brand(brand)
+        end
+
+        def load_manifest(manifest_file)
+          JSON.parse(File.read(manifest_file), symbolize_names: true)
+        rescue JSON::ParserError => e
+          puts "❌ Error parsing projects.json: #{e.message}"
+          exit 1
+        end
+
+        # Determine if project should be synced
+        def should_sync_project?(project)
+          # Only sync if project exists on SSD but NOT in local flat structure
+          return false unless project[:storage][:ssd][:exists]
+
+          # Skip if exists locally in flat structure
+          return false if project[:storage][:local][:exists] && project[:storage][:local][:structure] == 'flat'
+
+          true
+        end
+
+        # Sync a single project from SSD to local
+        def sync_project(project, ssd_backup, dry_run: false)
+          project_id = project[:id]
+          ssd_path = File.join(ssd_backup, project_id)
+
+          return { skipped: 1, files: 0, bytes: 0, reason: 'SSD path not found' } unless Dir.exist?(ssd_path)
+
+          # Check for flat folder conflict (stale manifest)
+          flat_path = File.join(brand_path, project_id)
+          return { skipped: 1, files: 0, bytes: 0, reason: 'Flat folder exists (stale manifest?)' } if Dir.exist?(flat_path)
+
+          # Determine local destination path (archived structure)
+          # Extract range from project ID (e.g., b65 → 60-69 range)
+          range = determine_range(project_id)
+          local_dir = File.join(brand_path, 'archived', range, project_id)
+
+          # Create local directory
+          FileUtils.mkdir_p(local_dir) if !dry_run && !Dir.exist?(local_dir)
+
+          # Sync light files
+          sync_light_files(ssd_path, local_dir, dry_run: dry_run)
+        end
+
+        # Determine range folder for project (e.g., b65 → 60-69)
+        def determine_range(project_id)
+          # FliVideo pattern: b40, b41, ... b99
+          if project_id =~ /^b(\d+)/
+            tens = (Regexp.last_match(1).to_i / 10) * 10
+            "#{tens}-#{tens + 9}"
+          else
+            # Legacy pattern or unknown: use first 3 chars
+            '000-099'
+          end
+        end
+
+        # Sync light files from SSD to local
+        def sync_light_files(ssd_path, local_dir, dry_run: false)
+          stats = { files: 0, bytes: 0 }
+
+          LIGHT_FILE_PATTERNS.each do |pattern|
+            Dir.glob(File.join(ssd_path, pattern)).each do |source_file|
+              next if heavy_file?(source_file)
+
+              copy_stats = copy_light_file(source_file, ssd_path, local_dir, dry_run: dry_run)
+              stats[:files] += copy_stats[:files]
+              stats[:bytes] += copy_stats[:bytes]
+            end
+          end
+
+          stats
+        end
+
+        # Check if file is a heavy video file
+        def heavy_file?(source_file)
+          HEAVY_FILE_PATTERNS.any? { |pattern| File.fnmatch(pattern, File.basename(source_file)) }
+        end
+
+        # Copy a single light file
+        def copy_light_file(source_file, ssd_path, local_dir, dry_run: false)
+          relative_path = source_file.sub("#{ssd_path}/", '')
+          dest_file = File.join(local_dir, relative_path)
+
+          # Skip if already synced (same size)
+          return { files: 0, bytes: 0 } if file_already_synced?(source_file, dest_file)
+
+          file_size = File.size(source_file)
+
+          if dry_run
+            puts "  [DRY-RUN] Would copy: #{relative_path} (#{format_bytes(file_size)})"
+          else
+            FileUtils.mkdir_p(File.dirname(dest_file))
+            FileUtils.cp(source_file, dest_file, preserve: true)
+            puts "  ✓ Copied: #{relative_path} (#{format_bytes(file_size)})"
+          end
+
+          { files: 1, bytes: file_size }
+        end
+
+        # Check if file is already synced (by size comparison)
+        def file_already_synced?(source_file, dest_file)
+          File.exist?(dest_file) && File.size(dest_file) == File.size(source_file)
+        end
+
+        # Format bytes into human-readable format
+        def format_bytes(bytes)
+          if bytes < 1024
+            "#{bytes}B"
+          elsif bytes < 1024 * 1024
+            "#{(bytes / 1024.0).round(1)}KB"
+          else
+            "#{(bytes / 1024.0 / 1024.0).round(1)}MB"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Appydave
   module Tools
-    VERSION = '0.17.1'
+    VERSION = '0.18.0'
   end
 end

data/lib/appydave/tools.rb

@@ -52,12 +52,13 @@ require 'appydave/tools/prompt_tools/prompt_completion'
 require 'appydave/tools/subtitle_processor/clean'
 require 'appydave/tools/subtitle_processor/join'
 
-require 'appydave/tools/vat/config'
-require 'appydave/tools/vat/project_resolver'
-require 'appydave/tools/vat/config_loader'
-require 'appydave/tools/vat/s3_operations'
-require 'appydave/tools/vat/project_listing'
-require 'appydave/tools/vat/manifest_generator'
+require 'appydave/tools/dam/config'
+require 'appydave/tools/dam/project_resolver'
+require 'appydave/tools/dam/config_loader'
+require 'appydave/tools/dam/s3_operations'
+require 'appydave/tools/dam/project_listing'
+require 'appydave/tools/dam/manifest_generator'
+require 'appydave/tools/dam/sync_from_ssd'
 
 require 'appydave/tools/youtube_automation/gpt_agent'
 

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appydave-tools",
-  "version": "0.17.1",
+  "version": "0.18.0",
   "description": "AppyDave YouTube Automation Tools",
   "scripts": {
     "release": "semantic-release"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: appydave-tools
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.18.0
 platform: ruby
 authors:
 - David Cruwys
@@ -214,6 +214,7 @@ files:
 - bin/bank_reconciliation.rb
 - bin/configuration.rb
 - bin/console
+- bin/dam
 - bin/generate_manifest.rb
 - bin/gpt_context.rb
 - bin/move_images.rb
@@ -223,7 +224,6 @@ files:
 - bin/subtitle_manager.rb
 - bin/subtitle_processor.rb
 - bin/sync_from_ssd.rb
-- bin/vat
 - bin/youtube_automation.rb
 - bin/youtube_manager.rb
 - docs/README.md
@@ -237,6 +237,10 @@ files:
 - docs/configuration/README.md
 - docs/configuration/channels.example.json
 - docs/configuration/settings.example.json
+- docs/dam/dam-vision.md
+- docs/dam/session-summary-2025-11-09.md
+- docs/dam/usage.md
+- docs/dam/vat-testing-plan.md
 - docs/development/CODEX-recommendations.md
 - docs/development/README.md
 - docs/development/cli-architecture-patterns.md
@@ -254,10 +258,6 @@ files:
 - docs/tools/subtitle-processor.md
 - docs/tools/youtube-automation.md
 - docs/tools/youtube-manager.md
-- docs/vat/dam-vision.md
-- docs/vat/session-summary-2025-11-09.md
-- docs/vat/usage.md
-- docs/vat/vat-testing-plan.md
 - exe/ad_config
 - exe/gpt_context
 - exe/prompt_tools
@@ -281,6 +281,13 @@ files:
 - lib/appydave/tools/configuration/models/settings_config.rb
 - lib/appydave/tools/configuration/models/youtube_automation_config.rb
 - lib/appydave/tools/configuration/openai.rb
+- lib/appydave/tools/dam/config.rb
+- lib/appydave/tools/dam/config_loader.rb
+- lib/appydave/tools/dam/manifest_generator.rb
+- lib/appydave/tools/dam/project_listing.rb
+- lib/appydave/tools/dam/project_resolver.rb
+- lib/appydave/tools/dam/s3_operations.rb
+- lib/appydave/tools/dam/sync_from_ssd.rb
 - lib/appydave/tools/debuggable.rb
 - lib/appydave/tools/gpt_context/_doc.md
 - lib/appydave/tools/gpt_context/file_collector.rb
@@ -301,12 +308,6 @@ files:
 - lib/appydave/tools/types/base_model.rb
 - lib/appydave/tools/types/hash_type.rb
 - lib/appydave/tools/types/indifferent_access_hash.rb
-- lib/appydave/tools/vat/config.rb
-- lib/appydave/tools/vat/config_loader.rb
-- lib/appydave/tools/vat/manifest_generator.rb
-- lib/appydave/tools/vat/project_listing.rb
-- lib/appydave/tools/vat/project_resolver.rb
-- lib/appydave/tools/vat/s3_operations.rb
 - lib/appydave/tools/version.rb
 - lib/appydave/tools/youtube_automation/_doc.md
 - lib/appydave/tools/youtube_automation/gpt_agent.rb