appydave-tools 0.17.0 → 0.18.0

data/lib/appydave/tools/dam/sync_from_ssd.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module Appydave
+  module Tools
+    module Dam
+      # Sync light files from SSD to local for a brand
+      # Only copies non-video files (subtitles, images, docs)
+      class SyncFromSsd
+        attr_reader :brand, :brand_info, :brand_path
+
+        # Light file patterns to include (everything except heavy video files)
+        LIGHT_FILE_PATTERNS = %w[
+          **/*.srt
+          **/*.vtt
+          **/*.txt
+          **/*.md
+          **/*.jpg
+          **/*.jpeg
+          **/*.png
+          **/*.webp
+          **/*.json
+          **/*.yml
+          **/*.yaml
+        ].freeze
+
+        # Heavy file patterns to exclude (video files)
+        HEAVY_FILE_PATTERNS = %w[
+          *.mp4
+          *.mov
+          *.avi
+          *.mkv
+          *.webm
+        ].freeze
+
+        def initialize(brand, brand_info: nil, brand_path: nil)
+          @brand_info = brand_info || load_brand_info(brand)
+          @brand = @brand_info.key # Use resolved brand key, not original input
+          @brand_path = brand_path || Config.brand_path(@brand)
+        end
+
+        # Sync light files from SSD for all projects in manifest
+        def sync(dry_run: false)
+          puts dry_run ? '🔍 DRY-RUN MODE - No files will be copied' : '📦 Syncing from SSD...'
+          puts ''
+
+          # Validate SSD is mounted
+          ssd_backup = brand_info.locations.ssd_backup
+          unless ssd_backup && !ssd_backup.empty?
+            puts "❌ SSD backup location not configured for brand '#{brand}'"
+            return
+          end
+
+          unless Dir.exist?(ssd_backup)
+            puts "❌ SSD not mounted at #{ssd_backup}"
+            return
+          end
+
+          # Load manifest
+          manifest_file = File.join(brand_path, 'projects.json')
+          unless File.exist?(manifest_file)
+            puts '❌ projects.json not found!'
+            puts "   Run: vat manifest #{brand}"
+            return
+          end
+
+          manifest = load_manifest(manifest_file)
+          puts "📋 Loaded manifest: #{manifest[:projects].size} projects"
+          puts "   Last updated: #{manifest[:config][:last_updated]}"
+          puts ''
+
+          # Filter projects to sync
+          projects_to_sync = manifest[:projects].select { |p| should_sync_project?(p) }
+
+          puts '🔍 Analysis:'
+          puts "   Total projects in manifest: #{manifest[:projects].size}"
+          puts "   Projects to sync: #{projects_to_sync.size}"
+          puts "   Skipped (already local): #{manifest[:projects].size - projects_to_sync.size}"
+          puts ''
+
+          if projects_to_sync.empty?
+            puts '✅ Nothing to sync - all projects either already local or not on SSD'
+            return
+          end
+
+          # Sync each project
+          total_stats = { files: 0, bytes: 0, skipped: 0 }
+
+          projects_to_sync.each do |project|
+            stats = sync_project(project, ssd_backup, dry_run: dry_run)
+
+            # Only show project if there are files to sync or a warning
+            if stats[:reason] || stats[:files]&.positive?
+              puts "📁 #{project[:id]}"
+
+              puts "  ⚠️  Skipped: #{stats[:reason]}" if stats[:reason]
+
+              puts "  #{stats[:files]} file(s), #{format_bytes(stats[:bytes])}" if stats[:files]&.positive?
+              puts ''
+            end
+
+            total_stats[:files] += stats[:files] || 0
+            total_stats[:bytes] += stats[:bytes] || 0
+            total_stats[:skipped] += stats[:skipped] || 0
+          end
+
+          puts ''
+          puts '=' * 60
+          puts 'Summary:'
+          puts "  Projects scanned: #{projects_to_sync.size}"
+          puts "  Files #{dry_run ? 'to copy' : 'copied'}: #{total_stats[:files]}"
+          puts "  Total size: #{format_bytes(total_stats[:bytes])}"
+          puts ''
+          puts '✅ Sync complete!'
+          puts '   Run without --dry-run to perform the sync' if dry_run
+        end
+
+        private
+
+        def load_brand_info(brand)
+          Appydave::Tools::Configuration::Config.configure
+          Appydave::Tools::Configuration::Config.brands.get_brand(brand)
+        end
+
+        def load_manifest(manifest_file)
+          JSON.parse(File.read(manifest_file), symbolize_names: true)
+        rescue JSON::ParserError => e
+          puts "❌ Error parsing projects.json: #{e.message}"
+          exit 1
+        end
+
+        # Determine if project should be synced
+        def should_sync_project?(project)
+          # Only sync if project exists on SSD but NOT in local flat structure
+          return false unless project[:storage][:ssd][:exists]
+
+          # Skip if exists locally in flat structure
+          return false if project[:storage][:local][:exists] && project[:storage][:local][:structure] == 'flat'
+
+          true
+        end
+
+        # Sync a single project from SSD to local
+        def sync_project(project, ssd_backup, dry_run: false)
+          project_id = project[:id]
+          ssd_path = File.join(ssd_backup, project_id)
+
+          return { skipped: 1, files: 0, bytes: 0, reason: 'SSD path not found' } unless Dir.exist?(ssd_path)
+
+          # Check for flat folder conflict (stale manifest)
+          flat_path = File.join(brand_path, project_id)
+          return { skipped: 1, files: 0, bytes: 0, reason: 'Flat folder exists (stale manifest?)' } if Dir.exist?(flat_path)
+
+          # Determine local destination path (archived structure)
+          # Extract range from project ID (e.g., b65 → 60-69 range)
+          range = determine_range(project_id)
+          local_dir = File.join(brand_path, 'archived', range, project_id)
+
+          # Create local directory
+          FileUtils.mkdir_p(local_dir) if !dry_run && !Dir.exist?(local_dir)
+
+          # Sync light files
+          sync_light_files(ssd_path, local_dir, dry_run: dry_run)
+        end
+
+        # Determine range folder for project (e.g., b65 → 60-69)
+        def determine_range(project_id)
+          # FliVideo pattern: b40, b41, ... b99
+          if project_id =~ /^b(\d+)/
+            tens = (Regexp.last_match(1).to_i / 10) * 10
+            "#{tens}-#{tens + 9}"
+          else
+            # Legacy pattern or unknown: use first 3 chars
+            '000-099'
+          end
+        end
+
+        # Sync light files from SSD to local
+        def sync_light_files(ssd_path, local_dir, dry_run: false)
+          stats = { files: 0, bytes: 0 }
+
+          LIGHT_FILE_PATTERNS.each do |pattern|
+            Dir.glob(File.join(ssd_path, pattern)).each do |source_file|
+              next if heavy_file?(source_file)
+
+              copy_stats = copy_light_file(source_file, ssd_path, local_dir, dry_run: dry_run)
+              stats[:files] += copy_stats[:files]
+              stats[:bytes] += copy_stats[:bytes]
+            end
+          end
+
+          stats
+        end
+
+        # Check if file is a heavy video file
+        def heavy_file?(source_file)
+          HEAVY_FILE_PATTERNS.any? { |pattern| File.fnmatch(pattern, File.basename(source_file)) }
+        end
+
+        # Copy a single light file
+        def copy_light_file(source_file, ssd_path, local_dir, dry_run: false)
+          relative_path = source_file.sub("#{ssd_path}/", '')
+          dest_file = File.join(local_dir, relative_path)
+
+          # Skip if already synced (same size)
+          return { files: 0, bytes: 0 } if file_already_synced?(source_file, dest_file)
+
+          file_size = File.size(source_file)
+
+          if dry_run
+            puts "  [DRY-RUN] Would copy: #{relative_path} (#{format_bytes(file_size)})"
+          else
+            FileUtils.mkdir_p(File.dirname(dest_file))
+            FileUtils.cp(source_file, dest_file, preserve: true)
+            puts "  ✓ Copied: #{relative_path} (#{format_bytes(file_size)})"
+          end
+
+          { files: 1, bytes: file_size }
+        end
+
+        # Check if file is already synced (by size comparison)
+        def file_already_synced?(source_file, dest_file)
+          File.exist?(dest_file) && File.size(dest_file) == File.size(source_file)
+        end
+
+        # Format bytes into human-readable format
+        def format_bytes(bytes)
+          if bytes < 1024
+            "#{bytes}B"
+          elsif bytes < 1024 * 1024
+            "#{(bytes / 1024.0).round(1)}KB"
+          else
+            "#{(bytes / 1024.0 / 1024.0).round(1)}MB"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Appydave
   module Tools
-    VERSION = '0.17.0'
+    VERSION = '0.18.0'
   end
 end

data/lib/appydave/tools.rb

@@ -52,12 +52,13 @@ require 'appydave/tools/prompt_tools/prompt_completion'
 require 'appydave/tools/subtitle_processor/clean'
 require 'appydave/tools/subtitle_processor/join'
 
-require 'appydave/tools/vat/config'
-require 'appydave/tools/vat/project_resolver'
-require 'appydave/tools/vat/config_loader'
-require 'appydave/tools/vat/s3_operations'
-require 'appydave/tools/vat/project_listing'
-require 'appydave/tools/vat/manifest_generator'
+require 'appydave/tools/dam/config'
+require 'appydave/tools/dam/project_resolver'
+require 'appydave/tools/dam/config_loader'
+require 'appydave/tools/dam/s3_operations'
+require 'appydave/tools/dam/project_listing'
+require 'appydave/tools/dam/manifest_generator'
+require 'appydave/tools/dam/sync_from_ssd'
 
 require 'appydave/tools/youtube_automation/gpt_agent'
 

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appydave-tools",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "AppyDave YouTube Automation Tools",
   "scripts": {
     "release": "semantic-release"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: appydave-tools
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - David Cruwys
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-11-09 00:00:00.000000000 Z
+date: 2025-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -214,6 +214,7 @@ files:
 - bin/bank_reconciliation.rb
 - bin/configuration.rb
 - bin/console
+- bin/dam
 - bin/generate_manifest.rb
 - bin/gpt_context.rb
 - bin/move_images.rb
@@ -223,7 +224,6 @@ files:
 - bin/subtitle_manager.rb
 - bin/subtitle_processor.rb
 - bin/sync_from_ssd.rb
-- bin/vat
 - bin/youtube_automation.rb
 - bin/youtube_manager.rb
 - docs/README.md
@@ -237,6 +237,10 @@ files:
 - docs/configuration/README.md
 - docs/configuration/channels.example.json
 - docs/configuration/settings.example.json
+- docs/dam/dam-vision.md
+- docs/dam/session-summary-2025-11-09.md
+- docs/dam/usage.md
+- docs/dam/vat-testing-plan.md
 - docs/development/CODEX-recommendations.md
 - docs/development/README.md
 - docs/development/cli-architecture-patterns.md
@@ -254,10 +258,6 @@ files:
 - docs/tools/subtitle-processor.md
 - docs/tools/youtube-automation.md
 - docs/tools/youtube-manager.md
-- docs/vat/dam-vision.md
-- docs/vat/session-summary-2025-11-09.md
-- docs/vat/usage.md
-- docs/vat/vat-testing-plan.md
 - exe/ad_config
 - exe/gpt_context
 - exe/prompt_tools
@@ -281,6 +281,13 @@ files:
 - lib/appydave/tools/configuration/models/settings_config.rb
 - lib/appydave/tools/configuration/models/youtube_automation_config.rb
 - lib/appydave/tools/configuration/openai.rb
+- lib/appydave/tools/dam/config.rb
+- lib/appydave/tools/dam/config_loader.rb
+- lib/appydave/tools/dam/manifest_generator.rb
+- lib/appydave/tools/dam/project_listing.rb
+- lib/appydave/tools/dam/project_resolver.rb
+- lib/appydave/tools/dam/s3_operations.rb
+- lib/appydave/tools/dam/sync_from_ssd.rb
 - lib/appydave/tools/debuggable.rb
 - lib/appydave/tools/gpt_context/_doc.md
 - lib/appydave/tools/gpt_context/file_collector.rb
@@ -301,12 +308,6 @@ files:
 - lib/appydave/tools/types/base_model.rb
 - lib/appydave/tools/types/hash_type.rb
 - lib/appydave/tools/types/indifferent_access_hash.rb
-- lib/appydave/tools/vat/config.rb
-- lib/appydave/tools/vat/config_loader.rb
-- lib/appydave/tools/vat/manifest_generator.rb
-- lib/appydave/tools/vat/project_listing.rb
-- lib/appydave/tools/vat/project_resolver.rb
-- lib/appydave/tools/vat/s3_operations.rb
 - lib/appydave/tools/version.rb
 - lib/appydave/tools/youtube_automation/_doc.md
 - lib/appydave/tools/youtube_automation/gpt_agent.rb

data/docs/vat/session-summary-2025-11-09.md

@@ -1,297 +0,0 @@
-# VAT S3 Operations - Session Summary
-**Date**: 2025-11-09
-**Status**: Ready to Continue - Archive/SSD Commands Next
-
----
-
-## What We Accomplished Today
-
-### 1. Fixed RuboCop Issues Properly ✅
-
-**Problem**: Initially disabled RuboCop cops instead of fixing underlying issues.
-
-**Solution**: Refactored S3Operations to use **dependency injection**:
-```ruby
-# Before (mocking singletons):
-allow_any_instance_of(BrandsConfig).to receive(:get_brand)
-S3Operations.new('test', 'test-project')
-
-# After (dependency injection):
-S3Operations.new('test', 'test-project',
-  brand_info: real_brand_info_object,
-  brand_path: brand_path,
-  s3_client: mock_s3_client
-)
-```
-
-**Benefits**:
-- Eliminated ALL `allow_any_instance_of` usage
-- Tests use real `BrandInfo` objects (not mocks)
-- More flexible, testable, SOLID design
-- Backward compatible (production code unchanged)
-
-**RuboCop Config**: Disabled `RSpec/MessageSpies` and `RSpec/StubbedMock` globally (legitimate for AWS SDK integration tests)
-
-### 2. Added S3 Cleanup Local Command ✅
-
-**New Command**: `vat s3-cleanup-local <brand> <project> --force [--dry-run]`
-
-**Features**:
-- Deletes files from local `s3-staging/` directory
-- Requires `--force` flag for safety
-- Supports `--dry-run` mode
-- Removes empty directories after cleanup
-- Shows deleted/failed counts
-- Auto-detection from PWD
-
-**Implementation**:
-- Added `cleanup_local` method to S3Operations (lib/appydave/tools/vat/s3_operations.rb:212-275)
-- Added `delete_local_file` private method (lib/appydave/tools/vat/s3_operations.rb:359-371)
-- Added CLI command handler
-- Added comprehensive help documentation
-
-**Tests**: 6 tests covering all scenarios (289 total tests, 90.59% coverage)
-
-### 3. Fixed S3 Status Command ✅
-
-**Problem**: Status only showed 3 states (synced, modified, S3 only) - missing "local only" files.
-
-**Solution**: Enhanced status to show all 4 states:
-- ✓ **[synced]** - Files match (MD5)
-- ⚠️ **[modified]** - Files differ
-- ☁️ **[S3 only]** - File in S3 but not local
-- 📁 **[local only]** - File local but not in S3 ⭐ NEW
-
-**Implementation**:
-- Rewrote `status` method to check both S3 AND local files (lib/appydave/tools/vat/s3_operations.rb:139-202)
-- Added `list_local_files` helper method (lib/appydave/tools/vat/s3_operations.rb:424-434)
-- Enhanced summary: Shows file counts and sizes for both S3 and local
-
-**Tests**: Added 2 new tests (local-only files, comprehensive summary)
-
-### 4. Renamed Cleanup Commands for Consistency ✅
-
-**Old Names**:
-- `vat s3-cleanup` → Delete S3 files
-- `vat cleanup-local` → Delete local files
-
-**New Names**:
-- `vat s3-cleanup-remote` → Delete S3 files
-- `vat s3-cleanup-local` → Delete local files
-
-**Backward Compatibility**: Old names still work but show deprecation warning
-
-**Changes**:
-- Updated command registration (bin/vat:13-24)
-- Renamed methods
-- Updated all help documentation
-- Added deprecation notices
-
----
-
-## Testing Status
-
-### Automated Tests ✅
-- **Session 1**: 289 examples, 0 failures, 90.59% coverage
-- **Session 2**: 297 examples, 0 failures, 90.69% coverage (+8 tests, +0.10% coverage)
-- **RuboCop**: Clean (no offenses)
-
-### Manual Testing (David) ✅
-**Session 1**:
-- ✅ `vat s3-cleanup-remote` - Tested and working
-- ✅ `vat s3-cleanup-local` - Tested and working
-- ✅ `vat s3-status` - Shows all 4 states correctly (synced, modified, S3 only, local only)
-
-**Session 2**:
-- ⏳ `vat archive` - Ready for manual testing
-
-### S3Operations Test Coverage
-- **Session 1**: 33 tests (upload, download, status, cleanup, cleanup_local)
-- **Session 2**: 41 tests (+8 archive tests)
-
----
-
-## Current State
-
-### Implemented VAT Commands
-1. ✅ `vat list [brand] [pattern]` - List brands/projects
-2. ✅ `vat s3-up <brand> <project>` - Upload to S3
-3. ✅ `vat s3-down <brand> <project>` - Download from S3
-4. ✅ `vat s3-status <brand> <project>` - Check sync status (all 4 states)
-5. ✅ `vat s3-cleanup-remote <brand> <project>` - Delete S3 files
-6. ✅ `vat s3-cleanup-local <brand> <project>` - Delete local files
-7. ✅ `vat archive <brand> <project>` - Copy to SSD backup ⭐ NEW (2025-11-09 Session 2)
-
-### Not Yet Implemented
-1. ⏳ `vat sync-ssd <brand>` - Restore from SSD
-
----
-
-## Session 2 (2025-11-09 Evening) - Archive Command Implementation ✅
-
-### What We Accomplished
-
-#### 1. Implemented Archive Command ✅
-**New Command**: `vat archive <brand> <project> [--force] [--dry-run]`
-
-**Features**:
-- Copies entire project directory to SSD backup location
-- Verifies SSD is mounted before archiving
-- Shows project size before copying
-- Optional: Delete local copy after successful archive (--force)
-- Dry-run support for preview
-- Auto-detection from PWD
-
-**Implementation**:
-- Added `archive` method to S3Operations (lib/appydave/tools/vat/s3_operations.rb:298-340)
-- Added helper methods:
-  - `copy_to_ssd` (lib/appydave/tools/vat/s3_operations.rb:493-522)
-  - `delete_local_project` (lib/appydave/tools/vat/s3_operations.rb:524-547)
-  - `calculate_directory_size` (lib/appydave/tools/vat/s3_operations.rb:549-556)
-- Added CLI command handler (bin/vat:148-156)
-- Added comprehensive help documentation (bin/vat:480-515)
-
-**Tests**: 8 new tests covering all scenarios (297 total tests, 90.69% coverage)
-
-**Test Coverage**:
-- ✅ SSD backup location not configured
-- ✅ SSD not mounted
-- ✅ Project does not exist
-- ✅ Dry-run preview
-- ✅ Copy without deleting local (default)
-- ✅ Warning when not using --force
-- ✅ Copy and delete with --force
-- ✅ Skip when already exists on SSD
-
-**Storage Strategy**: Local → S3 (90-day collaboration) → SSD (long-term archive)
-
-**Configuration**: Uses `ssd_backup` location from brands.json (already configured for all 6 brands)
-
----
-
-## Next Steps (For Future Session)
-
-### Priority 1: Sync from SSD Command ⏳
-Implement `vat sync-ssd <brand>` to restore projects from SSD:
-
-**Requirements** (from original discussion):
-- Copy entire project directory to SSD backup location
-- Verify copy completed successfully
-- Option to remove local copy after successful archive
-- Support dry-run mode
-- Show progress for large projects
-
-**Configuration** (already in brands.json):
-```json
-"locations": {
-  "video_projects": "/Users/davidcruwys/dev/video-projects/v-appydave",
-  "ssd_backup": "/Volumes/T7/youtube-PUBLISHED/appydave"
-}
-```
-
-**Implementation Plan**:
-1. Add `archive` method to new class (or S3Operations? Or separate ArchiveOperations?)
-2. CLI command handler
-3. Tests
-4. Help documentation
-
-### Priority 2: Sync from SSD Command
-Implement `vat sync-ssd <brand>` to restore projects from SSD:
-
-**Requirements**:
-- List available projects on SSD
-- Copy selected project(s) back to local
-- Smart sync (skip if already exists? overwrite? merge?)
-- Dry-run support
-
-### Priority 3: Documentation
-- Document AWS permissions strategy for team members
-- Update CLAUDE.md with latest VAT commands
-- Update usage documentation
-
----
-
-## Architecture Notes
-
-### Dependency Injection Pattern Established
-All new code should follow the pattern established in S3Operations:
-
-```ruby
-class MyOperations
-  def initialize(brand, project_id, brand_info: nil, brand_path: nil)
-    @brand = brand
-    @project_id = project_id
-    @brand_info = brand_info || load_from_config(brand)
-    @brand_path = brand_path || Config.brand_path(brand)
-  end
-
-  private
-
-  def load_from_config(brand)
-    Config.configure
-    Config.brands.get_brand(brand)
-  end
-end
-```
-
-**Tests**:
-```ruby
-let(:brand_info) { BrandInfo.new('test', test_data) }  # Real object
-
-def create_operations
-  described_class.new('test', 'test-project',
-    brand_info: brand_info,
-    brand_path: brand_path
-  )
-end
-```
-
-### File Locations
-- **S3 operations**: `lib/appydave/tools/vat/s3_operations.rb`
-- **S3 tests**: `spec/appydave/tools/vat/s3_operations_spec.rb`
-- **VAT CLI**: `bin/vat`
-- **Brands config**: `lib/appydave/tools/configuration/models/brands_config.rb`
-
----
-
-## Questions for Tomorrow
-
-1. **Archive behavior**: Should archive DELETE local files after successful copy, or just copy and leave local intact?
-2. **Archive scope**: Archive just the project directory, or also s3-staging/?
-3. **SSD sync**: Should it sync ALL projects from a brand, or let user select specific ones?
-4. **Conflict handling**: If project exists both locally and on SSD, what should sync-ssd do?
-
----
-
-## Key Files Modified
-
-### Session 1
-1. `lib/appydave/tools/vat/s3_operations.rb` - Refactored with DI, added cleanup_local, fixed status
-2. `spec/appydave/tools/vat/s3_operations_spec.rb` - Rewrote to use DI, added new tests
-3. `bin/vat` - Renamed cleanup commands, updated help
-4. `.rubocop.yml` - Disabled MessageSpies and StubbedMock cops
-
-### Session 2
-1. `lib/appydave/tools/vat/s3_operations.rb` - Added archive method and helper methods
-2. `spec/appydave/tools/vat/s3_operations_spec.rb` - Added 8 archive tests
-3. `bin/vat` - Added archive command handler and help documentation
-4. `docs/vat/session-summary-2025-11-09.md` - Updated with Session 2 accomplishments
-
----
-
-## Git Status
-**Session 1**:
-- 289 tests passing
-- RuboCop clean
-- S3 cleanup commands complete
-
-**Session 2**:
-- 297 tests passing (+8 archive tests)
-- 90.69% code coverage (+0.10%)
-- RuboCop clean
-- Archive command complete
-- Ready to commit
-
----
-
-**Next Session**: Implement `vat sync-ssd <brand>` command for restoring projects from SSD backup.