soft-sort 1.0.0

data/lib/soft_sort/configuration.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class Configuration
+    attr_reader :data
+
+    def initialize(data)
+      @data = data
+      validate!
+    end
+
+    def self.load(path = nil)
+      file = path || File.expand_path('../../config/default_rules.yml', __dir__)
+      new(YAML.safe_load_file(file, permitted_classes: [], aliases: false) || {})
+    rescue Psych::Exception, Errno::ENOENT => e
+      raise ConfigurationError, "invalid configuration: #{e.message}"
+    end
+
+    def categories = data.fetch('categories', {})
+    def context_rules = Array(data['context_rules'])
+    def projects = data.fetch('projects', {})
+    def exclusions = data.fetch('exclusions', {})
+    def organization = data.fetch('organization', {})
+
+    def extension_categories
+      categories.each_with_object({}) do |(name, cfg), h|
+        Array(cfg['extensions']).each do |e|
+          h[e.downcase] = [name, cfg['destination']]
+        end
+      end
+    end
+
+    def validate!
+      raise ConfigurationError, 'configuration must be a mapping' unless data.is_a?(Hash)
+
+      categories.each_value do |cfg|
+        unless cfg.is_a?(Hash) && safe_destination?(cfg['destination'].to_s)
+          raise ConfigurationError, 'category destination is invalid'
+        end
+      end
+      context_rules.each do |r|
+        unless r.is_a?(Hash) && r['name'] && Array(r['keywords']).any? && safe_destination?(r['destination'].to_s)
+          raise ConfigurationError, 'context rule is invalid'
+        end
+      end
+      projects.each_value do |project|
+        unless project.is_a?(Hash) && Array(project['aliases']).any? && safe_destination?(project['destination'].to_s)
+          raise ConfigurationError, 'project rule is invalid'
+        end
+      end
+      validate_organization!
+    end
+
+    def validate_organization!
+      date_format = organization.fetch('date_format', 'year_month')
+      date_source = organization.fetch('date_source', 'auto')
+      collision = organization.fetch('collision_policy', 'skip')
+      raise ConfigurationError, 'invalid date format' unless %w[year year_month year_month_day].include?(date_format)
+      raise ConfigurationError, 'invalid date source' unless %w[auto modified created captured].include?(date_source)
+      raise ConfigurationError, 'invalid collision policy' unless %w[skip review rename].include?(collision)
+      inspection = data.fetch('content_inspection', {})
+      limit = inspection.fetch('max_bytes', 1_048_576)
+      raise ConfigurationError, 'invalid content inspection size limit' unless limit.is_a?(Integer) && limit.positive?
+      ai = data.fetch('ai', {})
+      raise ConfigurationError, 'AI must be disabled or specify a provider' if ai.fetch('enabled', false) && ai.fetch('provider', 'none') == 'none'
+    end
+
+    def safe_destination?(value)
+      !value.empty? && !Pathname.new(value).absolute? && value.split(%r{[\\/]}).none? { |part| part == '..' }
+    end
+  end
+end

data/lib/soft_sort/duplicates/duplicate_detector.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class DuplicateDetector
+    def initialize(root)
+      @root = File.expand_path(root)
+    end
+
+    def find(files)
+      candidates = files.select(&:eligible).group_by(&:size).select { |_size, group| group.length > 1 }
+      candidates.values.flat_map do |group|
+        grouped = group.group_by { |file| Digest::SHA256.file(file.absolute_path).hexdigest }
+        grouped.values.select { |items| items.length > 1 }.map { |items| { hash: Digest::SHA256.file(items.first.absolute_path).hexdigest, files: items.map(&:relative_path), size: items.first.size } }
+      end
+    end
+  end
+end

data/lib/soft_sort/errors.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class ScanError < Error; end
+end

data/lib/soft_sort/execution/executor.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class Executor
+    def initialize(root, journal: TransactionJournal.new(root))
+      @root = File.expand_path(root)
+      @journal = journal
+    end
+
+    def execute(items, apply: false)
+      RootLock.new(@root).synchronize do
+        operations = items.map { |item| operation(item) }
+        record = @journal.create(@root, operations)
+        return record unless apply
+
+        record['status'] = 'in_progress'
+        @journal.update(record)
+        operations.each do |operation|
+          move(operation) if operation['action'] == 'move'
+          @journal.update(record)
+        end
+        record['status'] = operations.any? { |op| op['status'] == 'failed' } ? 'failed' : 'completed'
+        @journal.update(record)
+      end
+    rescue SystemCallError => e
+      record['status'] = 'failed' if record
+      record['error'] = e.message if record
+      @journal.update(record) if record
+      raise Error, "organization failed: #{e.message}"
+    end
+
+    private
+
+    def operation(item)
+      source = File.join(@root, item[:source])
+      { 'source' => item[:source], 'destination' => item[:destination], 'action' => item[:action],
+        'status' => item[:action] == 'move' ? 'pending' : 'skipped', 'sha256' => file_hash(source), 'size' => File.size(source) }
+    end
+
+    def move(op)
+      source = File.join(@root, op['source'])
+      destination = File.join(@root, op['destination'])
+      raise Error, 'source changed before move' unless File.file?(source) && file_hash(source) == op['sha256']
+      raise Error, 'destination already exists' if File.exist?(destination)
+      raise Error, 'destination escapes selected root' unless destination.start_with?(@root + File::SEPARATOR)
+
+      raise Error, 'source or destination is a symbolic link' if File.symlink?(source) || File.symlink?(destination)
+
+      FileUtils.mkdir_p(File.dirname(destination))
+      File.link(source, destination)
+      File.unlink(source)
+      op['status'] = 'moved'
+    rescue StandardError => e
+      op['status'] = 'failed'
+      op['error'] = e.message
+    end
+
+    def file_hash(path)
+      return nil unless File.file?(path)
+
+      digest = Digest::SHA256.new
+      File.open(path, 'rb') { |file| digest.update(file.read(1024 * 1024)) until file.eof? }
+      digest.hexdigest
+    end
+  end
+end

data/lib/soft_sort/execution/recovery_manager.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class RecoveryManager
+    def initialize(root)
+      @root = File.expand_path(root)
+      @journal = TransactionJournal.new(@root)
+    end
+
+    def inspect(transaction_id: nil)
+      records = @journal.all.select { |record| record['root'] == @root && recoverable_transaction?(record) }
+      records = records.select { |record| record['id'] == transaction_id } if transaction_id
+      records.filter_map { |record| reconcile(record) }
+    end
+
+    def apply(transaction_id: nil)
+      RootLock.new(@root).synchronize do
+        inspect(transaction_id: transaction_id).map do |report|
+          recovered = retry_recoverable(report)
+          record = @journal.all.find { |item| item['id'] == report['transaction'] }
+          if record
+            recovered['operations'].each do |operation|
+              journal_operation = record['operations'].find { |item| item['source'] == operation['source'] }
+              journal_operation['status'] = 'moved' if journal_operation && operation['state'] == 'completed'
+            end
+            record['status'] = recovered['conflicts'].zero? ? 'recovered' : 'recovery_required'
+            @journal.update(record)
+            recovered['status'] = record['status']
+          end
+          recovered
+        end
+      end
+    end
+
+    private
+
+    def recoverable_transaction?(record)
+      %w[planned prepared in_progress interrupted failed recovery_required partially_undone].include?(record['status'])
+    end
+
+    def reconcile(record)
+      states = record['operations'].map { |operation| reconcile_operation(operation) }
+      { 'transaction' => record['id'], 'status' => record['status'], 'operations' => states,
+        'completed' => states.count { |item| item['state'] == 'completed' },
+        'pending' => states.count { |item| item['state'] == 'pending' },
+        'conflicts' => states.count { |item| item['state'] == 'conflict' },
+        'recoverable' => states.count { |item| item['state'] == 'recoverable' } }
+    end
+
+    def reconcile_operation(operation)
+      return operation.merge('state' => 'skipped') unless operation['action'] == 'move'
+
+      source = File.join(@root, operation['source'])
+      destination = File.join(@root, operation['destination'])
+      return operation.merge('state' => 'unsafe') if File.symlink?(source) || File.symlink?(destination)
+      source_exists = File.file?(source)
+      destination_exists = File.file?(destination)
+      if source_exists && !destination_exists
+        return operation.merge('state' => Digest::SHA256.file(source).hexdigest == operation['sha256'] ? 'recoverable' : 'conflict')
+      end
+      return operation.merge('state' => 'completed') if !source_exists && destination_exists && hash(destination) == operation['sha256']
+      return operation.merge('state' => 'conflict') if source_exists && destination_exists
+
+      operation.merge('state' => 'recovery_required')
+    end
+
+    def retry_recoverable(report)
+      report['operations'].each do |operation|
+        next unless operation['state'] == 'recoverable'
+
+        source = File.join(@root, operation['source'])
+        destination = File.join(@root, operation['destination'])
+        FileUtils.mkdir_p(File.dirname(destination))
+        File.link(source, destination)
+        File.unlink(source)
+        operation['state'] = 'completed'
+      end
+      report
+    end
+
+    def hash(path)
+      Digest::SHA256.file(path).hexdigest
+    rescue SystemCallError
+      nil
+    end
+  end
+end

data/lib/soft_sort/execution/undo_manager.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class UndoManager
+    def initialize(root)
+      @root = File.expand_path(root)
+      @journal = TransactionJournal.new(@root)
+    end
+
+    def undo(transaction_id: nil, dry_run: false)
+      return undo_locked(transaction_id: transaction_id, dry_run: dry_run) if dry_run
+
+      RootLock.new(@root).synchronize { undo_locked(transaction_id: transaction_id, dry_run: false) }
+    end
+
+    private
+
+    def undo_locked(transaction_id:, dry_run:)
+      records = @journal.all.select { |record| record['root'] == @root }
+      record = transaction_id ? records.reverse.find { |item| item['id'] == transaction_id } : records.reverse.find { |item| item['status'] == 'completed' }
+      raise Error, 'transaction not found' unless record
+
+      results = record['operations'].reverse.map do |op|
+        next { 'source' => op['destination'], 'destination' => op['source'], 'status' => 'skipped' } unless op['status'] == 'moved'
+        from = File.join(@root, op['destination'])
+        to = File.join(@root, op['source'])
+        if dry_run
+          { 'source' => op['destination'], 'destination' => op['source'], 'status' => 'planned' }
+        elsif !File.file?(from) || File.exist?(to) || Digest::SHA256.file(from).hexdigest != op['sha256']
+          { 'source' => op['destination'], 'destination' => op['source'], 'status' => 'conflict' }
+        else
+          raise Error, 'undo source or destination is a symbolic link' if File.symlink?(from) || File.symlink?(to)
+
+          FileUtils.mkdir_p(File.dirname(to))
+          File.link(from, to)
+          File.unlink(from)
+          { 'source' => op['destination'], 'destination' => op['source'], 'status' => 'undone' }
+        end
+      end
+      record['status'] = results.any? { |item| item['status'] == 'conflict' } ? 'partially_undone' : 'undone'
+      @journal.update(record) unless dry_run
+      results
+    end
+  end
+end

data/lib/soft_sort/metadata/image_metadata.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class ImageMetadata
+    def self.read(file, max_bytes: 65_536)
+      return nil unless %w[png jpg jpeg].include?(file.extension)
+
+      bytes = File.open(file.absolute_path, 'rb') { |stream| stream.read(max_bytes) }
+      return png_metadata(bytes) if file.extension == 'png'
+
+      nil
+    rescue SystemCallError
+      nil
+    end
+
+    def self.png_metadata(bytes)
+      signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a].pack('C*')
+      return nil unless bytes && bytes.byteslice(0, 8) == signature && bytes.bytesize >= 20
+
+      { width: bytes.byteslice(16, 4).unpack1('N'), height: bytes.byteslice(20, 4).unpack1('N'), capture_time: nil }
+    end
+    private_class_method :png_metadata
+  end
+end

data/lib/soft_sort/planning/organization_planner.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class OrganizationPlanner
+    def initialize(root, config)
+      @root = File.expand_path(root)
+      @matcher = RuleMatcher.new(config)
+      @protect_existing = config.data.fetch('safety', {}).fetch('protect_existing_project_structures', true)
+      @organization = config.organization
+    end
+
+    def plan(files)
+      results = files.select(&:eligible).map { |file| build(file) }
+      apply_collision_policy(results)
+      destinations = results.group_by { |item| item[:destination] }.transform_values(&:length)
+      results.each do |result|
+        result[:action] = 'conflict' if destinations[result[:destination]] > 1 || existing_destination?(result)
+      end
+      results
+    end
+
+    private
+
+    def existing_destination?(result)
+      File.exist?(File.join(@root, result[:destination])) && result[:source] != result[:destination]
+    end
+
+    def build(file)
+      classification = @matcher.classify(file)
+      destination_root = date_destination(classification.destination, file)
+      destination = Pathname.new(File.join(destination_root, file.filename)).cleanpath.to_s
+      raise ConfigurationError, 'destination escapes selected root' if destination == '..' || destination.start_with?('../')
+
+      action = if destination == file.relative_path
+                 'skip'
+               elsif @protect_existing && file.parent_directory != '.'
+                 'skip'
+               elsif file.sensitive_reason
+                 'review'
+               elsif classification.requires_review
+                 'review'
+               else
+                 'move'
+               end
+      { source: file.relative_path, destination: destination, type_category: classification.type_category,
+        context_category: classification.context_category, matched_rule: classification.matched_rule,
+        confidence: classification.confidence, action: action,
+        warning: warning_for(file, classification) }
+    end
+
+    def date_destination(destination, file)
+      return destination unless @organization.fetch('date_folders', false)
+
+      timestamp = case @organization.fetch('date_source', 'auto')
+                  when 'created' then file.created_at
+                  when 'modified' then file.modified_at
+                  else file.created_at || file.modified_at
+                  end
+      return destination unless timestamp.respond_to?(:year)
+
+      parts = case @organization.fetch('date_format', 'year_month')
+              when 'year' then [timestamp.year]
+              when 'year_month_day' then [timestamp.year, format('%02d', timestamp.month), format('%02d', timestamp.day)]
+              else [timestamp.year, format('%02d', timestamp.month)]
+              end
+      File.join(destination, *parts.map(&:to_s))
+    end
+
+    def apply_collision_policy(results)
+      used = {}
+      results.each do |result|
+        collision = result[:action] == 'move' && (File.exist?(File.join(@root, result[:destination])) || used.key?(result[:destination].downcase))
+        unless collision
+          used[result[:destination].downcase] = true
+          next
+        end
+
+        case @organization.fetch('collision_policy', 'skip')
+        when 'rename'
+          result[:destination] = available_name(result[:destination], used)
+        when 'review'
+          result[:action] = 'review'
+          result[:warning] = 'destination collision requires review'
+        end
+        used[result[:destination].downcase] = true
+      end
+    end
+
+    def available_name(destination, used)
+      return destination unless used.key?(destination.downcase) || File.exist?(File.join(@root, destination))
+
+      extension = File.extname(destination)
+      stem = destination.delete_suffix(extension)
+      index = 1
+      candidate = "#{stem} (#{index})#{extension}"
+      while used.key?(candidate.downcase) || File.exist?(File.join(@root, candidate))
+        index += 1
+        candidate = "#{stem} (#{index})#{extension}"
+      end
+      candidate
+    end
+
+    def warning_for(file, classification)
+      return file.sensitive_reason if file.sensitive_reason
+      return 'existing directory structure protected' if protected_structure?(file)
+      return 'classification requires review' if classification.requires_review
+
+      nil
+    end
+
+    def protected_structure?(file)
+      @protect_existing && file.parent_directory != '.'
+    end
+  end
+end

data/lib/soft_sort/reporting/json_reporter.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class JsonReporter
+    def self.dump(object) = JSON.generate(object)
+  end
+end

data/lib/soft_sort/reporting/preview_formatter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class PreviewFormatter
+    def self.scan(report)
+      "Files: #{report[:total]} | Eligible: #{report[:eligible]} | Excluded: #{report[:excluded]} | Size: #{report[:bytes]} bytes\n" + report[:categories].map { |k, v|
+        "  #{k}: #{v}"
+      }.join("\n")
+    end
+
+    def self.plan(items)
+      items.map do |i|
+        "#{i[:action].upcase.ljust(8)} #{i[:source]} -> #{i[:destination]} (#{i[:type_category]}, #{i[:confidence]})"
+      end.join("\n")
+    end
+  end
+end

data/lib/soft_sort/reporting/statistics_reporter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class StatisticsReporter
+    def self.build(scan, plan, duplicates: [])
+      { total_files: scan[:files].length, eligible_files: scan[:files].count(&:eligible), total_bytes: scan[:files].select(&:eligible).sum(&:size),
+        planned_moves: plan.count { |item| item[:action] == 'move' }, conflicts: plan.count { |item| item[:action] == 'conflict' },
+        reviews: plan.count { |item| item[:action] == 'review' }, duplicate_groups: duplicates.length,
+        categories: plan.group_by { |item| item[:type_category] }.transform_values(&:length),
+        contexts: plan.group_by { |item| item[:context_category] || 'Unclassified' }.transform_values(&:length) }
+    end
+  end
+end

data/lib/soft_sort/scanning/directory_scanner.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class DirectoryScanner
+    DEFAULT_EXCLUDED = %w[.git node_modules .venv vendor/bundle __pycache__ .soft-sort].freeze
+    DEFAULT_TEMP = %w[part crdownload download tmp].freeze
+    attr_reader :root
+
+    def initialize(root, config:, recursive: false)
+      expanded = root.to_s.start_with?('~') ? File.expand_path(root) : root.to_s
+      @root = File.expand_path(expanded)
+      raise ScanError, "directory does not exist: #{root}" unless File.exist?(@root)
+      raise ScanError, "not a directory: #{root}" unless File.directory?(@root)
+
+      @config = config
+      @recursive = recursive
+    end
+
+    def scan
+      files = []
+      warnings = []
+      paths = @recursive ? Dir.glob(File.join(@root, '**', '*'), File::FNM_DOTMATCH) : Dir.children(@root).map { |n| File.join(@root, n) }
+      paths.each do |path|
+        next unless File.file?(path) || File.symlink?(path)
+
+        rel = Pathname.new(path).relative_path_from(Pathname.new(@root)).to_s
+        begin
+          reason = exclusion_reason(rel, path)
+          stat = File.lstat(path)
+          files << FileMetadata.new(absolute_path: path, relative_path: rel, filename: File.basename(path),
+                                    extension: File.extname(path).delete_prefix('.').downcase, size: stat.file? ? stat.size : 0,
+                                    modified_at: stat.mtime, created_at: birthtime(stat),
+                                    parent_directory: File.dirname(rel), eligible: reason.nil?, exclusion_reason: reason,
+                                    sensitive_reason: SensitiveFileDetector.reason(File.basename(path)))
+        rescue SystemCallError => e
+          warnings << "#{rel}: #{e.message}"
+        end
+      end
+      { files: files, warnings: warnings }
+    rescue SystemCallError => e
+      raise ScanError, "unable to scan #{@root}: #{e.message}"
+    end
+
+    private
+
+    def birthtime(stat)
+      stat.birthtime
+    rescue NotImplementedError, SystemCallError
+      nil
+    end
+
+    def exclusion_reason(rel, path)
+      return 'symbolic link' if File.symlink?(path)
+
+      parts = rel.split(File::SEPARATOR)
+      return 'hidden file or directory' if !@config.data.dig('safety', 'include_hidden') && parts.any? do |p|
+        p.start_with?('.')
+      end
+
+      dirs = DEFAULT_EXCLUDED + Array(@config.exclusions['directories'])
+      return 'excluded directory' if parts[0...-1].any? { |p| dirs.include?(p) }
+      if (Array(@config.exclusions['extensions']).map(&:downcase) + DEFAULT_TEMP).include?(File.extname(path).delete_prefix('.').downcase)
+        return 'temporary or incomplete download'
+      end
+
+      nil
+    end
+  end
+end

data/lib/soft_sort/scanning/file_metadata.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module SoftSort
+  FileMetadata = Struct.new(:absolute_path, :relative_path, :filename, :extension, :size, :modified_at, :created_at,
+                            :parent_directory, :eligible, :exclusion_reason, :sensitive_reason, keyword_init: true)
+end

data/lib/soft_sort/storage/root_lock.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class RootLock
+    def initialize(root)
+      @path = File.join(File.expand_path(root), '.soft-sort', 'root.lock')
+    end
+
+    def synchronize
+      FileUtils.mkdir_p(File.dirname(@path))
+      File.open(@path, File::RDWR | File::CREAT, 0o600) do |lock|
+        lock.flock(File::LOCK_EX)
+        yield
+      ensure
+        lock&.flock(File::LOCK_UN)
+      end
+    rescue SystemCallError => e
+      raise Error, "unable to lock selected root: #{e.message}"
+    end
+  end
+end

data/lib/soft_sort/storage/transaction_journal.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SoftSort
+  class TransactionJournal
+    def initialize(root)
+      @directory = File.join(File.expand_path(root), '.soft-sort')
+      @path = File.join(@directory, 'transactions.jsonl')
+      @warnings = []
+    end
+
+    attr_reader :warnings
+
+    def create(root, operations)
+      record = { 'id' => SecureRandom.uuid, 'root' => File.expand_path(root), 'started_at' => Time.now.utc.iso8601,
+                 'completed_at' => nil, 'status' => 'planned', 'operations' => operations }
+      append(record)
+      record
+    end
+
+    def update(record)
+      record['completed_at'] = Time.now.utc.iso8601 if %w[completed failed undone partially_undone].include?(record['status'])
+      append(record)
+      record
+    end
+
+    def all
+      return [] unless File.file?(@path)
+
+      records = {}
+      File.foreach(@path).with_index(1) do |line, line_number|
+        begin
+          record = JSON.parse(line)
+          records[record['id']] = record
+        rescue JSON::ParserError => e
+          @warnings << "line #{line_number}: malformed journal record (#{e.message})"
+        end
+      end
+      records.values
+    end
+
+    private
+
+    def append(record)
+      FileUtils.mkdir_p(@directory)
+      File.open(@path, File::WRONLY | File::APPEND | File::CREAT, 0o600) do |file|
+        file.flock(File::LOCK_EX)
+        file.write(JSON.generate(record))
+        file.write("\n")
+        file.flush
+        file.fsync
+      ensure
+        file&.flock(File::LOCK_UN)
+      end
+    end
+  end
+end

data/lib/soft_sort/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SoftSort
+  VERSION = '1.0.0'
+end