active_storage_dedup 1.0.0.alpha

data/lib/tasks/active_storage_dedup.rake

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+namespace :active_storage_dedup do
+  desc "Report duplicate blobs grouped by checksum and service"
+  task report_duplicates: :environment do
+    puts "Scanning for duplicate blobs...\n\n"
+
+    # Group blobs by checksum and service_name, find groups with duplicates
+    duplicate_groups = ActiveStorage::Blob
+      .select("checksum, service_name, COUNT(*) as blob_count")
+      .group(:checksum, :service_name)
+      .having("COUNT(*) > 1")
+      .order("blob_count DESC")
+
+    if duplicate_groups.empty?
+      puts "No duplicate blobs found!"
+      next
+    end
+
+    total_duplicates = 0
+    total_wasted_storage = 0
+
+    duplicate_groups.each do |group|
+      blobs = ActiveStorage::Blob
+        .where(checksum: group.checksum, service_name: group.service_name)
+        .order(:created_at)
+
+      keeper = blobs.first
+      duplicates = blobs[1..]
+
+      # Calculate wasted storage (size of duplicate blobs)
+      wasted_bytes = duplicates.sum(&:byte_size)
+      total_wasted_storage += wasted_bytes
+
+      puts "Checksum: #{group.checksum}"
+      puts "Service: #{group.service_name}"
+      puts "Filename: #{keeper.filename}"
+      puts "Total blobs: #{blobs.count}"
+      puts "Keeper blob ID: #{keeper.id} (#{keeper.attachments.count} attachments)"
+      puts "Duplicate blob IDs: #{duplicates.map(&:id).join(', ')}"
+      puts "Total attachments across duplicates: #{duplicates.sum { |b| b.attachments.count }}"
+      puts "Wasted storage: #{format_bytes(wasted_bytes)}"
+      puts "-" * 80
+      puts
+
+      total_duplicates += duplicates.count
+    end
+
+    puts "\nSummary:"
+    puts "Total duplicate groups: #{duplicate_groups.count}"
+    puts "Total duplicate blobs: #{total_duplicates}"
+    puts "Total wasted storage: #{format_bytes(total_wasted_storage)}"
+  end
+
+  desc "Clean up all duplicate blobs by merging them (sanity check)"
+  task cleanup_all: :environment do
+    puts "Running sanity check to find and merge duplicate blobs...\n\n"
+
+    # Run the deduplication job
+    ActiveStorageDedup::DeduplicationJob.perform_now
+
+    puts "\nCleanup complete! Check logs for details."
+  end
+
+  desc "Backfill reference_count for existing blobs"
+  task backfill_reference_count: :environment do
+    puts "Backfilling reference_count for all blobs...\n\n"
+
+    total_blobs = ActiveStorage::Blob.count
+    updated = 0
+
+    ActiveStorage::Blob.find_each.with_index do |blob, index|
+      actual_count = blob.attachments.count
+      current_count = blob.reference_count || 0
+
+      if actual_count != current_count
+        blob.update_column(:reference_count, actual_count)
+        updated += 1
+      end
+
+      if (index + 1) % 100 == 0
+        puts "Processed #{index + 1}/#{total_blobs} blobs..."
+      end
+    end
+
+    puts "\nBackfill complete!"
+    puts "Total blobs: #{total_blobs}"
+    puts "Updated: #{updated}"
+  end
+
+  # Helper method to format bytes into human-readable format
+  def format_bytes(bytes)
+    return "0 B" if bytes.zero?
+
+    units = ["B", "KB", "MB", "GB", "TB"]
+    exp = (Math.log(bytes) / Math.log(1024)).floor
+    exp = [exp, units.length - 1].min
+
+    format("%.2f %s", bytes.to_f / (1024**exp), units[exp])
+  end
+end

data/sig/active_storage_dedup.rbs

@@ -0,0 +1,4 @@
+module ActiveStorageDedup
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: active_storage_dedup
+version: !ruby/object:Gem::Version
+  version: 1.0.0.alpha
+platform: ruby
+authors:
+- coderhs
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  name: activestorage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: combustion
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: Prevents duplicate file uploads in Active Storage by reusing existing
+  blobs with matching checksums and service names
+email:
+- mailme@hsps.in
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/active_storage_dedup.rb
+- lib/active_storage_dedup/attachment_extension.rb
+- lib/active_storage_dedup/attachment_options.rb
+- lib/active_storage_dedup/blob_deduplication.rb
+- lib/active_storage_dedup/changes_extension.rb
+- lib/active_storage_dedup/configuration.rb
+- lib/active_storage_dedup/deduplication_job.rb
+- lib/active_storage_dedup/railtie.rb
+- lib/active_storage_dedup/version.rb
+- lib/generators/active_storage_dedup/install_generator.rb
+- lib/generators/active_storage_dedup/templates/README
+- lib/generators/active_storage_dedup/templates/add_active_storage_dedup.rb.erb
+- lib/generators/active_storage_dedup/templates/initializer.rb
+- lib/tasks/active_storage_dedup.rake
+- sig/active_storage_dedup.rbs
+homepage: https://github.com/coderhs/active_storage_dedup
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/coderhs/active_storage_dedup
+  source_code_uri: https://github.com/coderhs/active_storage_dedup
+  changelog_uri: https://github.com/coderhs/active_storage_dedup/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Deduplication for Active Storage uploads
+test_files: []