dis 1.2.0 → 1.3.1

data/lib/dis/storage.rb

@@ -3,7 +3,7 @@
 module Dis
   # = Dis Storage
   #
-  # This is the interface for interacting with the storage layers.
+  # Interface for interacting with the storage layers.
   #
   # All queries are scoped by object type, which will default to the table
   # name of the model. Take care to use your own scope if you interact with
@@ -17,8 +17,13 @@ module Dis
   # one writeable, non-delayed layer must exist.
   class Storage
     class << self
-      # Returns a hex digest for a given binary. Accepts files, strings
-      # and Fog models.
+      # Returns a hex digest for a given binary. Accepts File/IO objects,
+      # strings, and Fog models.
+      #
+      # @param file [File, IO, String, Fog::Model] the content to digest
+      # @yield [hash] if a block is given, yields the hex digest
+      # @yieldparam hash [String] the computed SHA1 hex digest
+      # @return [String] the SHA1 hex digest
       def file_digest(file)
         hash = case file
                when Fog::Model
@@ -32,15 +37,25 @@ module Dis
         hash
       end
 
-      # Exposes the layer set, which is an instance of
-      # <tt>Dis::Layers</tt>.
+      # Exposes the layer set.
+      #
+      # @return [Dis::Layers]
       def layers
         @layers ||= Dis::Layers.new
       end
 
       # Changes the type of an object. Kicks off a
-      # <tt>Dis::Jobs::ChangeType</tt> job if any delayed layers are defined.
+      # {Dis::Jobs::ChangeType} job if any delayed layers are defined.
+      #
+      # @param prev_type [String] the current type scope
+      # @param new_type [String] the new type scope
+      # @param key [String] the content hash
+      # @return [String] the content hash
+      # @raise [Dis::Errors::NoLayersError] if no writeable immediate
+      #   layers exist
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
       #
+      # @example
       #   Dis::Storage.change_type("old_things", "new_things", key)
       def change_type(prev_type, new_type, key)
         require_writeable_layers!
@@ -49,29 +64,37 @@ module Dis
         layers.immediate.writeable.each do |layer|
           layer.delete(prev_type, key)
         end
-        if layers.delayed.writeable.any?
-          Dis::Jobs::ChangeType.perform_later(prev_type, new_type, key)
-        end
+        enqueue_delayed_jobs(prev_type, new_type, key)
         key
       end
 
-      # Stores a file and returns a digest. Kicks off a
-      # <tt>Dis::Jobs::Store</tt> job if any delayed layers are defined.
+      # Stores a file and returns a content hash. Kicks off a
+      # {Dis::Jobs::Store} job if any delayed layers are defined.
       #
-      #   hash = Dis::Storage.store("things", File.open('foo.bin'))
+      # @param type [String] the type scope (e.g. table name)
+      # @param file [File, IO, String, Fog::Model] the content to store
+      # @return [String] the SHA1 content hash
+      # @raise [Dis::Errors::NoLayersError] if no writeable immediate
+      #   layers exist
+      #
+      # @example
+      #   hash = Dis::Storage.store("things", File.open("foo.bin"))
       #   # => "8843d7f92416211de9ebb963ff4ce28125932878"
       def store(type, file)
         require_writeable_layers!
         hash = store_immediately!(type, file)
-        if layers.delayed.writeable.any?
-          Dis::Jobs::Store.perform_later(type, hash)
-        end
+        Dis::Jobs::Store.perform_later(type, hash) if layers.delayed.writeable.any?
+        Dis::Jobs::Evict.perform_later if layers.cache?
         hash
       end
 
       # Transfers files from immediate layers to all delayed layers.
+      # Called internally by {Dis::Jobs::Store}.
       #
-      #   Dis::Storage.delayed_store("things", hash)
+      # @param type [String] the type scope
+      # @param hash [String] the content hash
+      # @return [void]
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
       def delayed_store(type, hash)
         file = get(type, hash)
         layers.delayed.writeable.each do |layer|
@@ -81,75 +104,171 @@ module Dis
 
       # Returns true if the file exists in any layer.
       #
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [Boolean]
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
+      #
+      # @example
       #   Dis::Storage.exists?("things", key) # => true
       def exists?(type, key)
         require_layers!
         layers.each do |layer|
           return true if layer.exists?(type, key)
+        rescue StandardError => e
+          report_layer_error(e, layer:, type:, key:)
         end
         false
       end
 
-      # Retrieves a file from the store.
+      # Retrieves a file from the store. If the first layer misses,
+      # the file is fetched from the next available layer and
+      # backfilled to all immediate layers.
       #
-      #   stuff = Dis::Storage.get("things", hash)
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [Fog::Model] the stored file
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
+      #   in any layer
       #
-      # If any misses are detected, it will try to fetch the file from the
-      # first available layer, then store it in all immediate layer.
-      #
-      # Returns an instance of Fog::Model.
+      # @example
+      #   file = Dis::Storage.get("things", hash)
+      #   file.body # => "file contents..."
       def get(type, key)
         require_layers!
-
         fetch_count = 0
         result = layers.inject(nil) do |res, layer|
-          res || lambda do
-            fetch_count += 1
-            layer.get(type, key)
-          end.call
-        end || raise(Dis::Errors::NotFoundError)
+          next res if res
 
-        store_immediately!(type, result) if fetch_count > 1
+          fetch_count += 1
+          fetch_from_layer(layer, type, key)
+        end || raise(Dis::Errors::NotFoundError)
+        backfill!(type, result) if fetch_count > 1
         result
       end
 
       # Returns the absolute file path from the first layer that has a
       # local copy, or nil if no layer stores files locally.
       #
-      #   Dis::Storage.file_path("things", key)
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [String, nil] the absolute file path, or nil
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
       def file_path(type, key)
         require_layers!
         layers.each do |layer|
           path = layer.file_path(type, key)
           return path if path
+        rescue StandardError => e
+          report_layer_error(e, layer:, type:, key:)
         end
         nil
       end
 
+      # Streams the contents of a file into the given file, fetching
+      # from the first layer that has it. Backfills faster layers if
+      # the content had to be fetched from further down.
+      #
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @param file [File] the destination, must respond to +path+
+      # @return [File] the file that was written to, positioned at
+      #   the start
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
+      def get_file(type, key, file)
+        require_layers!
+        fetch_count = 0
+        found = layers.detect do |layer|
+          fetch_count += 1
+          stream_from_layer(layer, type, key, file)
+        end
+        raise Dis::Errors::NotFoundError unless found
+
+        finalize_fetched_file(type, file, fetch_count)
+      end
+
       # Deletes a file from all layers. Kicks off a
-      # <tt>Dis::Jobs::Delete</tt> job if any delayed layers are defined.
-      # Returns true if the file existed in any immediate layers,
-      # or false if not.
-      #
-      #   Dis::Storage.delete("things", key)
-      #   # => true
-      #   Dis::Storage.delete("things", key)
-      #   # => false
+      # {Dis::Jobs::Delete} job if any delayed layers are defined.
+      #
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [Boolean] true if the file existed in any immediate
+      #   layer
+      # @raise [Dis::Errors::NoLayersError] if no writeable immediate
+      #   layers exist
+      #
+      # @example
+      #   Dis::Storage.delete("things", key) # => true
+      #   Dis::Storage.delete("things", key) # => false
       def delete(type, key)
         require_writeable_layers!
         deleted = false
         layers.immediate.writeable.each do |layer|
           deleted = true if layer.delete(type, key)
         end
-        if layers.delayed.writeable.any?
-          Dis::Jobs::Delete.perform_later(type, key)
-        end
+        Dis::Jobs::Delete.perform_later(type, key) if layers.delayed.writeable.any?
         deleted
       end
 
+      # Evicts cached files from all cache layers that exceed
+      # their size limit. Only evicts files that have been
+      # replicated to a non-cache writeable layer.
+      #
+      # @return [void]
+      def evict_caches
+        layers.cache.each { |layer| evict_cache(layer) }
+      end
+
+      # Returns content hashes from the model's table that exist in
+      # no non-cache layer.
+      #
+      # @param model [Class] an ActiveRecord model that includes
+      #   {Dis::Model}
+      # @yield [batch_size] called after each batch is checked
+      # @yieldparam batch_size [Integer] the number of keys in the
+      #   batch
+      # @return [Array<String>] content hashes with no backing file
+      #
+      # @example
+      #   Dis::Storage.missing_keys(Image)
+      def missing_keys(model)
+        attr = model.dis_attributes[:content_hash]
+        missing = []
+
+        model.where.not(attr => nil).in_batches(of: 200) do |batch|
+          keys = batch.pluck(attr)
+          missing.concat(uncovered_keys(keys.uniq, model.dis_type))
+          yield keys.size if block_given?
+        end
+        missing.uniq
+      end
+
+      # Returns a hash of layer => orphaned content hashes for files
+      # that exist in storage but have no matching database record.
+      #
+      # @param model [Class] an ActiveRecord model that includes
+      #   {Dis::Model}
+      # @return [Hash{Dis::Layer => Array<String>}] orphaned content
+      #   hashes per layer
+      #
+      # @example
+      #   Dis::Storage.orphaned_keys(Image)
+      def orphaned_keys(model)
+        layers.non_cache.each_with_object({}) do |layer, result|
+          orphans = layer_orphans(layer, model.dis_type, model,
+                                  model.dis_attributes[:content_hash])
+          result[layer] = orphans if orphans.any?
+        end
+      end
+
       # Deletes content from all delayed layers.
+      # Called internally by {Dis::Jobs::Delete}.
       #
-      #   Dis::Storage.delayed_delete("things", hash)
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [void]
       def delayed_delete(type, key)
         layers.delayed.writeable.each do |layer|
           layer.delete(type, key)
@@ -158,6 +277,85 @@ module Dis
 
       private
 
+      def enqueue_delayed_jobs(prev_type, new_type, key)
+        if layers.delayed.writeable.any?
+          Dis::Jobs::ChangeType.perform_later(
+            prev_type, new_type, key
+          )
+        end
+        Dis::Jobs::Evict.perform_later if layers.cache?
+      end
+
+      def uncovered_keys(keys, type)
+        remaining = keys.dup
+        layers.non_cache.each do |layer|
+          break if remaining.empty?
+
+          remaining -= layer.existing(type, remaining)
+        end
+        remaining
+      end
+
+      def layer_orphans(layer, type, model, attr)
+        stored = layer.stored_keys(type)
+        return [] if stored.empty?
+
+        referenced = model.where(attr => stored).pluck(attr)
+        stored - referenced
+      end
+
+      def evict_cache(layer)
+        return if layer.size <= layer.max_size
+
+        current_size = layer.size
+        layer.cached_files.each do |entry|
+          break if current_size <= layer.max_size
+
+          next unless replicated?(entry[:type], entry[:key])
+
+          layer.delete(entry[:type], entry[:key])
+          current_size -= entry[:size]
+        end
+      end
+
+      def replicated?(type, key)
+        layers.non_cache.writeable.any? do |l|
+          l.exists?(type, key)
+        rescue StandardError => e
+          report_layer_error(e, layer: l, type:, key:)
+          false
+        end
+      end
+
+      def finalize_fetched_file(type, file, fetch_count)
+        file.flush
+        backfill!(type, file) if fetch_count > 1
+        file.rewind
+        file
+      end
+
+      def stream_from_layer(layer, type, key, file)
+        file.truncate(0)
+        file.rewind
+        layer.stream(type, key, file)
+      rescue StandardError => e
+        report_layer_error(e, layer:, type:, key:)
+        false
+      end
+
+      def fetch_from_layer(layer, type, key)
+        layer.get(type, key)
+      rescue StandardError => e
+        report_layer_error(e, layer:, type:, key:)
+        nil
+      end
+
+      def backfill!(type, file)
+        store_immediately!(type, file)
+      rescue StandardError => e
+        report_layer_error(e, type:)
+      end
+
       def store_immediately!(type, file)
         file_digest(file) do |hash|
           layers.immediate.writeable.each do |layer|
@@ -174,6 +372,14 @@ module Dis
         raise Dis::Errors::NoLayersError unless layers.immediate.writeable.any?
       end
 
+      def report_layer_error(err, layer: nil, type: nil, key: nil)
+        Rails.error.report(
+          err, handled: true,
+               severity: :warning,
+               context: { layer: layer&.name, type:, key: }
+        )
+      end
+
       def digest
         Digest::SHA1
       end

data/lib/dis/validations/data_presence.rb

@@ -4,9 +4,17 @@ module Dis
   module Validations
     # = Dis Data Presence Validation
     #
+    # Validates that data has been assigned to a {Dis::Model} record.
+    # Empty strings are treated as missing data.
+    #
+    # @see Dis::Model::ClassMethods#validates_data_presence
     class DataPresence < ActiveModel::Validator
       # Validates that a record has data, either freshly assigned or
-      # persisted in the storage. Adds a `:blank` error on `:data`if not.
+      # persisted in the storage. Adds a +:blank+ error on +:data+
+      # if not.
+      #
+      # @param record [ActiveRecord::Base]
+      # @return [void]
       def validate(record)
         return if record.data? && record.content_hash != self.class.empty_hash
 

data/lib/dis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dis
-  VERSION = "1.2.0"
+  VERSION = "1.3.1"
 end

data/lib/dis.rb

@@ -5,7 +5,7 @@ require "digest/sha1"
 require "fog/core"
 require "fog/local"
 require "active_job"
-require "pmap"
+require "concurrent"
 require "dis/engine"
 require "dis/errors"
 require "dis/jobs"
@@ -16,5 +16,18 @@ require "dis/model"
 require "dis/storage"
 require "dis/validations"
 
+# Dis is a content-addressable store for file uploads in Rails.
+#
+# Files are stored as binary blobs keyed by the SHA1 digest of their
+# contents, enabling automatic deduplication. Storage is organized in
+# layers (see {Dis::Layer}) that can target local disk or any cloud
+# provider supported by Fog.
+#
+# Include {Dis::Model} in an ActiveRecord model to get started, and
+# configure layers via {Dis::Storage.layers}.
+#
+# @see Dis::Model
+# @see Dis::Storage
+# @see Dis::Layer
 module Dis
 end

data/lib/rails/generators/dis/install/templates/initializer.rb

@@ -9,6 +9,17 @@ Dis::Storage.layers << Dis::Layer.new(
   path: Rails.env
 )
 
+# You can also use a cache layer with bounded storage and LRU eviction:
+
+# Dis::Storage.layers << Dis::Layer.new(
+#   Fog::Storage.new(
+#     provider: "Local",
+#     local_root: Rails.root.join("tmp/dis")
+#   ),
+#   path: Rails.env,
+#   cache: 1.gigabyte
+# )
+
 # You can also add cloud storage:
 
 # require 'fog/aws/storage'

data/lib/tasks/dis.rake

@@ -1,58 +1,63 @@
 # frozen_string_literal: true
 
+require "ruby-progressbar"
+
 namespace :dis do
-  desc "Check stuff"
-  task consistency_check: :environment do
+  desc "List records with no backing file in any storage layer"
+  task missing: :environment do
     unless ENV["MODELS"]
-      puts "Usage: #{$PROGRAM_NAME} dis:consistency_check " \
-           "MODELS=Avatar,Document"
+      puts "Usage: #{$PROGRAM_NAME} dis:missing MODELS=Avatar,Document"
       exit
     end
 
     models = ENV["MODELS"].split(",").map(&:strip).map(&:constantize)
 
-    jobs = Set.new
-
     models.each do |model|
-      puts "-- #{model.name} --"
-
-      content_hash_attr = model.dis_attributes[:content_hash]
-      objects = model.pluck(content_hash_attr).uniq
-      global_missing = objects.dup
-
-      puts "Unique objects: #{objects.length}"
+      bar = ProgressBar.create(
+        title: model.name,
+        total: model.where.not(
+          model.dis_attributes[:content_hash] => nil
+        ).count,
+        format: "%t: |%B| %c/%C records"
+      )
 
-      Dis::Storage.layers.each do |layer|
-        print "Checking #{layer.name}... "
-
-        existing = layer.existing(model.dis_type, objects)
-        missing = objects - existing
-        global_missing -= existing
-        puts "#{existing.length} existing, #{missing.length} missing" +
-             (layer.readonly? ? " (read-only)" : "")
-
-        next unless layer.delayed? && !layer.readonly?
-
-        jobs += (missing - global_missing).pmap do |hash|
-          [model.dis_type, hash]
-        end.compact
+      missing = ActiveRecord::Base.logger.silence do
+        Dis::Storage.missing_keys(model) do |count|
+          bar.progress += count
+        end
       end
+      bar.finish
 
-      if global_missing.any?
-        puts "\n#{global_missing.length} objects are missing from all layers:"
-        pp global_missing
+      if missing.any?
+        puts "#{missing.length} missing:"
+        missing.each { |key| puts "  #{key}" }
+      else
+        puts "0 missing"
       end
+    end
+  end
 
-      puts
+  desc "List stored files with no matching database record"
+  task orphaned: :environment do
+    unless ENV["MODELS"]
+      puts "Usage: #{$PROGRAM_NAME} dis:orphaned MODELS=Avatar,Document"
+      exit
     end
 
-    if jobs.any?
-      print "#{jobs.length} objects can be transferred to delayed layers, " \
-            "queue now? (y/n) "
-      response = $stdin.gets.chomp
-      if /^y/i.match?(response)
-        puts "Queueing jobs..."
-        jobs.each { |type, hash| Dis::Jobs::Store.perform_later(type, hash) }
+    models = ENV["MODELS"].split(",").map(&:strip).map(&:constantize)
+
+    models.each do |model|
+      orphans = ActiveRecord::Base.logger.silence do
+        Dis::Storage.orphaned_keys(model)
+      end
+      if orphans.any?
+        orphans.each do |layer, keys|
+          puts "#{model.name} (#{layer.name}): " \
+               "#{keys.length} orphaned"
+          keys.each { |key| puts "  #{key}" }
+        end
+      else
+        puts "#{model.name}: 0 orphaned"
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dis
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Inge Jørgensen
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: fog-core
   requirement: !ruby/object:Gem::Requirement
@@ -58,33 +72,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pmap
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '7.1'
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '1.11'
 description: Dis is a Rails plugin that stores your file uploads and other binary
   blobs.
 email:
@@ -93,6 +107,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
 - README.md
 - lib/dis.rb
 - lib/dis/engine.rb
@@ -100,6 +115,7 @@ files:
 - lib/dis/jobs.rb
 - lib/dis/jobs/change_type.rb
 - lib/dis/jobs/delete.rb
+- lib/dis/jobs/evict.rb
 - lib/dis/jobs/store.rb
 - lib/dis/layer.rb
 - lib/dis/layers.rb
@@ -134,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: A file store for your Rails app
 test_files: []