dis 1.2.0 → 1.3.0

data/lib/dis/model.rb

@@ -7,7 +7,7 @@ module Dis
   # = Dis Model
   #
   # ActiveModel extension for the model holding your data. To use it,
-  # simply include the module in your model:
+  # include the module in your model:
   #
   #   class Document < ActiveRecord::Base
   #     include Dis::Model
@@ -36,24 +36,24 @@ module Dis
   #
   # == Usage
   #
-  # To save a file, simply assign to the <tt>file</tt> attribute.
+  # To save a file, assign to the <tt>file</tt> attribute.
   #
   #   document = Document.create(file: params.permit(:file))
   #
   # <tt>content_type</tt> and <tt>filename</tt> will automatically be set if
-  # the supplied object quacks like a file. <tt>content_length</tt> will always
-  # be set. <tt>content_hash</tt> won't be set until the record is being saved.
+  # the supplied object quacks like a file. <tt>content_length</tt> and
+  # <tt>content_hash</tt> will always be set.
   #
-  # If you don't care about filenames and content types and just want to store
-  # a binary blob, you can also just set the <tt>data</tt> attribute.
+  # To store a binary blob without filenames or content types, set the
+  # <tt>data</tt> attribute directly.
   #
   #   my_data = File.read('document.pdf')
   #   document.update(data: my_data)
   #
-  # The data won't be stored until the record is saved, and not unless
+  # The data won't be stored until the record is saved, and only if
   # the record is valid.
   #
-  # To retrieve your data, simply read the <tt>data</tt> attribute. The file
+  # To retrieve your data, read the <tt>data</tt> attribute. The file
   # will be lazily loaded from the store on demand and cached in memory as long
   # as the record stays in scope.
   #
@@ -73,7 +73,7 @@ module Dis
   #     validates_data_presence
   #   end
   #
-  # If you want to validate content types, size or similar, simply use standard
+  # If you want to validate content types, size or similar, use standard
   # Rails validations on the metadata attributes:
   #
   #   validates :content_type, presence: true, format: /\Aapplication\/pdf\z/
@@ -89,18 +89,26 @@ module Dis
       attribute :data, :binary
     end
 
-    # Returns the data as a binary string, or nil if no data has been set.
+    # Returns the data as a binary string, or nil if no data has
+    # been set.
+    #
+    # @return [String, nil]
     def data
       dis_data.read
     end
 
     # Returns true if data is set.
+    #
+    # @return [Boolean]
     def data?
       dis_data.any?
     end
 
-    # Assigns new data. This also sets <tt>content_length</tt>, and resets
-    # <tt>content_hash</tt> to nil.
+    # Assigns new data. This also sets +content_length+ and
+    # +content_hash+.
+    #
+    # @param raw_data [File, IO, String, nil] the content to store
+    # @return [void]
     def data=(raw_data)
       new_data = Dis::Model::Data.new(self, raw_data)
       attribute_will_change!("data") unless new_data == dis_data
@@ -113,30 +121,48 @@ module Dis
       dis_set :content_length, dis_data.content_length
     end
 
-    # Returns true if the data has been changed since the object was last saved.
+    # Returns true if the data has been changed since the object
+    # was last saved.
+    #
+    # @return [Boolean]
     def data_changed?
       changes.include?("data")
     end
 
+    # Returns true if the record has been persisted and its data
+    # has not been changed since the last save.
+    #
+    # @return [Boolean]
     def dis_stored?
       !(new_record? || data_changed?)
     end
 
-    # Assigns new data from an uploaded file. In addition to the actions
-    # performed by <tt>data=</tt>, this will set <tt>content_type</tt> and
-    # <tt>filename</tt>.
+    # Assigns new data from an uploaded file. In addition to the
+    # actions performed by {#data=}, this will set +content_type+
+    # and +filename+.
+    #
+    # @param file [ActionDispatch::Http::UploadedFile,
+    #   Rack::Test::UploadedFile] an uploaded file that responds to
+    #   +content_type+ and +original_filename+
+    # @return [void]
     def file=(file)
       self.data = file
       dis_set :content_type, file.content_type
       dis_set :filename, file.original_filename
     end
 
-    # Returns a file path to the data, preferring local storage paths.
+    # Returns a file path to the data, preferring local storage
+    # paths. Falls back to a tempfile path if no local layer has
+    # the file.
+    #
+    # @return [String]
     def data_file_path
       dis_data.file_path
     end
 
     # Returns the data as a temporary file.
+    #
+    # @return [Tempfile]
     delegate :tempfile, to: :dis_data
 
     private

data/lib/dis/storage.rb

@@ -3,7 +3,7 @@
 module Dis
   # = Dis Storage
   #
-  # This is the interface for interacting with the storage layers.
+  # Interface for interacting with the storage layers.
   #
   # All queries are scoped by object type, which will default to the table
   # name of the model. Take care to use your own scope if you interact with
@@ -17,8 +17,13 @@ module Dis
   # one writeable, non-delayed layer must exist.
   class Storage
     class << self
-      # Returns a hex digest for a given binary. Accepts files, strings
-      # and Fog models.
+      # Returns a hex digest for a given binary. Accepts File/IO objects,
+      # strings, and Fog models.
+      #
+      # @param file [File, IO, String, Fog::Model] the content to digest
+      # @yield [hash] if a block is given, yields the hex digest
+      # @yieldparam hash [String] the computed SHA1 hex digest
+      # @return [String] the SHA1 hex digest
       def file_digest(file)
         hash = case file
                when Fog::Model
@@ -32,15 +37,25 @@ module Dis
         hash
       end
 
-      # Exposes the layer set, which is an instance of
-      # <tt>Dis::Layers</tt>.
+      # Exposes the layer set.
+      #
+      # @return [Dis::Layers]
       def layers
         @layers ||= Dis::Layers.new
       end
 
       # Changes the type of an object. Kicks off a
-      # <tt>Dis::Jobs::ChangeType</tt> job if any delayed layers are defined.
+      # {Dis::Jobs::ChangeType} job if any delayed layers are defined.
       #
+      # @param prev_type [String] the current type scope
+      # @param new_type [String] the new type scope
+      # @param key [String] the content hash
+      # @return [String] the content hash
+      # @raise [Dis::Errors::NoLayersError] if no writeable immediate
+      #   layers exist
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
+      #
+      # @example
       #   Dis::Storage.change_type("old_things", "new_things", key)
       def change_type(prev_type, new_type, key)
         require_writeable_layers!
@@ -49,29 +64,37 @@ module Dis
         layers.immediate.writeable.each do |layer|
           layer.delete(prev_type, key)
         end
-        if layers.delayed.writeable.any?
-          Dis::Jobs::ChangeType.perform_later(prev_type, new_type, key)
-        end
+        enqueue_delayed_jobs(prev_type, new_type, key)
         key
       end
 
-      # Stores a file and returns a digest. Kicks off a
-      # <tt>Dis::Jobs::Store</tt> job if any delayed layers are defined.
+      # Stores a file and returns a content hash. Kicks off a
+      # {Dis::Jobs::Store} job if any delayed layers are defined.
+      #
+      # @param type [String] the type scope (e.g. table name)
+      # @param file [File, IO, String, Fog::Model] the content to store
+      # @return [String] the SHA1 content hash
+      # @raise [Dis::Errors::NoLayersError] if no writeable immediate
+      #   layers exist
       #
-      #   hash = Dis::Storage.store("things", File.open('foo.bin'))
+      # @example
+      #   hash = Dis::Storage.store("things", File.open("foo.bin"))
       #   # => "8843d7f92416211de9ebb963ff4ce28125932878"
       def store(type, file)
         require_writeable_layers!
         hash = store_immediately!(type, file)
-        if layers.delayed.writeable.any?
-          Dis::Jobs::Store.perform_later(type, hash)
-        end
+        Dis::Jobs::Store.perform_later(type, hash) if layers.delayed.writeable.any?
+        Dis::Jobs::Evict.perform_later if layers.cache?
         hash
       end
 
       # Transfers files from immediate layers to all delayed layers.
+      # Called internally by {Dis::Jobs::Store}.
       #
-      #   Dis::Storage.delayed_store("things", hash)
+      # @param type [String] the type scope
+      # @param hash [String] the content hash
+      # @return [void]
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
       def delayed_store(type, hash)
         file = get(type, hash)
         layers.delayed.writeable.each do |layer|
@@ -81,75 +104,148 @@ module Dis
 
       # Returns true if the file exists in any layer.
       #
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [Boolean]
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
+      #
+      # @example
       #   Dis::Storage.exists?("things", key) # => true
       def exists?(type, key)
         require_layers!
         layers.each do |layer|
           return true if layer.exists?(type, key)
+        rescue StandardError => e
+          report_layer_error(e, layer:, type:, key:)
         end
         false
       end
 
-      # Retrieves a file from the store.
-      #
-      #   stuff = Dis::Storage.get("things", hash)
+      # Retrieves a file from the store. If the first layer misses,
+      # the file is fetched from the next available layer and
+      # backfilled to all immediate layers.
       #
-      # If any misses are detected, it will try to fetch the file from the
-      # first available layer, then store it in all immediate layer.
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [Fog::Model] the stored file
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
+      #   in any layer
       #
-      # Returns an instance of Fog::Model.
+      # @example
+      #   file = Dis::Storage.get("things", hash)
+      #   file.body # => "file contents..."
       def get(type, key)
         require_layers!
-
         fetch_count = 0
         result = layers.inject(nil) do |res, layer|
-          res || lambda do
-            fetch_count += 1
-            layer.get(type, key)
-          end.call
-        end || raise(Dis::Errors::NotFoundError)
+          next res if res
 
-        store_immediately!(type, result) if fetch_count > 1
+          fetch_count += 1
+          fetch_from_layer(layer, type, key)
+        end || raise(Dis::Errors::NotFoundError)
+        backfill!(type, result) if fetch_count > 1
         result
       end
 
       # Returns the absolute file path from the first layer that has a
       # local copy, or nil if no layer stores files locally.
       #
-      #   Dis::Storage.file_path("things", key)
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [String, nil] the absolute file path, or nil
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
       def file_path(type, key)
         require_layers!
         layers.each do |layer|
           path = layer.file_path(type, key)
           return path if path
+        rescue StandardError => e
+          report_layer_error(e, layer:, type:, key:)
         end
         nil
       end
 
       # Deletes a file from all layers. Kicks off a
-      # <tt>Dis::Jobs::Delete</tt> job if any delayed layers are defined.
-      # Returns true if the file existed in any immediate layers,
-      # or false if not.
-      #
-      #   Dis::Storage.delete("things", key)
-      #   # => true
-      #   Dis::Storage.delete("things", key)
-      #   # => false
+      # {Dis::Jobs::Delete} job if any delayed layers are defined.
+      #
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [Boolean] true if the file existed in any immediate
+      #   layer
+      # @raise [Dis::Errors::NoLayersError] if no writeable immediate
+      #   layers exist
+      #
+      # @example
+      #   Dis::Storage.delete("things", key) # => true
+      #   Dis::Storage.delete("things", key) # => false
       def delete(type, key)
         require_writeable_layers!
         deleted = false
         layers.immediate.writeable.each do |layer|
           deleted = true if layer.delete(type, key)
         end
-        if layers.delayed.writeable.any?
-          Dis::Jobs::Delete.perform_later(type, key)
-        end
+        Dis::Jobs::Delete.perform_later(type, key) if layers.delayed.writeable.any?
         deleted
       end
 
+      # Evicts cached files from all cache layers that exceed
+      # their size limit. Only evicts files that have been
+      # replicated to a non-cache writeable layer.
+      #
+      # @return [void]
+      def evict_caches
+        layers.cache.each { |layer| evict_cache(layer) }
+      end
+
+      # Returns content hashes from the model's table that exist in
+      # no non-cache layer.
+      #
+      # @param model [Class] an ActiveRecord model that includes
+      #   {Dis::Model}
+      # @yield [batch_size] called after each batch is checked
+      # @yieldparam batch_size [Integer] the number of keys in the
+      #   batch
+      # @return [Array<String>] content hashes with no backing file
+      #
+      # @example
+      #   Dis::Storage.missing_keys(Image)
+      def missing_keys(model)
+        attr = model.dis_attributes[:content_hash]
+        missing = []
+
+        model.where.not(attr => nil).in_batches(of: 200) do |batch|
+          keys = batch.pluck(attr)
+          missing.concat(uncovered_keys(keys.uniq, model.dis_type))
+          yield keys.size if block_given?
+        end
+        missing.uniq
+      end
+
+      # Returns a hash of layer => orphaned content hashes for files
+      # that exist in storage but have no matching database record.
+      #
+      # @param model [Class] an ActiveRecord model that includes
+      #   {Dis::Model}
+      # @return [Hash{Dis::Layer => Array<String>}] orphaned content
+      #   hashes per layer
+      #
+      # @example
+      #   Dis::Storage.orphaned_keys(Image)
+      def orphaned_keys(model)
+        layers.non_cache.each_with_object({}) do |layer, result|
+          orphans = layer_orphans(layer, model.dis_type, model,
+                                  model.dis_attributes[:content_hash])
+          result[layer] = orphans if orphans.any?
+        end
+      end
+
       # Deletes content from all delayed layers.
+      # Called internally by {Dis::Jobs::Delete}.
       #
-      #   Dis::Storage.delayed_delete("things", hash)
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @return [void]
       def delayed_delete(type, key)
         layers.delayed.writeable.each do |layer|
           layer.delete(type, key)
@@ -158,6 +254,69 @@ module Dis
 
       private
 
+      def enqueue_delayed_jobs(prev_type, new_type, key)
+        if layers.delayed.writeable.any?
+          Dis::Jobs::ChangeType.perform_later(
+            prev_type, new_type, key
+          )
+        end
+        Dis::Jobs::Evict.perform_later if layers.cache?
+      end
+
+      def uncovered_keys(keys, type)
+        remaining = keys.dup
+        layers.non_cache.each do |layer|
+          break if remaining.empty?
+
+          remaining -= layer.existing(type, remaining)
+        end
+        remaining
+      end
+
+      def layer_orphans(layer, type, model, attr)
+        stored = layer.stored_keys(type)
+        return [] if stored.empty?
+
+        referenced = model.where(attr => stored).pluck(attr)
+        stored - referenced
+      end
+
+      def evict_cache(layer)
+        return if layer.size <= layer.max_size
+
+        current_size = layer.size
+        layer.cached_files.each do |entry|
+          break if current_size <= layer.max_size
+
+          next unless replicated?(entry[:type], entry[:key])
+
+          layer.delete(entry[:type], entry[:key])
+          current_size -= entry[:size]
+        end
+      end
+
+      def replicated?(type, key)
+        layers.non_cache.writeable.any? do |l|
+          l.exists?(type, key)
+        rescue StandardError => e
+          report_layer_error(e, layer: l, type:, key:)
+          false
+        end
+      end
+
+      def fetch_from_layer(layer, type, key)
+        layer.get(type, key)
+      rescue StandardError => e
+        report_layer_error(e, layer:, type:, key:)
+        nil
+      end
+
+      def backfill!(type, file)
+        store_immediately!(type, file)
+      rescue StandardError => e
+        report_layer_error(e, type:)
+      end
+
       def store_immediately!(type, file)
         file_digest(file) do |hash|
           layers.immediate.writeable.each do |layer|
@@ -174,6 +333,14 @@ module Dis
         raise Dis::Errors::NoLayersError unless layers.immediate.writeable.any?
       end
 
+      def report_layer_error(err, layer: nil, type: nil, key: nil)
+        Rails.error.report(
+          err, handled: true,
+               severity: :warning,
+               context: { layer: layer&.name, type:, key: }
+        )
+      end
+
       def digest
         Digest::SHA1
       end

data/lib/dis/validations/data_presence.rb

@@ -4,9 +4,17 @@ module Dis
   module Validations
     # = Dis Data Presence Validation
     #
+    # Validates that data has been assigned to a {Dis::Model} record.
+    # Empty strings are treated as missing data.
+    #
+    # @see Dis::Model::ClassMethods#validates_data_presence
     class DataPresence < ActiveModel::Validator
       # Validates that a record has data, either freshly assigned or
-      # persisted in the storage. Adds a `:blank` error on `:data`if not.
+      # persisted in the storage. Adds a +:blank+ error on +:data+
+      # if not.
+      #
+      # @param record [ActiveRecord::Base]
+      # @return [void]
       def validate(record)
         return if record.data? && record.content_hash != self.class.empty_hash
 

data/lib/dis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dis
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/lib/dis.rb

@@ -5,7 +5,7 @@ require "digest/sha1"
 require "fog/core"
 require "fog/local"
 require "active_job"
-require "pmap"
+require "concurrent"
 require "dis/engine"
 require "dis/errors"
 require "dis/jobs"
@@ -16,5 +16,18 @@ require "dis/model"
 require "dis/storage"
 require "dis/validations"
 
+# Dis is a content-addressable store for file uploads in Rails.
+#
+# Files are stored as binary blobs keyed by the SHA1 digest of their
+# contents, enabling automatic deduplication. Storage is organized in
+# layers (see {Dis::Layer}) that can target local disk or any cloud
+# provider supported by Fog.
+#
+# Include {Dis::Model} in an ActiveRecord model to get started, and
+# configure layers via {Dis::Storage.layers}.
+#
+# @see Dis::Model
+# @see Dis::Storage
+# @see Dis::Layer
 module Dis
 end

data/lib/rails/generators/dis/install/templates/initializer.rb

@@ -9,6 +9,17 @@ Dis::Storage.layers << Dis::Layer.new(
   path: Rails.env
 )
 
+# You can also use a cache layer with bounded storage and LRU eviction:
+
+# Dis::Storage.layers << Dis::Layer.new(
+#   Fog::Storage.new(
+#     provider: "Local",
+#     local_root: Rails.root.join("tmp/dis")
+#   ),
+#   path: Rails.env,
+#   cache: 1.gigabyte
+# )
+
 # You can also add cloud storage:
 
 # require 'fog/aws/storage'

data/lib/tasks/dis.rake

@@ -1,58 +1,63 @@
 # frozen_string_literal: true
 
+require "ruby-progressbar"
+
 namespace :dis do
-  desc "Check stuff"
-  task consistency_check: :environment do
+  desc "List records with no backing file in any storage layer"
+  task missing: :environment do
     unless ENV["MODELS"]
-      puts "Usage: #{$PROGRAM_NAME} dis:consistency_check " \
-           "MODELS=Avatar,Document"
+      puts "Usage: #{$PROGRAM_NAME} dis:missing MODELS=Avatar,Document"
       exit
     end
 
     models = ENV["MODELS"].split(",").map(&:strip).map(&:constantize)
 
-    jobs = Set.new
-
     models.each do |model|
-      puts "-- #{model.name} --"
-
-      content_hash_attr = model.dis_attributes[:content_hash]
-      objects = model.pluck(content_hash_attr).uniq
-      global_missing = objects.dup
-
-      puts "Unique objects: #{objects.length}"
+      bar = ProgressBar.create(
+        title: model.name,
+        total: model.where.not(
+          model.dis_attributes[:content_hash] => nil
+        ).count,
+        format: "%t: |%B| %c/%C records"
+      )
 
-      Dis::Storage.layers.each do |layer|
-        print "Checking #{layer.name}... "
-
-        existing = layer.existing(model.dis_type, objects)
-        missing = objects - existing
-        global_missing -= existing
-        puts "#{existing.length} existing, #{missing.length} missing" +
-             (layer.readonly? ? " (read-only)" : "")
-
-        next unless layer.delayed? && !layer.readonly?
-
-        jobs += (missing - global_missing).pmap do |hash|
-          [model.dis_type, hash]
-        end.compact
+      missing = ActiveRecord::Base.logger.silence do
+        Dis::Storage.missing_keys(model) do |count|
+          bar.progress += count
+        end
       end
+      bar.finish
 
-      if global_missing.any?
-        puts "\n#{global_missing.length} objects are missing from all layers:"
-        pp global_missing
+      if missing.any?
+        puts "#{missing.length} missing:"
+        missing.each { |key| puts "  #{key}" }
+      else
+        puts "0 missing"
       end
+    end
+  end
 
-      puts
+  desc "List stored files with no matching database record"
+  task orphaned: :environment do
+    unless ENV["MODELS"]
+      puts "Usage: #{$PROGRAM_NAME} dis:orphaned MODELS=Avatar,Document"
+      exit
     end
 
-    if jobs.any?
-      print "#{jobs.length} objects can be transferred to delayed layers, " \
-            "queue now? (y/n) "
-      response = $stdin.gets.chomp
-      if /^y/i.match?(response)
-        puts "Queueing jobs..."
-        jobs.each { |type, hash| Dis::Jobs::Store.perform_later(type, hash) }
+    models = ENV["MODELS"].split(",").map(&:strip).map(&:constantize)
+
+    models.each do |model|
+      orphans = ActiveRecord::Base.logger.silence do
+        Dis::Storage.orphaned_keys(model)
+      end
+      if orphans.any?
+        orphans.each do |layer, keys|
+          puts "#{model.name} (#{layer.name}): " \
+               "#{keys.length} orphaned"
+          keys.each { |key| puts "  #{key}" }
+        end
+      else
+        puts "#{model.name}: 0 orphaned"
       end
     end
   end