dis 1.1.21 → 1.3.0

data/lib/dis/layer.rb

@@ -3,52 +3,47 @@
 module Dis
   # = Dis Layer
   #
-  # Represents a layer of storage. It's a wrapper around
-  # <tt>Fog::Storage</tt>, any provider supported by Fog should be usable.
-  #
-  # ==== Options
-  #
-  # * <tt>:delayed</tt> - Delayed layers will be processed outside of
-  #   the request cycle by ActiveJob.
-  # * <tt>:readonly</tt> - Readonly layers can only be read from,
-  #   not written to.
-  # * <tt>:public</tt> - Objects stored in public layers will have the
-  #   public readable flag set if supported by the storage provider.
-  # * <tt>:path</tt> - Directory name to use for the store. For Amazon S3,
-  #   this will be the name of the bucket.
-  #
-  # ==== Examples
-  #
-  # This creates a local storage layer. It's a good idea to have a local layer
-  # first, this provides you with a cache on disk that will be faster than
-  # reading from the cloud.
+  # Represents a layer of storage. Wraps a +Fog::Storage+ connection;
+  # any provider supported by Fog should be usable.
   #
+  # @example Local storage layer
   #   Dis::Layer.new(
-  #     Fog::Storage.new({
-  #       provider: 'Local',
-  #       local_root: Rails.root.join('db', 'dis')
-  #     }),
+  #     Fog::Storage.new(
+  #       provider: "Local",
+  #       local_root: Rails.root.join("db/dis")
+  #     ),
   #     path: Rails.env
   #   )
   #
-  # This creates a delayed layer on Amazon S3. ActiveJob will kick in and
-  # and transfer content from one of the immediate layers later at it's
-  # leisure.
-  #
+  # @example Delayed layer on Amazon S3
   #   Dis::Layer.new(
-  #     Fog::Storage.new({
-  #       provider:              'AWS',
-  #       aws_access_key_id:     YOUR_AWS_ACCESS_KEY_ID,
+  #     Fog::Storage.new(
+  #       provider: "AWS",
+  #       aws_access_key_id: YOUR_AWS_ACCESS_KEY_ID,
   #       aws_secret_access_key: YOUR_AWS_SECRET_ACCESS_KEY
-  #     }),
+  #     ),
   #     path: "my_bucket",
   #     delayed: true
   #   )
   class Layer
     include Dis::Logging
 
+    # @return [Fog::Storage] the underlying Fog connection
     attr_reader :connection
 
+    # @param connection [Fog::Storage] a Fog storage connection
+    # @param options [Hash] layer configuration options
+    # @option options [Boolean] :delayed (false) process writes
+    #   asynchronously via ActiveJob
+    # @option options [Boolean] :readonly (false) only allow reads
+    # @option options [Boolean] :public (false) set the public
+    #   readable flag on stored objects (provider-dependent)
+    # @option options [String] :path (nil) directory or bucket name
+    # @option options [Integer, false] :cache (false) enable bounded
+    #   cache with this soft size limit in bytes. Cannot be combined
+    #   with +:delayed+ or +:readonly+
+    # @raise [ArgumentError] if +:cache+ is combined with +:delayed+
+    #   or +:readonly+
     def initialize(connection, options = {})
       options     = default_options.merge(options)
       @connection = connection
@@ -56,44 +51,102 @@ module Dis
       @readonly   = options[:readonly]
       @public     = options[:public]
       @path       = options[:path]
+      @cache      = options[:cache]
+      validate_cache_options!
     end
 
     # Returns true if the layer is a delayed layer.
+    #
+    # @return [Boolean]
     def delayed?
       @delayed
     end
 
     # Returns true if the layer isn't a delayed layer.
+    #
+    # @return [Boolean]
     def immediate?
       !delayed?
     end
 
     # Returns true if the layer is public.
+    #
+    # @return [Boolean]
     def public?
       @public
     end
 
     # Returns true if the layer is read only.
+    #
+    # @return [Boolean]
     def readonly?
       @readonly
     end
 
     # Returns true if the layer is writeable.
+    #
+    # @return [Boolean]
     def writeable?
       !readonly?
     end
 
-    # Stores a file.
+    # Returns true if the layer is a cache layer.
     #
-    #   key = Digest::SHA1.file(file.path).hexdigest
-    #   layer.store("documents", key, path)
+    # @return [Boolean]
+    def cache?
+      !!@cache
+    end
+
+    # Returns the cache size limit in bytes, or nil if not a cache.
     #
-    # The key must be a hex digest of the file content. If an object with the
-    # supplied hash already exists, no action will be performed. In other
-    # words, no data will be overwritten if a hash collision occurs.
+    # @return [Integer, nil]
+    def max_size
+      @cache if cache?
+    end
+
+    # Returns the total size in bytes of all files stored locally.
+    # Returns 0 for non-local providers.
     #
-    # Returns an instance of Fog::Model, or raises an error if the layer
-    # is readonly.
+    # @return [Integer]
+    def size
+      return 0 unless connection.respond_to?(:local_root)
+
+      root = local_root_path
+      return 0 unless root.exist?
+
+      root.glob("**/*").sum { |f| f.file? ? f.size : 0 }
+    end
+
+    # Returns cached file entries sorted by mtime ascending
+    # (oldest first).
+    #
+    # @return [Array<Hash>] each entry has keys +:path+
+    #   (Pathname), +:type+ (String), +:key+ (String), +:mtime+
+    #   (Time), +:size+ (Integer)
+    def cached_files
+      return [] unless connection.respond_to?(:local_root)
+
+      root = local_root_path
+      return [] unless root.exist?
+
+      entries = root.glob("**/*").select(&:file?)
+      entries.filter_map { |f| cached_file_entry(f, root) }
+             .sort_by { |e| e[:mtime] }
+    end
+
+    # Stores a file. The key must be a hex digest of the file
+    # content. If an object with the supplied hash already exists,
+    # no action will be performed.
+    #
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @param file [File, IO, String, Fog::Model] the content
+    # @return [Fog::Model] the stored file
+    # @raise [Dis::Errors::ReadOnlyError] if the layer is readonly
+    #
+    # @example
+    #   key = Digest::SHA1.file(file.path).hexdigest
+    #   layer.store("documents", key, file)
     def store(type, key, file)
       raise Dis::Errors::ReadOnlyError if readonly?
 
@@ -104,18 +157,42 @@ module Dis
 
     # Returns all the given keys that exist in the layer.
     #
-    #    layer.existing("documents", keys)
+    # @param type [String] the type scope
+    # @param keys [Array<String>] content hashes to check
+    # @return [Array<String>] the subset of keys that exist
     def existing(type, keys)
       return [] if keys.empty?
 
-      list = directory(type, keys.first).files.map(&:key)
+      futures = keys.map do |key|
+        Concurrent::Promises.future { key if exists?(type, key) }
+      end
+      futures.filter_map(&:value!)
+    end
+
+    # Returns all content hashes stored under the given type.
+    #
+    # @param type [String] the type scope
+    # @return [Array<String>] content hashes
+    def stored_keys(type)
+      dir = connection.directories.get(path || "")
+      return [] unless dir
 
-      keys.select { |key| list.include?(key_component(type, key)) }
+      prefix = "#{type}/"
+      dir.files.filter_map do |file|
+        next unless file.key.start_with?(prefix)
+
+        parts = file.key.delete_prefix(prefix).split("/")
+        next unless parts.length == 2
+
+        "#{parts[0]}#{parts[1]}"
+      end
     end
 
-    # Returns true if a object with the given key exists.
+    # Returns true if an object with the given key exists.
     #
-    #    layer.exists?("documents", key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [Boolean]
     def exists?(type, key)
       if directory(type, key)&.files&.head(key_component(type, key))
         true
@@ -126,22 +203,44 @@ module Dis
 
     # Retrieves a file from the store.
     #
-    #    layer.get("documents", key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [Fog::Model, nil] the file, or nil if not found
     def get(type, key)
       dir = directory(type, key)
       return unless dir
 
-      debug_log("Get #{type}/#{key} from #{name}") do
+      result = debug_log("Get #{type}/#{key} from #{name}") do
         dir.files.get(key_component(type, key))
       end
+      touch_file(type, key) if result && cache?
+      result
     end
 
-    # Deletes a file from the store.
+    # Returns the absolute file path for a locally stored file, or
+    # nil if the provider is not local or the file does not exist.
     #
-    #   layer.delete("documents", key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [String, nil]
+    def file_path(type, key)
+      return unless connection.respond_to?(:local_root)
+      return unless exists?(type, key)
+
+      File.join(
+        connection.local_root,
+        directory_component(type, key),
+        key_component(type, key)
+      )
+    end
+
+    # Deletes a file from the store.
     #
-    # Returns true if the file was deleted, or false if it could not be found.
-    # Raises an error if the layer is readonly.
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [Boolean] true if the file was deleted, false if not
+    #   found
+    # @raise [Dis::Errors::ReadOnlyError] if the layer is readonly
     def delete(type, key)
       raise Dis::Errors::ReadOnlyError if readonly?
 
@@ -152,6 +251,9 @@ module Dis
 
     # Returns a name for the layer.
     #
+    # @return [String]
+    #
+    # @example
     #   layer.name # => "Fog::Storage::Local::Real/development"
     def name
       "#{connection.class.name}/#{path}"
@@ -160,7 +262,39 @@ module Dis
     private
 
     def default_options
-      { delayed: false, readonly: false, public: false, path: nil }
+      { delayed: false, readonly: false, public: false,
+        path: nil, cache: false }
+    end
+
+    def validate_cache_options!
+      return unless cache?
+
+      if delayed?
+        raise ArgumentError,
+              "cache layers cannot be delayed"
+      end
+      return unless readonly?
+
+      raise ArgumentError,
+            "cache layers cannot be readonly"
+    end
+
+    def local_root_path
+      root = Pathname.new(connection.local_root)
+      path ? root.join(path) : root
+    end
+
+    def cached_file_entry(file, root)
+      parts = file.relative_path_from(root).to_s.split("/")
+      return unless parts.length == 3
+
+      { path: file, type: parts[0], key: parts[1] + parts[2],
+        mtime: file.mtime, size: file.size }
+    end
+
+    def touch_file(type, key)
+      fp = file_path(type, key)
+      FileUtils.touch(fp) if fp
     end
 
     def directory_component(_type, _key)
@@ -168,7 +302,7 @@ module Dis
     end
 
     def key_component(type, key)
-      [type, key[0...2], key[2..key.length]].compact.join("/")
+      [type, key[0...2], key[2..]].compact.join("/")
     end
 
     def delete!(type, key)

data/lib/dis/layers.rb

@@ -3,10 +3,15 @@
 module Dis
   # = Dis Layers
   #
-  # Represents a collection of layers.
+  # Represents a filterable collection of {Dis::Layer} instances.
+  # Supports chained filtering by layer properties.
+  #
+  # @example
+  #   Dis::Storage.layers.delayed.writeable.each { |l| ... }
   class Layers
     include Enumerable
 
+    # @param layers [Array<Dis::Layer>] initial layers
     def initialize(layers = [])
       @layers = layers
     end
@@ -15,6 +20,8 @@ module Dis
     delegate :<<, to: :@layers
 
     # Clears all layers from the collection.
+    #
+    # @return [void]
     def clear!
       @layers = []
     end
@@ -25,43 +32,87 @@ module Dis
     end
 
     # Returns a new instance containing only the delayed layers.
+    #
+    # @return [Dis::Layers]
     def delayed
       self.class.new select(&:delayed?)
     end
 
     # Returns true if one or more delayed layers exist.
+    #
+    # @return [Boolean]
     def delayed?
-      delayed.any?
+      any?(&:delayed?)
     end
 
     # Returns a new instance containing only the immediate layers.
+    #
+    # @return [Dis::Layers]
     def immediate
       self.class.new select(&:immediate?)
     end
 
     # Returns true if one or more immediate layers exist.
+    #
+    # @return [Boolean]
     def immediate?
-      immediate.any?
+      any?(&:immediate?)
     end
 
     # Returns a new instance containing only the readonly layers.
+    #
+    # @return [Dis::Layers]
     def readonly
       self.class.new select(&:readonly?)
     end
 
     # Returns true if one or more readonly layers exist.
+    #
+    # @return [Boolean]
     def readonly?
-      readonly.any?
+      any?(&:readonly?)
     end
 
     # Returns a new instance containing only the writeable layers.
+    #
+    # @return [Dis::Layers]
     def writeable
       self.class.new select(&:writeable?)
     end
 
     # Returns true if one or more writeable layers exist.
+    #
+    # @return [Boolean]
     def writeable?
-      writeable.any?
+      any?(&:writeable?)
+    end
+
+    # Returns a new instance containing only the cache layers.
+    #
+    # @return [Dis::Layers]
+    def cache
+      self.class.new select(&:cache?)
+    end
+
+    # Returns true if one or more cache layers exist.
+    #
+    # @return [Boolean]
+    def cache?
+      any?(&:cache?)
+    end
+
+    # Returns a new instance containing only the non-cache layers.
+    #
+    # @return [Dis::Layers]
+    def non_cache
+      self.class.new reject(&:cache?)
+    end
+
+    # Returns true if one or more non-cache layers exist.
+    #
+    # @return [Boolean]
+    def non_cache?
+      any? { |l| !l.cache? }
     end
   end
 end

data/lib/dis/model/class_methods.rb

@@ -4,13 +4,21 @@ module Dis
   module Model
     module ClassMethods
       # Returns the mapping of attribute names.
+      #
+      # @return [Hash{Symbol => Symbol}]
       def dis_attributes
         default_dis_attributes.merge(@dis_attributes ||= {})
       end
 
-      # Sets the current mapping of attribute names. Use this if you want to
-      # override the attributes and database columns that Dis will use.
+      # Sets the current mapping of attribute names. Use this if you
+      # want to override the attributes and database columns that
+      # Dis will use. Valid keys: +:content_hash+, +:content_type+,
+      # +:content_length+, +:filename+.
+      #
+      # @param new_attributes [Hash{Symbol => Symbol}] attribute
+      #   name overrides
       #
+      # @example
       #   class Document < ActiveRecord::Base
       #     include Dis::Model
       #     self.dis_attributes = { filename: :my_custom_filename }
@@ -20,8 +28,11 @@ module Dis
       end
 
       # Returns the storage type name, which Dis will use for
-      # directory scoping. Defaults to the name of the database table.
+      # directory scoping. Defaults to the table name.
+      #
+      # @return [String]
       #
+      # @example
       #   class Document < ActiveRecord::Base; end
       #   Document.dis_type # => "documents"
       def dis_type
@@ -30,17 +41,23 @@ module Dis
 
       # Sets the storage type name.
       #
-      # Take care not to set the same name for multiple models, this will
-      # cause data loss when a record is destroyed.
+      # Take care not to set the same name for multiple models,
+      # this will cause data loss when a record is destroyed.
+      #
+      # @param new_type [String] the new type scope
+      # @return [void]
       def dis_type=(new_type)
         @dis_type = new_type
       end
 
       # Adds a presence validation on the +data+ attribute.
       #
-      # This is better than using `validates :data, presence: true`, since
-      # that would cause it to load the data from storage on each save.
+      # This is preferred over +validates :data, presence: true+,
+      # which would load the data from storage on each save.
+      #
+      # @return [void]
       #
+      # @example
       #   class Document < ActiveRecord::Base
       #     include Dis::Model
       #     validates_data_presence

data/lib/dis/model/data.rb

@@ -7,39 +7,56 @@ module Dis
     # Facilitates communication between the model and the storage,
     # and holds any newly assigned data before the record is saved.
     class Data
+      # @param record [ActiveRecord::Base] the model instance
+      # @param raw [File, IO, String, nil] newly assigned data
       def initialize(record, raw = nil)
         @record = record
         @raw = raw
       end
 
       # Returns true if two Data objects represent the same data.
+      #
+      # @param other [Dis::Model::Data, #read, Object] the object to
+      #   compare
+      # @return [Boolean]
       def ==(other)
-        return false unless other.respond_to?(:read)
-
-        # TODO: This can be made faster by
-        # comparing hashes for stored objects.
-        other.read == read
+        if !raw? && other.is_a?(self.class) && !other.changed?
+          content_hash == other.content_hash
+        elsif other.respond_to?(:read)
+          other.read == read
+        else
+          false
+        end
       end
 
       # Returns true if data exists either in memory or in storage.
+      #
+      # @return [Boolean]
       def any?
         raw? || stored?
       end
 
       # Returns the data as a binary string.
+      #
+      # @return [String, nil]
       def read
         @read ||= read_from(closest)
       end
 
-      # Will be true if data has been explicitely set.
+      # Will be true if data has been explicitly set.
       #
+      # @return [Boolean]
+      #
+      # @example
       #   Dis::Model::Data.new(record).changed? # => false
-      #   Dis::Model::Data.new(record, new_file).changed? # => true
+      #   Dis::Model::Data.new(record, file).changed? # => true
       def changed?
         raw?
       end
 
-      # Returns the length of the data.
+      # Returns the length of the data in bytes.
+      #
+      # @return [Integer]
       def content_length
         if raw? && raw.respond_to?(:length)
           raw.length
@@ -48,9 +65,13 @@ module Dis
         end
       end
 
-      # Expires a data object from the storage if it's no longer being used
-      # by existing records. This is triggered from callbacks on the record
-      # whenever they are changed or destroyed.
+      # Expires a data object from the storage if it's no longer
+      # being used by existing records. This is triggered from
+      # callbacks on the record whenever they are changed or
+      # destroyed.
+      #
+      # @param hash [String] the content hash to expire
+      # @return [void]
       def expire(hash)
         return if hash.blank?
 
@@ -61,14 +82,41 @@ module Dis
         end
       end
 
-      # Stores the data. Returns a hash of the content for reference.
+      # Stores the data and returns the content hash.
+      #
+      # @return [String] the SHA1 content hash
+      # @raise [Dis::Errors::NoDataError] if no data has been
+      #   assigned
       def store!
         raise Dis::Errors::NoDataError unless raw?
 
         Dis::Storage.store(storage_type, raw)
       end
 
+      # Clears cached data and tempfiles, allowing them to be
+      # garbage collected. Subsequent calls to +read+ or +tempfile+
+      # will re-fetch from storage.
+      #
+      # @return [void]
+      def reset_read_cache!
+        @read = nil
+        return unless @tempfile
+
+        @tempfile.close!
+        @tempfile = nil
+      end
+
+      # Returns the file path to the data. Prefers a local storage
+      # path to avoid unnecessary copies, falls back to a tempfile.
+      #
+      # @return [String]
+      def file_path
+        local_path || tempfile.path
+      end
+
       # Writes the data to a temporary file.
+      #
+      # @return [Tempfile]
       def tempfile
         unless @tempfile
           @tempfile = Tempfile.new(binmode: true)
@@ -78,6 +126,12 @@ module Dis
         @tempfile
       end
 
+      protected
+
+      def content_hash
+        @record[@record.class.dis_attributes[:content_hash]]
+      end
+
       private
 
       def closest
@@ -88,10 +142,6 @@ module Dis
         end
       end
 
-      def content_hash
-        @record[@record.class.dis_attributes[:content_hash]]
-      end
-
       def raw?
         raw ? true : false
       end
@@ -129,6 +179,12 @@ module Dis
         Dis::Storage.get(storage_type, content_hash)
       end
 
+      def local_path
+        return if raw?
+
+        Dis::Storage.file_path(storage_type, content_hash)
+      end
+
       attr_reader :raw
     end
   end