dis 1.2.0 → 1.3.1

data/lib/dis/layer.rb

@@ -3,52 +3,47 @@
 module Dis
   # = Dis Layer
   #
-  # Represents a layer of storage. It's a wrapper around
-  # <tt>Fog::Storage</tt>, any provider supported by Fog should be usable.
-  #
-  # ==== Options
-  #
-  # * <tt>:delayed</tt> - Delayed layers will be processed outside of
-  #   the request cycle by ActiveJob.
-  # * <tt>:readonly</tt> - Readonly layers can only be read from,
-  #   not written to.
-  # * <tt>:public</tt> - Objects stored in public layers will have the
-  #   public readable flag set if supported by the storage provider.
-  # * <tt>:path</tt> - Directory name to use for the store. For Amazon S3,
-  #   this will be the name of the bucket.
-  #
-  # ==== Examples
-  #
-  # This creates a local storage layer. It's a good idea to have a local layer
-  # first, this provides you with a cache on disk that will be faster than
-  # reading from the cloud.
+  # Represents a layer of storage. Wraps a +Fog::Storage+ connection;
+  # any provider supported by Fog should be usable.
   #
+  # @example Local storage layer
   #   Dis::Layer.new(
-  #     Fog::Storage.new({
-  #       provider: 'Local',
-  #       local_root: Rails.root.join('db', 'dis')
-  #     }),
+  #     Fog::Storage.new(
+  #       provider: "Local",
+  #       local_root: Rails.root.join("db/dis")
+  #     ),
   #     path: Rails.env
   #   )
   #
-  # This creates a delayed layer on Amazon S3. ActiveJob will kick in and
-  # and transfer content from one of the immediate layers later at it's
-  # leisure.
-  #
+  # @example Delayed layer on Amazon S3
   #   Dis::Layer.new(
-  #     Fog::Storage.new({
-  #       provider:              'AWS',
-  #       aws_access_key_id:     YOUR_AWS_ACCESS_KEY_ID,
+  #     Fog::Storage.new(
+  #       provider: "AWS",
+  #       aws_access_key_id: YOUR_AWS_ACCESS_KEY_ID,
   #       aws_secret_access_key: YOUR_AWS_SECRET_ACCESS_KEY
-  #     }),
+  #     ),
   #     path: "my_bucket",
   #     delayed: true
   #   )
   class Layer
     include Dis::Logging
 
+    # @return [Fog::Storage] the underlying Fog connection
     attr_reader :connection
 
+    # @param connection [Fog::Storage] a Fog storage connection
+    # @param options [Hash] layer configuration options
+    # @option options [Boolean] :delayed (false) process writes
+    #   asynchronously via ActiveJob
+    # @option options [Boolean] :readonly (false) only allow reads
+    # @option options [Boolean] :public (false) set the public
+    #   readable flag on stored objects (provider-dependent)
+    # @option options [String] :path (nil) directory or bucket name
+    # @option options [Integer, false] :cache (false) enable bounded
+    #   cache with this soft size limit in bytes. Cannot be combined
+    #   with +:delayed+ or +:readonly+
+    # @raise [ArgumentError] if +:cache+ is combined with +:delayed+
+    #   or +:readonly+
     def initialize(connection, options = {})
       options     = default_options.merge(options)
       @connection = connection
@@ -56,44 +51,102 @@ module Dis
       @readonly   = options[:readonly]
       @public     = options[:public]
       @path       = options[:path]
+      @cache      = options[:cache]
+      validate_cache_options!
     end
 
     # Returns true if the layer is a delayed layer.
+    #
+    # @return [Boolean]
     def delayed?
       @delayed
     end
 
     # Returns true if the layer isn't a delayed layer.
+    #
+    # @return [Boolean]
     def immediate?
       !delayed?
     end
 
     # Returns true if the layer is public.
+    #
+    # @return [Boolean]
     def public?
       @public
     end
 
     # Returns true if the layer is read only.
+    #
+    # @return [Boolean]
     def readonly?
       @readonly
     end
 
     # Returns true if the layer is writeable.
+    #
+    # @return [Boolean]
     def writeable?
       !readonly?
     end
 
-    # Stores a file.
+    # Returns true if the layer is a cache layer.
     #
-    #   key = Digest::SHA1.file(file.path).hexdigest
-    #   layer.store("documents", key, path)
+    # @return [Boolean]
+    def cache?
+      !!@cache
+    end
+
+    # Returns the cache size limit in bytes, or nil if not a cache.
+    #
+    # @return [Integer, nil]
+    def max_size
+      @cache if cache?
+    end
+
+    # Returns the total size in bytes of all files stored locally.
+    # Returns 0 for non-local providers.
     #
-    # The key must be a hex digest of the file content. If an object with the
-    # supplied hash already exists, no action will be performed. In other
-    # words, no data will be overwritten if a hash collision occurs.
+    # @return [Integer]
+    def size
+      return 0 unless connection.respond_to?(:local_root)
+
+      root = local_root_path
+      return 0 unless root.exist?
+
+      root.glob("**/*").sum { |f| f.file? ? f.size : 0 }
+    end
+
+    # Returns cached file entries sorted by mtime ascending
+    # (oldest first).
     #
-    # Returns an instance of Fog::Model, or raises an error if the layer
-    # is readonly.
+    # @return [Array<Hash>] each entry has keys +:path+
+    #   (Pathname), +:type+ (String), +:key+ (String), +:mtime+
+    #   (Time), +:size+ (Integer)
+    def cached_files
+      return [] unless connection.respond_to?(:local_root)
+
+      root = local_root_path
+      return [] unless root.exist?
+
+      entries = root.glob("**/*").select(&:file?)
+      entries.filter_map { |f| cached_file_entry(f, root) }
+             .sort_by { |e| e[:mtime] }
+    end
+
+    # Stores a file. The key must be a hex digest of the file
+    # content. If an object with the supplied hash already exists,
+    # no action will be performed.
+    #
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @param file [File, IO, String, Fog::Model] the content
+    # @return [Fog::Model] the stored file
+    # @raise [Dis::Errors::ReadOnlyError] if the layer is readonly
+    #
+    # @example
+    #   key = Digest::SHA1.file(file.path).hexdigest
+    #   layer.store("documents", key, file)
     def store(type, key, file)
       raise Dis::Errors::ReadOnlyError if readonly?
 
@@ -104,18 +157,42 @@ module Dis
 
     # Returns all the given keys that exist in the layer.
     #
-    #    layer.existing("documents", keys)
+    # @param type [String] the type scope
+    # @param keys [Array<String>] content hashes to check
+    # @return [Array<String>] the subset of keys that exist
     def existing(type, keys)
       return [] if keys.empty?
 
-      list = directory(type, keys.first).files.map(&:key)
+      futures = keys.map do |key|
+        Concurrent::Promises.future { key if exists?(type, key) }
+      end
+      futures.filter_map(&:value!)
+    end
+
+    # Returns all content hashes stored under the given type.
+    #
+    # @param type [String] the type scope
+    # @return [Array<String>] content hashes
+    def stored_keys(type)
+      dir = connection.directories.get(path || "")
+      return [] unless dir
+
+      prefix = "#{type}/"
+      dir.files.filter_map do |file|
+        next unless file.key.start_with?(prefix)
 
-      keys.select { |key| list.include?(key_component(type, key)) }
+        parts = file.key.delete_prefix(prefix).split("/")
+        next unless parts.length == 2
+
+        "#{parts[0]}#{parts[1]}"
+      end
     end
 
-    # Returns true if a object with the given key exists.
+    # Returns true if an object with the given key exists.
     #
-    #    layer.exists?("documents", key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [Boolean]
     def exists?(type, key)
       if directory(type, key)&.files&.head(key_component(type, key))
         true
@@ -126,37 +203,64 @@ module Dis
 
     # Retrieves a file from the store.
     #
-    #    layer.get("documents", key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [Fog::Model, nil] the file, or nil if not found
     def get(type, key)
       dir = directory(type, key)
       return unless dir
 
-      debug_log("Get #{type}/#{key} from #{name}") do
+      result = debug_log("Get #{type}/#{key} from #{name}") do
         dir.files.get(key_component(type, key))
       end
+      refresh_cache_mtime(local_file_path(type, key)) if result && cache?
+      result
     end
 
-    # Returns the absolute file path for a locally stored file, or nil
-    # if the provider is not local or the file does not exist.
+    # Writes the contents of a file into the given IO. Local layers
+    # copy from disk; remote layers stream in chunks, never holding
+    # the whole body in memory.
     #
-    #    layer.file_path("documents", key)
-    def file_path(type, key)
-      return unless connection.respond_to?(:local_root)
-      return unless exists?(type, key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @param io [IO] the destination to write to
+    # @return [Boolean] true if the file was found and written
+    def stream(type, key, io)
+      local = local_file_path(type, key)
+      if local
+        copy_local(local, io)
+        return true
+      end
 
-      File.join(
-        connection.local_root,
-        directory_component(type, key),
-        key_component(type, key)
-      )
+      dir = directory(type, key)
+      return false unless dir
+
+      debug_log("Stream #{type}/#{key} from #{name}") do
+        !fetch_chunks(dir, type, key, io).nil?
+      end
     end
 
-    # Deletes a file from the store.
+    # Returns the absolute file path for a locally stored file, or
+    # nil if the provider is not local or the file does not exist.
+    # On cache layers this counts as a read and may refresh the
+    # file's mtime.
     #
-    #   layer.delete("documents", key)
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [String, nil]
+    def file_path(type, key)
+      path = local_file_path(type, key)
+      refresh_cache_mtime(path) if cache?
+      path
+    end
+
+    # Deletes a file from the store.
     #
-    # Returns true if the file was deleted, or false if it could not be found.
-    # Raises an error if the layer is readonly.
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @return [Boolean] true if the file was deleted, false if not
+    #   found
+    # @raise [Dis::Errors::ReadOnlyError] if the layer is readonly
     def delete(type, key)
       raise Dis::Errors::ReadOnlyError if readonly?
 
@@ -167,6 +271,9 @@ module Dis
 
     # Returns a name for the layer.
     #
+    # @return [String]
+    #
+    # @example
     #   layer.name # => "Fog::Storage::Local::Real/development"
     def name
       "#{connection.class.name}/#{path}"
@@ -175,7 +282,65 @@ module Dis
     private
 
     def default_options
-      { delayed: false, readonly: false, public: false, path: nil }
+      { delayed: false, readonly: false, public: false,
+        path: nil, cache: false }
+    end
+
+    def validate_cache_options!
+      return unless cache?
+
+      if delayed?
+        raise ArgumentError,
+              "cache layers cannot be delayed"
+      end
+      return unless readonly?
+
+      raise ArgumentError,
+            "cache layers cannot be readonly"
+    end
+
+    def local_root_path
+      root = Pathname.new(connection.local_root)
+      path ? root.join(path) : root
+    end
+
+    def cached_file_entry(file, root)
+      parts = file.relative_path_from(root).to_s.split("/")
+      return unless parts.length == 3
+
+      { path: file, type: parts[0], key: parts[1] + parts[2],
+        mtime: file.mtime, size: file.size }
+    end
+
+    def local_file_path(type, key)
+      return unless connection.respond_to?(:local_root)
+      return unless exists?(type, key)
+
+      File.join(
+        connection.local_root,
+        directory_component(type, key),
+        key_component(type, key)
+      )
+    end
+
+    def refresh_cache_mtime(path)
+      return unless path
+      return unless File.mtime(path) < 1.minute.ago
+
+      FileUtils.touch(path)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def copy_local(path, io)
+      refresh_cache_mtime(path) if cache?
+      IO.copy_stream(path, io)
+    end
+
+    def fetch_chunks(dir, type, key, io)
+      dir.files.get(key_component(type, key)) do |chunk, _rem, _total|
+        io.write(chunk)
+      end
     end
 
     def directory_component(_type, _key)
@@ -183,7 +348,7 @@ module Dis
     end
 
     def key_component(type, key)
-      [type, key[0...2], key[2..key.length]].compact.join("/")
+      [type, key[0...2], key[2..]].compact.join("/")
     end
 
     def delete!(type, key)

data/lib/dis/layers.rb

@@ -3,10 +3,15 @@
 module Dis
   # = Dis Layers
   #
-  # Represents a collection of layers.
+  # Represents a filterable collection of {Dis::Layer} instances.
+  # Supports chained filtering by layer properties.
+  #
+  # @example
+  #   Dis::Storage.layers.delayed.writeable.each { |l| ... }
   class Layers
     include Enumerable
 
+    # @param layers [Array<Dis::Layer>] initial layers
     def initialize(layers = [])
       @layers = layers
     end
@@ -15,6 +20,8 @@ module Dis
     delegate :<<, to: :@layers
 
     # Clears all layers from the collection.
+    #
+    # @return [void]
     def clear!
       @layers = []
     end
@@ -25,43 +32,87 @@ module Dis
     end
 
     # Returns a new instance containing only the delayed layers.
+    #
+    # @return [Dis::Layers]
     def delayed
       self.class.new select(&:delayed?)
     end
 
     # Returns true if one or more delayed layers exist.
+    #
+    # @return [Boolean]
     def delayed?
-      delayed.any?
+      any?(&:delayed?)
     end
 
     # Returns a new instance containing only the immediate layers.
+    #
+    # @return [Dis::Layers]
     def immediate
       self.class.new select(&:immediate?)
     end
 
     # Returns true if one or more immediate layers exist.
+    #
+    # @return [Boolean]
     def immediate?
-      immediate.any?
+      any?(&:immediate?)
     end
 
     # Returns a new instance containing only the readonly layers.
+    #
+    # @return [Dis::Layers]
     def readonly
       self.class.new select(&:readonly?)
     end
 
     # Returns true if one or more readonly layers exist.
+    #
+    # @return [Boolean]
     def readonly?
-      readonly.any?
+      any?(&:readonly?)
     end
 
     # Returns a new instance containing only the writeable layers.
+    #
+    # @return [Dis::Layers]
     def writeable
       self.class.new select(&:writeable?)
     end
 
     # Returns true if one or more writeable layers exist.
+    #
+    # @return [Boolean]
     def writeable?
-      writeable.any?
+      any?(&:writeable?)
+    end
+
+    # Returns a new instance containing only the cache layers.
+    #
+    # @return [Dis::Layers]
+    def cache
+      self.class.new select(&:cache?)
+    end
+
+    # Returns true if one or more cache layers exist.
+    #
+    # @return [Boolean]
+    def cache?
+      any?(&:cache?)
+    end
+
+    # Returns a new instance containing only the non-cache layers.
+    #
+    # @return [Dis::Layers]
+    def non_cache
+      self.class.new reject(&:cache?)
+    end
+
+    # Returns true if one or more non-cache layers exist.
+    #
+    # @return [Boolean]
+    def non_cache?
+      any? { |l| !l.cache? }
     end
   end
 end

data/lib/dis/model/class_methods.rb

@@ -4,13 +4,21 @@ module Dis
   module Model
     module ClassMethods
       # Returns the mapping of attribute names.
+      #
+      # @return [Hash{Symbol => Symbol}]
       def dis_attributes
         default_dis_attributes.merge(@dis_attributes ||= {})
       end
 
-      # Sets the current mapping of attribute names. Use this if you want to
-      # override the attributes and database columns that Dis will use.
+      # Sets the current mapping of attribute names. Use this if you
+      # want to override the attributes and database columns that
+      # Dis will use. Valid keys: +:content_hash+, +:content_type+,
+      # +:content_length+, +:filename+.
+      #
+      # @param new_attributes [Hash{Symbol => Symbol}] attribute
+      #   name overrides
       #
+      # @example
       #   class Document < ActiveRecord::Base
       #     include Dis::Model
       #     self.dis_attributes = { filename: :my_custom_filename }
@@ -20,8 +28,11 @@ module Dis
       end
 
       # Returns the storage type name, which Dis will use for
-      # directory scoping. Defaults to the name of the database table.
+      # directory scoping. Defaults to the table name.
+      #
+      # @return [String]
       #
+      # @example
       #   class Document < ActiveRecord::Base; end
       #   Document.dis_type # => "documents"
       def dis_type
@@ -30,17 +41,23 @@ module Dis
 
       # Sets the storage type name.
       #
-      # Take care not to set the same name for multiple models, this will
-      # cause data loss when a record is destroyed.
+      # Take care not to set the same name for multiple models,
+      # this will cause data loss when a record is destroyed.
+      #
+      # @param new_type [String] the new type scope
+      # @return [void]
       def dis_type=(new_type)
         @dis_type = new_type
       end
 
       # Adds a presence validation on the +data+ attribute.
       #
-      # This is better than using `validates :data, presence: true`, since
-      # that would cause it to load the data from storage on each save.
+      # This is preferred over +validates :data, presence: true+,
+      # which would load the data from storage on each save.
+      #
+      # @return [void]
       #
+      # @example
       #   class Document < ActiveRecord::Base
       #     include Dis::Model
       #     validates_data_presence