dis 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf532ee41185ec1d1fb96f36e9cd697a7a1c0ece44da5a30b2ff01d8a6c77010
-  data.tar.gz: 8d338d164af992ca38b9af28840c97eabc33f6434791a5505b5a602dd1aa0b3f
+  metadata.gz: cbda8b2db5161c2dd8357c1f06102a27dfc4ff3e1a3eb79974b29abb6fa27c4d
+  data.tar.gz: 4cd6907621cae900a48f021f2bd0976d5721cfaa1a4a368af0f54e728c572543
 SHA512:
-  metadata.gz: 8db87ae2b81b5639fc9312a3dbad3580e6b96886bcb22ddcd6d834efa7452e7348738c37d656ea86c343041cbde683b68f4277a28125f0467de4878b3ca2d3b0
-  data.tar.gz: bde07caf197b9a2ae72f3403c3b0f4ad189dbca90259768354eaa96ae7f026edcb3bc6fa86c3b2378361b9eb76ef8a9f5a8446b8584dd5b375861f0de546c740
+  metadata.gz: 2dc2ee898c2864535927cd472c46c6f7d7db082e1adfe95b4366b9a7679af772ff64c26475dcab2257a49284663170269d03627dd69416fe87a89ee6be8b3491
+  data.tar.gz: 5c43189d589f280df560496e3fe37ef50ec19217fd031e93f551439e3ce97804d5e71a33b53acda960e96e0a5fe1855fa1ac8caca8881599810b1e790f56d582

data/README.md

@@ -104,6 +104,36 @@ class DocumentsController < ApplicationController
 end
 ```
 
+### Accessing the data
+
+Which accessor you want depends on how long the data needs to stay
+valid.
+
+`data` returns the content as a binary string.
+
+```ruby
+document.data # => "foobar"
+```
+
+`with_data_file` yields a path, for tools that want a file name rather
+than the bytes. It is valid for the duration of the block, so resolve
+anything lazy before returning.
+
+```ruby
+document.with_data_file { |path| Vips::Image.new_from_file(path.to_s).avg }
+```
+
+`open_data` returns an open file, valid until you close it — even if
+the content is deleted or evicted from a cache layer meanwhile. Use it
+when the reader outlives the current call stack, such as a response
+body handed to the web server.
+
+```ruby
+document.open_data { |file| file.read }
+
+file = document.open_data # caller closes it
+```
+
 ## Layers
 
 The underlying storage consists of one or more layers. Each layer

data/lib/dis/layer.rb

@@ -213,25 +213,45 @@ module Dis
       result = debug_log("Get #{type}/#{key} from #{name}") do
         dir.files.get(key_component(type, key))
       end
-      touch_file(type, key) if result && cache?
+      refresh_cache_mtime(local_file_path(type, key)) if result && cache?
       result
     end
 
+    # Writes the contents of a file into the given IO. Local layers
+    # copy from disk; remote layers stream in chunks, never holding
+    # the whole body in memory.
+    #
+    # @param type [String] the type scope
+    # @param key [String] the content hash
+    # @param io [IO] the destination to write to
+    # @return [Boolean] true if the file was found and written
+    def stream(type, key, io)
+      local = local_file_path(type, key)
+      if local
+        copy_local(local, io)
+        return true
+      end
+
+      dir = directory(type, key)
+      return false unless dir
+
+      debug_log("Stream #{type}/#{key} from #{name}") do
+        !fetch_chunks(dir, type, key, io).nil?
+      end
+    end
+
     # Returns the absolute file path for a locally stored file, or
     # nil if the provider is not local or the file does not exist.
+    # On cache layers this counts as a read and may refresh the
+    # file's mtime.
     #
     # @param type [String] the type scope
     # @param key [String] the content hash
     # @return [String, nil]
     def file_path(type, key)
-      return unless connection.respond_to?(:local_root)
-      return unless exists?(type, key)
-
-      File.join(
-        connection.local_root,
-        directory_component(type, key),
-        key_component(type, key)
-      )
+      path = local_file_path(type, key)
+      refresh_cache_mtime(path) if cache?
+      path
     end
 
     # Deletes a file from the store.
@@ -292,9 +312,35 @@ module Dis
         mtime: file.mtime, size: file.size }
     end
 
-    def touch_file(type, key)
-      fp = file_path(type, key)
-      FileUtils.touch(fp) if fp
+    def local_file_path(type, key)
+      return unless connection.respond_to?(:local_root)
+      return unless exists?(type, key)
+
+      File.join(
+        connection.local_root,
+        directory_component(type, key),
+        key_component(type, key)
+      )
+    end
+
+    def refresh_cache_mtime(path)
+      return unless path
+      return unless File.mtime(path) < 1.minute.ago
+
+      FileUtils.touch(path)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def copy_local(path, io)
+      refresh_cache_mtime(path) if cache?
+      IO.copy_stream(path, io)
+    end
+
+    def fetch_chunks(dir, type, key, io)
+      dir.files.get(key_component(type, key)) do |chunk, _rem, _total|
+        io.write(chunk)
+      end
     end
 
     def directory_component(_type, _key)

data/lib/dis/model/data.rb

@@ -93,37 +93,42 @@ module Dis
         Dis::Storage.store(storage_type, raw)
       end
 
-      # Clears cached data and tempfiles, allowing them to be
-      # garbage collected. Subsequent calls to +read+ or +tempfile+
-      # will re-fetch from storage.
+      # Clears cached data, allowing it to be garbage collected.
+      # Subsequent calls to +read+ will re-fetch from storage.
       #
       # @return [void]
       def reset_read_cache!
         @read = nil
-        return unless @tempfile
-
-        @tempfile.close!
-        @tempfile = nil
       end
 
-      # Returns the file path to the data. Prefers a local storage
-      # path to avoid unnecessary copies, falls back to a tempfile.
+      # Yields a path to the data, removing any temporary copy
+      # afterwards.
       #
-      # @return [String]
-      def file_path
-        local_path || tempfile.path
+      # @yieldparam path [Pathname] path to the data
+      # @return [Object] the return value of the block
+      def with_file
+        path = local_path
+        return yield(Pathname.new(path)) if path
+
+        file = materialize
+        begin
+          yield(Pathname.new(file.path))
+        ensure
+          close_and_unlink(file)
+        end
       end
 
-      # Writes the data to a temporary file.
+      # Returns the data as an open file. Any temporary copy is
+      # unlinked first, leaving the kernel to reclaim it on close.
       #
-      # @return [Tempfile]
-      def tempfile
-        unless @tempfile
-          @tempfile = Tempfile.new(binmode: true)
-          @tempfile.write(@read || read_from(closest))
-          @tempfile.open
-        end
-        @tempfile
+      # @return [File] an open file, positioned at the start
+      def open
+        path = local_path
+        return File.open(path, "rb") if path
+
+        file = materialize
+        File.unlink(file.path)
+        file
       end
 
       protected
@@ -185,6 +190,48 @@ module Dis
         Dis::Storage.file_path(storage_type, content_hash)
       end
 
+      # Writes the data to a new temporary file. The caller owns it.
+      def materialize
+        file = Tempfile.create
+        file.binmode
+        fill(file)
+        file.rewind
+        file
+      rescue StandardError
+        close_and_unlink(file) if file
+        raise
+      end
+
+      def fill(file)
+        if raw?
+          write_raw_to(file)
+        else
+          Dis::Storage.get_file(storage_type, content_hash, file)
+        end
+      end
+
+      def write_raw_to(file)
+        if raw.respond_to?(:read)
+          rewind_raw
+          IO.copy_stream(raw, file)
+          rewind_raw
+        else
+          file.write(raw)
+        end
+        file.flush
+      end
+
+      def rewind_raw
+        raw.rewind if raw.respond_to?(:rewind)
+      end
+
+      def close_and_unlink(file)
+        file.close unless file.closed?
+        File.unlink(file.path)
+      rescue Errno::ENOENT
+        nil
+      end
+
       attr_reader :raw
     end
   end

data/lib/dis/model.rb

@@ -151,19 +151,36 @@ module Dis
       dis_set :filename, file.original_filename
     end
 
-    # Returns a file path to the data, preferring local storage
-    # paths. Falls back to a tempfile path if no local layer has
-    # the file.
+    # Yields the path to a file containing the data, for tools that
+    # need a file name rather than the bytes. The path is valid for
+    # the duration of the block only, so resolve anything lazy before
+    # returning.
     #
-    # @return [String]
-    def data_file_path
-      dis_data.file_path
+    # @yieldparam path [Pathname] path to the data
+    # @return [Object] the return value of the block
+    #
+    # @example
+    #   document.with_data_file { |path| Vips::Image.new_from_file(path.to_s).avg }
+    def with_data_file(&)
+      dis_data.with_file(&)
     end
 
-    # Returns the data as a temporary file.
+    # Returns the data as an open, read-only file, or yields it and
+    # closes it afterwards. The data stays readable until the file is
+    # closed, even if the content is deleted or evicted meanwhile.
     #
-    # @return [Tempfile]
-    delegate :tempfile, to: :dis_data
+    # @yieldparam file [File] an open file, positioned at the start
+    # @return [File, Object] the open file, or the block's value
+    def open_data
+      file = dis_data.open
+      return file unless block_given?
+
+      begin
+        yield file
+      ensure
+        file.close unless file.closed?
+      end
+    end
 
     private
 

data/lib/dis/storage.rb

@@ -166,6 +166,29 @@ module Dis
         nil
       end
 
+      # Streams the contents of a file into the given file, fetching
+      # from the first layer that has it. Backfills faster layers if
+      # the content had to be fetched from further down.
+      #
+      # @param type [String] the type scope
+      # @param key [String] the content hash
+      # @param file [File] the destination, must respond to +path+
+      # @return [File] the file that was written to, positioned at
+      #   the start
+      # @raise [Dis::Errors::NoLayersError] if no layers are configured
+      # @raise [Dis::Errors::NotFoundError] if the file is not found
+      def get_file(type, key, file)
+        require_layers!
+        fetch_count = 0
+        found = layers.detect do |layer|
+          fetch_count += 1
+          stream_from_layer(layer, type, key, file)
+        end
+        raise Dis::Errors::NotFoundError unless found
+
+        finalize_fetched_file(type, file, fetch_count)
+      end
+
       # Deletes a file from all layers. Kicks off a
       # {Dis::Jobs::Delete} job if any delayed layers are defined.
       #
@@ -304,6 +327,22 @@ module Dis
         end
       end
 
+      def finalize_fetched_file(type, file, fetch_count)
+        file.flush
+        backfill!(type, file) if fetch_count > 1
+        file.rewind
+        file
+      end
+
+      def stream_from_layer(layer, type, key, file)
+        file.truncate(0)
+        file.rewind
+        layer.stream(type, key, file)
+      rescue StandardError => e
+        report_layer_error(e, layer:, type:, key:)
+        false
+      end
+
       def fetch_from_layer(layer, type, key)
         layer.get(type, key)
       rescue StandardError => e

data/lib/dis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dis
-  VERSION = "1.3.0"
+  VERSION = "1.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dis
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Inge Jørgensen
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: A file store for your Rails app
 test_files: []