file_pool 0.5.1 → 0.6.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 133469293b9b495e160951b1abd81d5716aa06b1e9abb18a0f1b98b6904f3ad8
+  data.tar.gz: 1b4befbb63377f359174c7e77caf5e95a831412d709efa688dc1cdfb6b2daf5b
+SHA512:
+  metadata.gz: 28b215968194546b52830604ac034616a2a78d55db738a83d91ac8e2a5ffaf703e658e4535e1865dc37b102026725a3119770c0ac8f4f346d0aed43387be78ad
+  data.tar.gz: f3345fd4c4a9e71dc4a3f02e499ae1b85c2bc4dd4ad9a5c65a573f85f5c99cbe17ac850123cf5bad7dab79f167e2da0e5bc81536be583da9057371f9015c0fb4

data/lib/file_pool/version.rb

@@ -1,3 +1,3 @@
 module FilePool
-  VERSION = "0.5.1"
+  VERSION = "0.6.2"
 end

data/lib/file_pool.rb

@@ -27,20 +27,20 @@ module FilePool
   #     encryption/decryption in bytes. Larger blocks need more memory and less time (less IO).
   #     Defaults to 1'048'576 (1 MiB).
   #   * :copy_source (true,false)
-  #     if +false+ files added to the pool are hard-linked with the source if source and file pool 
+  #     if +false+ files added to the pool are hard-linked with the source if source and file pool
   #     are on the same file system (default). If set to +true+ files are always copied into the pool.
   #   * :mode (Integer)
   #     File mode to set on all files added to the pool. E.g. +mode:+ +0640+ for +rw-r-----+ or symbolic "u=wrx,go=rx"
   #     (see Ruby stdlib FileUtils#chmod).
-  #     Note that the desired mode is not set if the file is hard-linked with the source. 
+  #     Note that the desired mode is not set if the file is hard-linked with the source.
   #     Use +copy_source:true+ when to ensure.
   #   * :owner
-  #     Owner of the files added to the pool. 
-  #     Note that the desired owner is not set if the file is hard-linked with the source. 
+  #     Owner of the files added to the pool.
+  #     Note that the desired owner is not set if the file is hard-linked with the source.
   #     Use +copy_source:true+ when to ensure.
   #   * :group
-  #     Group of the files added to the pool. 
-  #     Note that the desired group is not set if the file is hard-linked with the source. 
+  #     Group of the files added to the pool.
+  #     Note that the desired group is not set if the file is hard-linked with the source.
   #     Use +copy_source:true+ when to ensure.
   def self.setup root, options={}
     unless(unknown = options.keys - [:encryption_block_size, :secrets_file, :copy_source, :mode, :owner, :group]).empty?
@@ -70,32 +70,50 @@ module FilePool
   #
   # path (String)::
   #   path of the file to add.
+  # options (Hash)::
+  #   :background (true,false) adding large files can take long (esp. with encryption), +true+ won't block, default is +false+
   #
   # === Return Value:
   #
   # :: *String* containing a new unique ID for the file added.
-  def self.add! orig_path
+  def self.add! orig_path, options = {}
     newid = uuid
-    target = path newid
 
-    if @@crypted_mode
-      FileUtils.mkpath(id2dir_secured newid)
-      path = crypt(orig_path)      
-    else
-      path = orig_path
-      FileUtils.mkpath(id2dir newid)
-    end
+    raise Errno::ENOENT unless File.exist?(orig_path)
 
-    if !@@copy_source and (File.stat(path).dev == File.stat(File.dirname(target)).dev)
-      FileUtils.link(path, target)
-    else
-      FileUtils.copy(path, target)     
+    child = fork do
+      target = path newid
+
+      if @@crypted_mode
+        FileUtils.mkpath(id2dir_secured newid)
+        path = encrypt_to_tempfile(orig_path)
+      else
+        path = orig_path
+        FileUtils.mkpath(id2dir newid)
+      end
+
+      # try quick hard-linking, copy if forced or hard-linking impossible
+      begin
+        raise Errno::EXDEV if @@copy_source
+        FileUtils.link(path, target)
+      rescue Errno::EXDEV
+        FileUtils.copy(path, target)
+      end
+
+      # don't chmod if orginal file is same as target (hard-linked)
+      if File.stat(orig_path).ino != File.stat(File.dirname(target)).ino
+        FileUtils.chmod(@@mode, target) if @@mode
+        FileUtils.chown(@@owner, @@group, target)
+      end
     end
 
-    # don't chmod if orginal file is same as target (hard-linked) 
-    if File.stat(orig_path).ino != File.stat(File.dirname(target)).ino
-      FileUtils.chmod(@@mode, target) if @@mode
-      FileUtils.chown(@@owner, @@group, target)
+
+    if options[:background]
+      # don't wait, avoid zombies
+      Process.detach(child)
+    else
+      # block until done
+      Process.waitpid(child)
     end
 
     newid
@@ -110,18 +128,49 @@ module FilePool
   #
   # source (String)::
   #   path of the file to add.
+  # options (Hash)::
+  #   :background (true,false) adding large files can take long (esp. with encryption), +true+ won't block, default is +false+
   #
   # === Return Value:
   #
   # :: *String* containing a new unique ID for the file added.
   # :: +false+ when the file could not be stored.
-  def self.add path
-    self.add!(path)
+  def self.add path, options = {}
+    self.add!(path, options)
 
   rescue Exception
     return false
   end
 
+  #
+  # Add a new file from a stream to the file pool
+  #
+  def self.add_stream in_stream
+    newid = uuid
+
+    # ensure target path exists
+    if @@crypted_mode
+      FileUtils.mkpath(id2dir_secured newid)
+    else
+      FileUtils.mkpath(id2dir newid)
+    end
+
+    child = fork do
+      target = path(newid)
+      # create the target file to write
+      open(target,'w') do |out_stream|
+        encrypt in_stream, out_stream
+      end
+
+      FileUtils.chmod(@@mode, target) if @@mode
+      FileUtils.chown(@@owner, @@group, target)
+    end
+
+    Process.detach(child)
+
+    newid
+  end
+
   #
   # Return the file's path corresponding to the passed file ID, no matter if it
   # exists or not. In encrypting mode the file is first decrypted and the
@@ -141,7 +190,7 @@ module FilePool
   #
   # :: *String*, absolute path of the file in the pool or to temporary location if it was decrypted.
   def self.path fid, options={}
-    options[:decrypt] = true unless options[:decrypt] == false
+    options[:decrypt] = options.fetch(:decrypt, true)
 
     raise InvalidFileId unless valid?(fid)
 
@@ -151,7 +200,7 @@ module FilePool
       if @@crypted_mode
         if options[:decrypt]
           # return path of decrypted file (tmp path)
-          decrypt id2dir_secured(fid) + "/#{fid}"
+          decrypt_to_tempfile id2dir_secured(fid) + "/#{fid}"
         else
           id2dir_secured(fid) + "/#{fid}"
         end
@@ -171,6 +220,33 @@ module FilePool
     end
   end
 
+  # Returns an IO object providing an (unencrypted) stream of data of the given file ID
+  # To get the stream of the encrypted data pass :decrypt => false, as an option.
+  #
+  # === Parameters:
+  #
+  # fid (String)::
+  #   File ID which was generated by a previous #add operation.
+  #
+  # options (Hash)::
+  #   :decrypt (true,false) In encryption mode don't decrypt, but prioved the encrypted data. Defaults to +true+.
+  #
+  # === Return Value:
+  #
+  # :: *IO*, IO stream open for reading
+  #
+  def self.stream fid, options={}
+    options[:decrypt] = options.fetch(:decrypt, true)
+
+    if path = path(fid, :decrypt => false)
+      if @@crypted_mode and options[:decrypt]
+        decrypt_to_stream path
+      else
+        open(path)
+      end
+    end
+  end
+
   #
   # Remove a previously added file by its ID. Same as FilePool.remove,
   # but throws exceptions on failure.
@@ -271,7 +347,7 @@ module FilePool
   end
 
   #
-  # Crypt a file and store the result in the temp.
+  # Crypt a file and store the result a Tempfile
   #
   # Returns the path to the crypted file.
   #
@@ -282,28 +358,19 @@ module FilePool
   #
   # === Return Value:
   #
-  # :: *String*Path and name of the crypted file.
-  def self.crypt path
+  # :: *String* Path and name of the crypted file.
+  def self.encrypt_to_tempfile path
     # Crypt the file in the temp folder and copy after
-    cipher = create_cipher
-    result = Tempfile.new 'FilePool-encrypt'
-
-    result.set_encoding Encoding::BINARY,Encoding::BINARY
-    buf = ''.encode(Encoding::BINARY)
+    tempfile = Tempfile.new 'FilePool-encrypt'
+    tempfile.sync = true
 
-    File.open(path) do |inf|
-      while inf.read(@@block_size, buf)
-        result << cipher.update(buf)
-      end
-      result << cipher.final
-    end
+    encrypt open(path), tempfile
 
-    result.close
-    result.path
+    tempfile.path
   end
 
   #
-  # Decrypt a file and give a path to it.
+  # Decrypt file to a file in /tmp
   #
   # Returns the path to the decrypted file.
   #
@@ -314,26 +381,76 @@ module FilePool
   #
   # === Return Value:
   #
-  # :: *String*Path and name of the crypted file.
-  def self.decrypt path
-    decipher = create_decipher
-    # Now decrypt the data:
-    output = Tempfile.new 'FilePool-decrypt'
+  # :: *String* Path and name of the decrypted file.
+  def self.decrypt_to_tempfile path
+    tmpfile = Tempfile.new 'FilePool-decrypt'
+    tmpfile.sync = true
 
-    output.set_encoding Encoding::BINARY,Encoding::BINARY
-    buf = ''.encode(Encoding::BINARY)
+    File.open(path) do |encrypted_file|
+      decrypt encrypted_file, tmpfile
+    end
+
+    tmpfile.path
+  end
 
-    File.open(path) do |inf|
-      while inf.read(@@block_size, buf)
-        output << decipher.update(buf)
+  #
+  # Decrypt file to a stream (IO) non-blocking by forking a child process.
+  #
+  # === Parameters:
+  #
+  # path (String)::
+  #   path of the file to decrypt.
+  #
+  # === Return Value:
+  #
+  # :: *IO* decrypted data as stream
+  def self.decrypt_to_stream path
+
+    pipe_read, pipe_write = IO.pipe
+
+    child = fork do
+      # in child process: decrypting
+      pipe_read.close
+
+      # open encrypted file
+      File.open(path) do |encrypted_file|
+        decrypt encrypted_file, pipe_write
       end
-      output << decipher.final
+
+      pipe_write.close
     end
 
-    output.open # re-open for reading, prevents early deletion of tempfile
-    output.path
+    # in parent
+    Process.detach(child) # not waiting for it
+    pipe_write.close
+    pipe_read
   end
 
+
+  # decrypts data stream +in_stream+ (IO: readable) to +out+ (IO: writeable), blocking
+  def self.decrypt in_stream, out_stream
+    crypt(in_stream, out_stream, :decrypt)
+  end
+
+  # encrypts data stream +in+ (IO: readable) to +out+ (IO: writeable), blocking
+  def self.encrypt in_stream, out_stream
+    crypt(in_stream, out_stream, :encrypt)
+  end
+
+  def self.crypt in_stream, out_stream, direction
+    cipher = (direction == :encrypt) ? create_cipher : create_decipher
+
+    buf = ''.encode(Encoding::BINARY)
+    out_stream.set_encoding Encoding::BINARY,Encoding::BINARY
+    in_stream.set_encoding Encoding::BINARY,Encoding::BINARY
+
+    while in_stream.read(@@block_size, buf)
+      out_stream << cipher.update(buf)
+    end
+    out_stream << cipher.final
+
+    nil
+  end
   #
   # Creates a cipher to encrypt data.
   #

metadata

@@ -1,104 +1,77 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: file_pool
-version: !ruby/object:Gem::Version 
-  hash: 9
-  prerelease: 
-  segments: 
-  - 0
-  - 5
-  - 1
-  version: 0.5.1
+version: !ruby/object:Gem::Version
+  version: 0.6.2
 platform: ruby
-authors: 
-- "robokopp (Robert Anni\xC3\xA9s)"
-autorequire: 
+authors:
+- robokopp (Robert Anniés)
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2018-03-26 00:00:00 +02:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2022-01-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: uuidtools
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 2
-        - 1
-        - 2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
         version: 2.1.2
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: pry
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |
   FilePool helps to manage a large number of files in a Ruby
   project. It takes care of the storage of files in a balanced directory
   tree and generates unique identifiers for all files.
-
-email: 
+email:
 - robokopp@fernwerk.net
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
+- README.md
+files:
 - README.md
-files: 
 - lib/file_pool.rb
 - lib/file_pool/version.rb
-- README.md
-has_rdoc: true
 homepage: https://github.com/robokopp/file_pool
 licenses: []
-
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ~>
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 2
-      - 0
-      version: "2.0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.6.2
-signing_key: 
-specification_version: 3
+rubygems_version: 3.2.22
+signing_key:
+specification_version: 4
 summary: Manage a large number files in a pool
 test_files: []
-