file_pool 0.2.0 → 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 73a8c6c237185c15b0385f37405649571c3668db
+  data.tar.gz: 9f6a23a5c02abe1394956d3ee74590f477224595
+SHA512:
+  metadata.gz: f348a4491e460c47cc3d5e288cb365aa3ba965cebcfd2858646c03e680a587951538406a3615e3e1d7ba9371c6f058e47c3578cae725ce69612b146691bf3102
+  data.tar.gz: fb19cf25d543f63d3c364a96bd08f9ad2a4a52d56c46826eb78fec6361419faceab4c8865d6a37a6948fae964c0c78ee9aa95ce798527593b53726cc69f045f5

data/README.md

@@ -2,8 +2,9 @@
 
 FilePool helps to manage a large number of files in a Ruby project. It
 takes care of the storage of files in a balanced directory tree and
-generates unique identifiers for all files. It also comes in handy
-when dealing with only a few files.
+generates unique identifiers for all files.
+
+Optionally FilePool can use symmetric encryption for all files managed.
 
 FilePool does not deal with file meta information. It's only purpose
 is to return a file's location given a file identifier, which was
@@ -17,7 +18,7 @@ the 3 first hexadecimal digits of a UUID as path. For example:
     0/c/f/0cfb082a-fd57-490c-978b-e47d5948bc8b
     6/1/d/61ddfe33-13f3-4f71-9234-5fbbf5c4fc2c
 
-FilePool is tested with Ruby 1.8.7 and 1.9.3.
+FilePool is tested with Ruby 1.8.7, 2.0.0 and 2.2.1.
 
 ## Installation
 
@@ -36,13 +37,14 @@ Or install it yourself as:
 ## Usage
 
 ### Setup
-The root path and optionally a Logger must be defined:
+
+Set up the root path under which all files will reside:
 
     FilePool.setup '/var/lib/files'
 
 In a Rails project the file pool setup would be placed in an intializer:
 
-    config/initializers/file_pool.
+    config/initializers/file_pool.rb
 
 ### Example Usage
 
@@ -58,6 +60,21 @@ Remove a file
 
     FilePool.remove(fid)
 
+### Encryption
+
+FilePool can store files symmetrically encrypted using AES-256-CBC. In order to
+switch FilePool to encryption pass the location of a file containing the secret:
+
+    FilePool.setup '/var/lib/files', :secrets_file => '/etc/filepool_secrets.yml'
+
+This file is initialized with a random secret if it is not present. The key and
+initialization vector stored there will be used for all files. When you request
+a file with the FilePool#path method FilePool decrypts it first and returns a path to
+the decrypted file.
+
+In encryption mode the filepool root directory name is suffixed with "_secured",
+so that mixed mode (encryped and plain) operation is possible.
+
 ### Maintenance
 
 FilePool has a straight forward way of storing files. It doesn't use

data/lib/file_pool/version.rb

@@ -1,3 +1,3 @@
 module FilePool
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/file_pool.rb

@@ -1,77 +1,10 @@
 # -*- coding: utf-8 -*-
 require 'file_pool/version'
 require 'uuidtools'
-=begin
-<em>Robert Anniés (2012)</em>
+require 'tempfile'
+require 'openssl'
+require 'yaml'
 
-== Introduction
-
-FilePool helps to manage a large number of files in a Ruby project. It
-takes care of the storage of files in a balanced directory tree and
-generates unique identifiers for all files. It also comes in handy
-when delaing with only a few files.
-
-FilePool does not deal with file meta information. It's only purpose
-is to return a file's location given a file identifier, which was
-generated when the file was added to the pool.
-
-The identifiers are strings of UUID Type 4 (random), which are also
-used as file names. The directory tree is a 3 level structure using
-the 3 first hexadecimal digits of a UUID as path. For example:
-
-  0/d/6/0d6f8dd9-8deb-4500-bb85-2d0796241963
-  0/c/f/0cfb082a-fd57-490c-978b-e47d5948bc8b
-  6/1/d/61ddfe33-13f3-4f71-9234-5fbbf5c4fc2c
-
-== Examples
-
-=== Setup
-The root path and optionally a Logger must be defined:
-
-  FilePool.setup '/var/lib/files'
-
-In a Rails project the file pool setup should be placed in an intializer:
-
-  config/initializers/file_pool.rb
-
-=== Usage
-
-Adding files (perhaps after completed upload)
-
-  fid = FilePool.add('/Temp/p348dvhn4')
-
-Get location of previously added file
-
-  path = FilePool.path(fid)
-
-Remove a file
-
-  FilePool.remove(fid)
-
-== Maintenance
-
-FilePool has a straight forward way of storing files. It doesn't use
-any form of index. As long as you stick to directory structure
-outlined above you can:
-
-* move the entire pool somewhere else
-* split the pool using symbolic links or mount points to remote file systems
-* merge file pools by copying them into one
-
-There is no risk of overwriting, because UUID type 4 file names are
-unique. (up to an extremely small collision probability).
-
-== Notes
-
-Make sure to store the generated file identifiers safely. There is no
-way of identifying a file again when it's ID is lost. In doubt generate a hash
-value from the file and store it somewhere else.
-
-For large files the pool root should be on the same file system as the files
-added to the pool. Then adding a file returns immediately. Otherwise
-files will be copied which may take a significant time.
-
-=end
 module FilePool
 
   class InvalidFileId < Exception; end
@@ -84,8 +17,12 @@ module FilePool
   #
   # root (String)::
   #   absolute path of the file pool's root directory under which all files will be stored.
-  def self.setup root
+  # config_file_path (String)::
+  #   path to the config file of the filepool.
+  def self.setup root, options={}
     @@root = root
+    @@crypted_mode = false
+    configure options[:secrets_file]
   end
 
   #
@@ -110,7 +47,12 @@ module FilePool
     newid = uuid
     target = path newid
 
-    FileUtils.mkpath(id2dir newid)
+    if @@crypted_mode
+      FileUtils.mkpath(id2dir_secured newid)
+      path = crypt(path)
+    else
+      FileUtils.mkpath(id2dir newid)
+    end
     FileUtils.link(path, target)
 
     return newid
@@ -142,19 +84,52 @@ module FilePool
   end
 
   #
-  # Return the path of a previously added file by its ID.
+  # Return the file's path corresponding to the passed file ID, no matter if it
+  # exists or not. In encrypting mode the file is first decrypted and the
+  # returned path will point to a temporary location of the decrypted file.
+  #
+  # To get the path of the encrypted file pass :decrypt => false, as an option.
   #
   # === Parameters:
   #
   # fid (String)::
   #   File ID which was generated by a previous #add operation.
   #
+  # options (Hash)::
+  #   :decrypt (true,false) In encryption mode don't decrypt, but return the encrypted file's path. Defaults to +true+.
+  #
   # === Return Value:
   #
-  # :: *String*, absolute path of the file in the pool.
-  def self.path fid
+  # :: *String*, absolute path of the file in the pool or to temporary location if it was decrypted.
+  def self.path fid, options={}
+    options[:decrypt] = true unless options[:decrypt] == false
+
     raise InvalidFileId unless valid?(fid)
-    id2dir(fid) + "/#{fid}"
+
+    # file present in pool?
+    if File.file?(id2dir_secured(fid) + "/#{fid}")
+      # present in secured tree
+      if @@crypted_mode
+        if options[:decrypt]
+          # return path of decrypted file (tmp path)
+          decrypt id2dir_secured(fid) + "/#{fid}"
+        else
+          id2dir_secured(fid) + "/#{fid}"
+        end
+      else
+        id2dir_secured(fid) + "/#{fid}"
+      end
+    elsif File.file?(id2dir(fid) + "/#{fid}")
+      # present in plain tree
+      id2dir(fid) + "/#{fid}"
+    else
+      # not present
+      if @@crypted_mode
+        id2dir_secured(fid) + "/#{fid}"
+      else
+        id2dir(fid) + "/#{fid}"
+      end
+    end
   end
 
   #
@@ -166,7 +141,7 @@ module FilePool
   # fid (String)::
   #   File ID which was generated by a previous #add operation.
   def self.remove! fid
-    FileUtils.rm path(fid)
+    FileUtils.rm path(fid, :decrypt => false)
   end
 
   #
@@ -204,7 +179,8 @@ module FilePool
   #     Time and Date of last add operation
 
   def self.stat
-    all_files = Dir.glob("#{root}/*/*/*/*")
+    all_files = Dir.glob("#{root}_secured/*/*/*/*")
+    all_files << Dir.glob("#{root}/*/*/*/*")
     all_stats = all_files.map{|f| File.stat(f) }
 
     {
@@ -227,6 +203,11 @@ module FilePool
     "#{root}/#{fid[0,1]}/#{fid[1,1]}/#{fid[2,1]}"
   end
 
+  # secured path from fid without file name
+  def self.id2dir_secured fid
+    "#{root}_secured/#{fid[0,1]}/#{fid[1,1]}/#{fid[2,1]}"
+  end
+
   # return a new UUID type 4 (random) as String
   def self.uuid
     UUIDTools::UUID.random_create.to_s
@@ -250,4 +231,110 @@ module FilePool
     (sortedarr[medpt1] + sortedarr[medpt2]).to_f / 2
   end
 
+  #
+  # Crypt a file and store the result in the temp.
+  #
+  # Returns the path to the crypted file.
+  #
+  # === Parameters:
+  #
+  # path (String)::
+  #   path of the file to crypt.
+  #
+  # === Return Value:
+  #
+  # :: *String*Path and name of the crypted file.
+  def self.crypt path
+    # Crypt the file in the temp folder and copy after
+    cipher = create_cipher
+    result = Tempfile.new uuid
+    crypted_content = cipher.update(File.read(path))
+    crypted_content << cipher.final
+    result.write crypted_content
+    result.close
+    result.path
+  end
+
+  #
+  # Decrypt a file and give a path to it.
+  #
+  # Returns the path to the decrypted file.
+  #
+  # === Parameters:
+  #
+  # path (String)::
+  #   path of the file to decrypt.
+  #
+  # === Return Value:
+  #
+  # :: *String*Path and name of the crypted file.
+  def self.decrypt path
+    decipher = create_decipher
+    # Now decrypt the data:
+    decrypted_content = decipher.update(File.read(path))
+    decrypted_content << decipher.final
+    # Put it in a temp file
+    output = Tempfile.new uuid
+    output.write decrypted_content
+    output.open
+    output.path
+  end
+
+  #
+  # Creates a cipher to encrypt data.
+  #
+  # Returns the cipher.
+  #
+  # === Return Value:
+  #
+  # :: *Openssl*Cipher object.
+  def self.create_cipher
+    cipher = OpenSSL::Cipher::AES.new(256, :CBC)
+    cipher.encrypt
+    cipher.key = @@key
+    cipher.iv  = @@iv
+    cipher
+  end
+
+  #
+  # Creates a decipher to decrypt data.
+  #
+  # Returns the decipher.
+  #
+  # === Return Value:
+  #
+  # :: *Openssl*Cipher object
+  def self.create_decipher
+    decipher = OpenSSL::Cipher::AES.new(256, :CBC)
+    decipher.decrypt
+    decipher.key = @@key
+    decipher.iv  = @@iv
+    decipher
+  end
+
+  #
+  # Retrieves configuration from config file or creates
+  # a new one in case there's none available.
+  #
+  def self.configure config_file
+    unless config_file.nil?
+      @@crypted_mode = true
+      begin
+        config = YAML.load_file(config_file)
+        @@iv  = config[:iv]
+        @@key = config[:key]
+      rescue Errno::ENOENT
+        cipher = OpenSSL::Cipher::AES.new(256, :CBC)
+        @@iv  = cipher.random_iv
+        @@key = cipher.random_key
+        cipher.key = @@key
+        cfg = File.open(config_file, 'w')
+        cfg.write({:iv => @@iv, :key => @@key}.to_yaml)
+        cfg.close
+        File.chmod(0400, config_file)
+      rescue => other_error
+        raise "FilePool: Could not load secrets from #{config_file}: #{other_error}"
+      end
+    end
+  end
 end

data/test/test_file_pool.rb

@@ -1,5 +1,8 @@
 require 'rubygems'
-require 'shoulda'
+require 'bundler/setup'
+
+require 'test/unit'
+require 'shoulda-context'
 require 'file_pool'
 
 class FilePoolTest < Test::Unit::TestCase
@@ -21,7 +24,7 @@ class FilePoolTest < Test::Unit::TestCase
       assert UUIDTools::UUID.parse(fid).valid?
 
       md5_orig = Digest::MD5.hexdigest(File.open(@test_dir+"/a").read)
-      md5_pooled = Digest::MD5.hexdigest(File.open(@pool_root + "/#{fid[0,1]}/#{fid[1,1]}/#{fid[2,1]}/#{fid}").read)
+      md5_pooled = Digest::MD5.hexdigest(File.open(FilePool.path(fid)).read)
 
       assert_equal md5_orig, md5_pooled
     end

data/test/test_file_pool_encryption.rb

@@ -0,0 +1,100 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'test/unit'
+require 'shoulda-context'
+require 'file_pool'
+
+class FilePoolEncryptionTest < Test::Unit::TestCase
+
+  def setup
+    @test_dir = "#{File.dirname(__FILE__)}/files"
+    @pool_root = "#{File.dirname(__FILE__)}/fp_root"
+    @file_pool_config = "#{File.dirname(__FILE__)}/file_pool_cfg.yml"
+    FilePool.setup @pool_root, :secrets_file => @file_pool_config
+  end
+
+  def teardown
+    FileUtils.rm_r(Dir.glob @pool_root+"/*")
+    FileUtils.rm_r(Dir.glob "#{@pool_root}_secured/*")
+    FileUtils.rm_r(Dir.glob @file_pool_config)
+  end
+
+  context "File Pool" do
+    should "store encrypted files" do
+      fid = FilePool.add(@test_dir+"/a")
+
+      assert UUIDTools::UUID.parse(fid).valid?
+
+      md5_orig = Digest::MD5.hexdigest(File.open(@test_dir+"/a").read)
+      md5_pooled = Digest::MD5.hexdigest(File.open(FilePool.path(fid)).read)
+
+      assert_equal md5_orig, md5_pooled
+    end
+
+    should "return path from stored encrypted files is in the tmp folder" do
+
+      fida = FilePool.add(@test_dir+"/a")
+      assert UUIDTools::UUID.parse(fida).valid?
+
+      fidb = FilePool.add(@test_dir+"/b")
+      assert UUIDTools::UUID.parse(fidb).valid?
+
+      fidc = FilePool.add(@test_dir+"/c")
+      assert UUIDTools::UUID.parse(fidc).valid?
+
+      fidd = FilePool.add!(@test_dir+"/d")
+      assert UUIDTools::UUID.parse(fidd).valid?
+
+      assert_equal Digest::MD5.hexdigest(File.open(@test_dir+"/a").read),
+                     Digest::MD5.hexdigest(File.open(FilePool.path(fida)).read)
+      assert_equal Digest::MD5.hexdigest(File.open(@test_dir+"/b").read),
+                     Digest::MD5.hexdigest(File.open(FilePool.path(fidb)).read)
+      assert_equal Digest::MD5.hexdigest(File.open(@test_dir+"/c").read),
+                     Digest::MD5.hexdigest(File.open(FilePool.path(fidc)).read)
+      assert_equal Digest::MD5.hexdigest(File.open(@test_dir+"/d").read),
+                     Digest::MD5.hexdigest(File.open(FilePool.path(fidd)).read)
+
+      assert_equal Dir.tmpdir, File.dirname(FilePool.path(fida))
+      assert_equal Dir.tmpdir, File.dirname(FilePool.path(fidb))
+      assert_equal Dir.tmpdir, File.dirname(FilePool.path(fidc))
+      assert_equal Dir.tmpdir,File.dirname( FilePool.path(fidd))
+    end
+
+    should "remove files from encrypted pool" do
+
+      fidb = FilePool.add(@test_dir+"/b")
+      fidc = FilePool.add!(@test_dir+"/c")
+      fidd = FilePool.add!(@test_dir+"/d")
+
+      path_c = FilePool.path(fidc, :decrypt => false)
+      FilePool.remove(fidc)
+
+      assert !File.exist?(path_c)
+      assert File.exist?(FilePool.path(fidb, :decrypt => false))
+      assert File.exist?(FilePool.path(fidd, :decrypt => false))
+
+    end
+
+    should "throw exceptions when using add! and remove! on failure in encrypted mode" do
+      assert_raises(FilePool::InvalidFileId) do
+        FilePool.remove!("invalid-id")
+      end
+
+      assert_raises(Errno::ENOENT) do
+        FilePool.remove!("61e9b2d1-1738-440d-9b3d-e3c64876f2b0")
+      end
+
+      assert_raises(Errno::ENOENT) do
+        FilePool.add!("/not/here/foo.png")
+      end
+
+    end
+
+    should "not throw exceptions when using add and remove on failure in encrypted mode" do
+      assert !FilePool.remove("invalid-id")
+      assert !FilePool.remove("61e9b2d1-1738-440d-9b3d-e3c64876f2b0")
+      assert !FilePool.add("/not/here/foo.png")
+    end
+  end
+end

metadata

@@ -1,103 +1,69 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: file_pool
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
-  segments: 
-  - 0
-  - 2
-  - 0
-  version: 0.2.0
+version: !ruby/object:Gem::Version
+  version: 0.3.0
 platform: ruby
-authors: 
-- "robokopp (Robert Anni\xC3\xA9s)"
+authors:
+- robokopp (Robert Anniés)
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-09-17 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: shoulda
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+date: 2017-02-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: uuidtools
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 2
-        - 1
-        - 2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
         version: 2.1.2
   type: :runtime
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
 description: |
   FilePool helps to manage a large number of files in a Ruby
   project. It takes care of the storage of files in a balanced directory
   tree and generates unique identifiers for all files.
-
-email: 
+email:
 - robokopp@fernwerk.net
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
+- README.md
+files:
 - README.md
-files: 
 - lib/file_pool.rb
 - lib/file_pool/version.rb
-- README.md
 - test/test_file_pool.rb
+- test/test_file_pool_encryption.rb
 homepage: https://github.com/robokopp/file_pool
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Manage a large number files in a pool
-test_files: 
+test_files:
 - test/test_file_pool.rb
+- test/test_file_pool_encryption.rb
 has_rdoc: