massive 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8833bbf616ef0395fd5e89b3911aacb5af0418e
-  data.tar.gz: 9a23e1181998a64e1bfc3aa43fa71f136f253967
+  metadata.gz: 6add55169f9bbe782d2ee00b8a033a21aae4d9cb
+  data.tar.gz: 3e7ef6017201515489a3e8f04fb41b8669b701cd
 SHA512:
-  metadata.gz: dc17619c526974f5e6d762e6d55756b55d39b234c02edc95c4b7b8ef6365e2ca0109514e921bd4d89887849d2cd725bf5eaac2bb304914cb2b026a6d8a6c5638
-  data.tar.gz: 1948810ce93bb7c160833cc7b2a10ab38386afd41735f59f72df185a050db990fbf5acc2c3bdeab598bf2b1d8884290b8d71c377a2919ec95dd41458adb318a7
+  metadata.gz: a2470c8e5230e4b2f4e12a47e93722e16da720971c15c67179aa8f053a1907c1322309f19fdef0572c43211d0b56f62a7d159d6a0c25422ab61cd339deab9a18
+  data.tar.gz: 2e48d557a63fa84092f83acabca19ba70b3d21559bbb0f96f25dee3ee97c16b96e0daac07ccf26c40377046787184d8c5810ec1f00b9831ad54f100073a73b5f

data/Gemfile

@@ -4,8 +4,6 @@ gemspec
 
 gem 'rake'
 
-gem 'fog'
-
 group(:development) do
   gem 'debugger'
   gem 'guard-rspec', require: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    massive (0.2.0)
+    massive (0.3.0)
       active_model_serializers
       file_processor (= 0.2.0)
       mongoid (~> 4.0.0.beta)
@@ -37,26 +37,8 @@ GEM
     debugger-ruby_core_source (1.3.4)
     diff-lcs (1.2.5)
     docile (1.1.3)
-    excon (0.33.0)
     ffi (1.9.3)
     file_processor (0.2.0)
-    fog (1.22.0)
-      fog-brightbox
-      fog-core (~> 1.21, >= 1.21.1)
-      fog-json
-      nokogiri (~> 1.5, >= 1.5.11)
-    fog-brightbox (0.0.2)
-      fog-core
-      fog-json
-    fog-core (1.22.0)
-      builder
-      excon (~> 0.33)
-      formatador (~> 0.2)
-      mime-types
-      net-scp (~> 1.1)
-      net-ssh (>= 2.1.3)
-    fog-json (1.0.0)
-      multi_json (~> 1.0)
     formatador (0.2.4)
     guard (2.6.1)
       formatador (>= 0.2.4)
@@ -75,8 +57,6 @@ GEM
       rb-inotify (>= 0.9)
     lumberjack (1.0.5)
     method_source (0.8.2)
-    mime-types (2.2)
-    mini_portile (0.6.0)
     minitest (5.3.3)
     mongoid (4.0.0.beta1)
       activemodel (>= 4.0.0)
@@ -89,11 +69,6 @@ GEM
       connection_pool (~> 2.0)
       optionable (~> 0.2.0)
     multi_json (1.10.0)
-    net-scp (1.2.1)
-      net-ssh (>= 2.6.5)
-    net-ssh (2.9.1)
-    nokogiri (1.6.2.1)
-      mini_portile (= 0.6.0)
     optionable (0.2.0)
     origin (2.1.1)
     pry (0.9.12.6)
@@ -150,7 +125,6 @@ PLATFORMS
 DEPENDENCIES
   database_cleaner
   debugger
-  fog
   guard-rspec
   massive!
   rake

data/README.md

@@ -168,23 +168,21 @@ Notice that we didn't had to specify how the step would calculate the total coun
 
 We also didn't have to specify how the job would iterate through each row, it is already defined. We just get a CSV::Row, which will be a Hash-like structure where the header of the CSV is the key, so we can just pass it to `User.create`. Of course this is a simple example, you should protect the attributes, or even pass only the ones you want from the CSV.
 
-The `Massive::File` has support for [Fog::Storage][http://fog.io/storage/]. To use it yoy must define the `fog_credentials` and optionally the `fog_directory` and `fog_authenticated_url_expiration`:
+The `Massive::File` has support for getting files from Amazon AWS S3 service. To use it you must define the `storage_config`:
 
 ```ruby
-  Massive.fog_credentials = {
-    provider: 'AWS',
-    aws_access_key_id: 'INSERT-YOUR-AWS-KEY-HERE',
-    aws_secret_access_key: 'INSERT-YOUR-AWS-SECRET-HERE'
+  Massive.storage_config = {
+    key: 'INSERT-YOUR-AWS-KEY-HERE',
+    secret: 'INSERT-YOUR-AWS-SECRET-HERE'
+    directory: 'your-bucket-here',              # defaults to 'massive'
+    expiration: 30.minutes                      # defaults to 1.hour
   }
-
-  Massive.fog_directory = 'your-bucket-here' # defaults to 'massive'
-  Massive.fog_authenticated_url_expiration = 30.minutes # defaults to 1.hour
 ```
 
 Then set the `filename` field when creating the `Massive::File` instead of setting its `url`. Notice that the filename should point to the full path within the bucket, not just the actual filename.
 
 ```ruby
-  process = Massive::FileProcess.new(file_attributes: { filename: '/path/to/my/file.csv' })
+  process = Massive::FileProcess.new(file_attributes: { filename: 'path/to/my/file.csv' })
 ```
 
 ## Contributing

data/lib/massive.rb

@@ -23,38 +23,32 @@ module Massive
   autoload :ProcessSerializer, 'massive/process_serializer'
   autoload :StepSerializer,    'massive/step_serializer'
 
+  module Authenticators
+    autoload :S3,              'massive/authenticators/s3'
+  end
+
   class Cancelled < StandardError; end
 
   def self.redis
     @redis ||= Resque.redis
   end
 
-  def self.fog_credentials
-    @fog_credentials
-  end
-
-  def self.fog_credentials=(values=nil)
-    @fog_credentials = values
-  end
-
-  def self.fog_authenticated_url_expiration
-    @fog_authenticated_url_expiration
-  end
-
-  def self.fog_authenticated_url_expiration=(value=nil)
-    @fog_authenticated_url_expiration = value
-  end
-
-  def self.fog_directory
-    @fog_directory
+  def self.storage_config
+    @storage_config
   end
 
-  def self.fog_directory=(directory=nil)
-    @fog_directory = directory
+  def self.storage_config=(value)
+    @storage_config ||= {}
+    @storage_config.merge!(value)
   end
 
-  self.fog_directory = 'massive'
-  self.fog_authenticated_url_expiration = 1 * 60 * 60
+  self.storage_config = {
+    directory: 'massive',
+    provider: Massive::Authenticators::S3,
+    key: nil,
+    secret: nil,
+    expiration: 1 * 60 * 60 # 1 hour
+  }
 end
 
 require "resque"

data/lib/massive/authenticators/s3.rb

@@ -0,0 +1,37 @@
+module Massive
+  module Authenticators
+    class S3
+      def initialize(filename)
+        @filename = filename
+      end
+
+      def url
+        "https://#{Massive.storage_config[:directory]}.s3.amazonaws.com/#{@filename}#{authentication_params}"
+      end
+
+      private
+
+      def authentication_params
+        if Massive.storage_config[:key] && Massive.storage_config[:secret]
+          "?Expires=#{expiration}&AWSAccessKeyId=#{Massive.storage_config[:key]}&Signature=#{signature}"
+        end
+      end
+
+      def expiration
+        @expiration ||= Time.now.to_i + Massive.storage_config[:expiration]
+      end
+
+      def signature
+        CGI.escape(
+          Base64.encode64(
+            OpenSSL::HMAC.digest(
+              OpenSSL::Digest.new('sha1'),
+              Massive.storage_config[:secret],
+              "GET\n\n\n#{expiration}\n/#{Massive.storage_config[:directory]}/#{@filename}".encode("UTF-8")
+            )
+          ).gsub("\n", "")
+        )
+      end
+    end
+  end
+end

data/lib/massive/file.rb

@@ -40,7 +40,7 @@ module Massive
     end
 
     def url
-      read_attribute(:url).presence || authenticated_url
+      read_attribute(:url).presence || authenticator.url
     end
 
     private
@@ -59,26 +59,8 @@ module Massive
       }
     end
 
-    def authenticated_url
-      if can_use_fog?
-        fog_file.respond_to?(:url) ? fog_file.url(Time.current.to_i + Massive.fog_authenticated_url_expiration) : fog_file.public_url
-      end
-    end
-
-    def can_use_fog?
-      filename && Massive.fog_credentials.present?
-    end
-
-    def fog_connection
-      @fog_connection ||= Fog::Storage.new(Massive.fog_credentials)
-    end
-
-    def fog_directory
-      @fog_directory ||= fog_connection.directories.get(Massive.fog_directory)
-    end
-
-    def fog_file
-      @fog_file ||= fog_directory.files.get(filename)
+    def authenticator
+      @authenticator ||= Massive.storage_config[:provider].new(filename)
     end
   end
 end

data/lib/massive/version.rb

@@ -1,3 +1,3 @@
 module Massive
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/models/massive/authenticators/s3_spec.rb

@@ -0,0 +1,119 @@
+require "spec_helper"
+
+describe Massive::Authenticators::S3 do
+  let(:filename) { 'some/path/my-filename.png' }
+  subject(:authenticator) { described_class.new(filename) }
+
+  after do
+    Massive.storage_config = {
+      key: nil,
+      secret: nil
+    }
+  end
+
+  context 'when no key/secret are set' do
+    it "returns the given url without any authentication params" do
+      expect(authenticator.url).to eq("https://#{Massive.storage_config[:directory]}.s3.amazonaws.com/#{filename}")
+    end
+  end
+
+  context 'when key/secret are set' do
+    before do
+      Massive.storage_config[:key] = 'some-key'
+      Massive.storage_config[:secret] = 'some-secret'
+      Time.stub(:now).and_return(Time.parse("2014-05-15T13:25:45Z"))
+    end
+
+    def parsed_query(url)
+      CGI.parse(URI.parse(url).query || '').with_indifferent_access
+    end
+
+    it "returns a url pointing to the proper bucket" do
+      expect(URI.parse(authenticator.url).host).to eq("#{Massive.storage_config[:directory]}.s3.amazonaws.com")
+    end
+
+    it "returns a url using https" do
+      expect(URI.parse(authenticator.url).scheme).to eq("https")
+    end
+
+    it "returns a url with the filename as path" do
+      expect(URI.parse(authenticator.url).path).to eq("/#{filename}")
+    end
+
+    it "returns a url with the expiration as a query string parameter using a timestamped format" do
+      now = Time.now.tap { |now| Time.stub(:now).and_return(now) }
+
+      expect(parsed_query(authenticator.url)[:Expires].first.to_i).to eq(now.to_i + Massive.storage_config[:expiration])
+    end
+
+    it "returns a url with the AWSAccessKeyId as a query string parameter using the configured key" do
+      expect(parsed_query(authenticator.url)[:AWSAccessKeyId].first).to eq(Massive.storage_config[:key])
+    end
+
+    it "returns a url with the Signature as a query string parameter properly signing the GET request" do
+      expect(parsed_query(authenticator.url)[:Signature].first).to eq("FD38leXqSdFYGYrAXgNF8cX98os=")
+    end
+
+    context "when changing the expiration" do
+      before do
+        Time.stub(:now).and_return(Time.parse("2009-08-01T20:03:27Z"))
+      end
+
+      it "returns a url with the Signature properly signing based on the current time" do
+        expect(parsed_query(authenticator.url)[:Signature].first).to eq("ehM0n71BUPduV9WwWE73PIMmQYM=")
+      end
+    end
+
+    context "when changing the secret" do
+      before do
+        Massive.storage_config[:secret] = 'other-secret'
+      end
+
+      it "returns a url with the Signature based the configured secret" do
+        expect(parsed_query(authenticator.url)[:Signature].first).to eq("mE6pvT9pRLOUcufs+fX45vbbATQ=")
+      end
+    end
+
+    context "when changing the key" do
+      before do
+        Massive.storage_config[:key] = 'other-key'
+      end
+
+      it "returns a url with the AWSAccessKeyId as a query string parameter using the configured key" do
+        expect(parsed_query(authenticator.url)[:AWSAccessKeyId].first).to eq(Massive.storage_config[:key])
+      end
+    end
+
+    context "when changing the directory" do
+      before do
+        Massive.storage_config[:directory] = 'other-directory'
+      end
+
+      it "returns a url with the expiration as a query string parameter using a timestamped format" do
+        now = Time.now.tap { |now| Time.stub(:now).and_return(now) }
+
+        expect(parsed_query(authenticator.url)[:Expires].first.to_i).to eq(now.to_i + Massive.storage_config[:expiration])
+      end
+
+      it "returns a url pointing to the proper bucket" do
+        expect(URI.parse(authenticator.url).host).to eq("#{Massive.storage_config[:directory]}.s3.amazonaws.com")
+      end
+
+      it "returns a url with the Signature based the configured directory" do
+        expect(parsed_query(authenticator.url)[:Signature].first).to eq("f37FDYE7lcewcpApQhFYBEUQjhs=")
+      end
+    end
+
+    context "when changing the filename" do
+      let(:filename) { 'some/other/file.txt' }
+
+      it "returns a url with the filename as path" do
+        expect(URI.parse(authenticator.url).path).to eq("/#{filename}")
+      end
+
+      it "returns a url with the Signature based the configured directory" do
+        expect(parsed_query(authenticator.url)[:Signature].first).to eq("fQCrfdk1FhSRZnecyZ2+Jye2HUY=")
+      end
+    end
+  end
+end

data/spec/models/massive/file_spec.rb

@@ -18,7 +18,11 @@ describe Massive::File do
 
   subject(:file)  { process.file = Massive::File.new(url: url, encoding: encoding, col_sep: col_sep) }
 
-  before { FileProcessor::CSV.stub(:new).with(file.url, expected_options).and_return(processor) }
+  def stub_processor
+    FileProcessor::CSV.stub(:new).with(file.url, expected_options).and_return(processor)
+  end
+
+  before { stub_processor }
 
   describe "#processor" do
     it "creates a new instance of the CSV file processor, enabling headers but without encoding and separator" do
@@ -50,28 +54,20 @@ describe Massive::File do
       end
     end
 
-    describe "when using Fog" do
-      let(:filename) { 'my-file.txt' }
-      let(:fog_connection) { double(Fog::Storage) }
-      let(:fog_directory) { double('Directory') }
-      let(:fog_file) { double('File') }
-      let(:authenticated_url) { 'http://my-auth.url.com' }
+    describe "when passing a filename" do
+      let(:filename) { 'my/path/my-file.txt' }
+      let(:url) { 'http://my-auth.url.com' }
+      let(:provider) { double(Massive::Authenticators::S3, url: url) }
 
-      subject(:file)  { Massive::File.new(filename: filename, encoding: encoding, col_sep: col_sep) }
-
-      before do
-        Massive.fog_credentials = { provider: 'AWS', aws_access_key_id: 'some-key', aws_secret_access_key: 'some-secret' }
-        Massive.fog_authenticated_url_expiration = 1.hour
-        Massive.fog_directory = 'my-bucket'
+      subject(:file) { Massive::File.new(filename: filename, encoding: encoding, col_sep: col_sep) }
 
-        Fog::Storage.stub(:new).with(Massive.fog_credentials).and_return(fog_connection)
-        fog_connection.stub_chain(:directories, :get).with(Massive.fog_directory).and_return(fog_directory)
-        fog_directory.stub_chain(:files, :get).with(filename).and_return(fog_file)
-        fog_file.stub(:url).with(Time.current.to_i + Massive.fog_authenticated_url_expiration).and_return(authenticated_url)
+      def stub_processor
+        Massive.storage_config[:provider].stub(:new).with(filename).and_return(provider)
+        FileProcessor::CSV.stub(:new).with(file.url, expected_options).and_return(processor)
       end
 
-      it "creates a new instance of the CSV file processor, pointing its URL to the authenticated fog url" do
-        FileProcessor::CSV.should_receive(:new).with(authenticated_url, expected_options).and_return(processor)
+      it "creates a new instance of the CSV file processor, pointing its URL to the authenticator provider url" do
+        FileProcessor::CSV.should_receive(:new).with(url, expected_options).and_return(processor)
         file.processor.should eq(processor)
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: massive
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Vicente Mundim
@@ -86,6 +86,7 @@ files:
 - README.md
 - Rakefile
 - lib/massive.rb
+- lib/massive/authenticators/s3.rb
 - lib/massive/cancelling.rb
 - lib/massive/file.rb
 - lib/massive/file_job.rb
@@ -109,6 +110,7 @@ files:
 - massive.gemspec
 - spec/fixtures/custom_job.rb
 - spec/fixtures/custom_step.rb
+- spec/models/massive/authenticators/s3_spec.rb
 - spec/models/massive/cancelling_spec.rb
 - spec/models/massive/file_job_spec.rb
 - spec/models/massive/file_spec.rb
@@ -155,6 +157,7 @@ summary: Parallelize processing of large files and/or data using Resque, Redis a
 test_files:
 - spec/fixtures/custom_job.rb
 - spec/fixtures/custom_step.rb
+- spec/models/massive/authenticators/s3_spec.rb
 - spec/models/massive/cancelling_spec.rb
 - spec/models/massive/file_job_spec.rb
 - spec/models/massive/file_spec.rb