massive 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6add55169f9bbe782d2ee00b8a033a21aae4d9cb
-  data.tar.gz: 3e7ef6017201515489a3e8f04fb41b8669b701cd
+  metadata.gz: d0ad236ec732665de7fcf03668e8ce4675da4344
+  data.tar.gz: acba7ff61e45ff129881fa0d86b49480eaa85912
 SHA512:
-  metadata.gz: a2470c8e5230e4b2f4e12a47e93722e16da720971c15c67179aa8f053a1907c1322309f19fdef0572c43211d0b56f62a7d159d6a0c25422ab61cd339deab9a18
-  data.tar.gz: 2e48d557a63fa84092f83acabca19ba70b3d21559bbb0f96f25dee3ee97c16b96e0daac07ccf26c40377046787184d8c5810ec1f00b9831ad54f100073a73b5f
+  metadata.gz: 1c922037b0dc37a497a6cced3880e88436912e6d3c8ff35ab8fc80c18f3375fbf4262551ea66ad82b7c5d065d0b20cb168c752400c28382df774027423687fe3
+  data.tar.gz: 122c02cc041df10b737ac8d6457d2bda985bb2930037672df0da4febe828648a9029829c08161597e9a5def1442e3f176062856fb15f52b7d4b72965f3d3c3b7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    massive (0.3.0)
+    massive (0.4.0)
       active_model_serializers
       file_processor (= 0.2.0)
       mongoid (~> 4.0.0.beta)

data/lib/massive.rb

@@ -22,6 +22,7 @@ module Massive
 
   autoload :ProcessSerializer, 'massive/process_serializer'
   autoload :StepSerializer,    'massive/step_serializer'
+  autoload :FileSerializer,    'massive/file_serializer'
 
   module Authenticators
     autoload :S3,              'massive/authenticators/s3'
@@ -33,6 +34,14 @@ module Massive
     @redis ||= Resque.redis
   end
 
+  def self.split_jobs
+    @split_jobs
+  end
+
+  def self.split_jobs=(value)
+    @split_jobs = value
+  end
+
   def self.storage_config
     @storage_config
   end

data/lib/massive/file_serializer.rb

@@ -0,0 +1,9 @@
+module Massive
+  class FileSerializer < ActiveModel::Serializer
+    attributes :id, :url, :encoding, :col_sep, :total_count, :use_headers, :headers, :sample_data
+
+    def id
+      object.id.to_s
+    end
+  end
+end

data/lib/massive/job.rb

@@ -26,7 +26,26 @@ module Massive
     end
 
     def self.queue
-      :massive_job
+      if split_jobs
+        :"#{queue_prefix}_#{Kernel.rand(split_jobs) + 1}"
+      else
+        queue_prefix
+      end
+    end
+
+    def self.queue_prefix(value=nil)
+      @queue_prefix = value if !value.nil?
+      @queue_prefix || :massive_job
+    end
+
+    def self.split_jobs(value=nil)
+      @split_jobs = value if !value.nil?
+      @split_jobs.nil? ? Massive.split_jobs : @split_jobs
+    end
+
+    def self.cancel_when_failed(value=nil)
+      @cancel_when_failed = value if !value.nil?
+      @cancel_when_failed
     end
 
     def enqueue
@@ -101,6 +120,7 @@ module Massive
       step.save
       notify(:failed)
 
+      process.cancel if self.class.cancel_when_failed
       raise e
     end
 

data/lib/massive/process.rb

@@ -5,7 +5,7 @@ module Massive
 
     field :cancelled_at, type: Time
 
-    embeds_many :steps, class_name: 'Massive::Step'
+    has_many :steps, class_name: 'Massive::Step', dependent: :destroy
 
     def self.find_step(process_id, step_id)
       find(process_id).steps.find(step_id)

data/lib/massive/process_serializer.rb

@@ -3,10 +3,15 @@ module Massive
     attributes :id, :created_at, :updated_at, :processed_percentage
     attribute :completed?, key: :completed
 
+    has_one :file
     has_many :steps
 
     def id
       object.id.to_s
     end
+
+    def include_file?
+      object.respond_to?(:file)
+    end
   end
 end

data/lib/massive/step.rb

@@ -9,7 +9,7 @@ module Massive
     include Massive::Locking
     include Massive::Notifications
 
-    embedded_in :process, class_name: 'Massive::Process'
+    belongs_to :process, class_name: 'Massive::Process'
     embeds_many :jobs,    class_name: 'Massive::Job'
 
     field :total_count,  type: Integer

data/lib/massive/version.rb

@@ -1,3 +1,3 @@
 module Massive
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/models/massive/file_serializer_spec.rb

@@ -0,0 +1,40 @@
+require "spec_helper"
+
+describe Massive::FileSerializer do
+  let(:headers) { ['name', 'description', 'price'] }
+  let(:sample_data) { [['Some name', 'Some desc', 1234], ['Other name', 'Other desc', 5678]] }
+  let(:file) { Massive::File.new(url: 'http://some.url.com', encoding: 'utf8', col_sep: ';', total_count: 1234, use_headers: true, headers: headers, sample_data: sample_data) }
+  subject(:serialized) { described_class.new(file).as_json(root: false) }
+
+  it "serializes file id as string" do
+    serialized[:id].should eq(file.id.to_s)
+  end
+
+  it "serializes url" do
+    serialized[:url].should eq(file.url)
+  end
+
+  it "serializes encoding" do
+    serialized[:encoding].should eq(file.encoding)
+  end
+
+  it "serializes col_sep" do
+    serialized[:col_sep].should eq(file.col_sep)
+  end
+
+  it "serializes total_count" do
+    serialized[:total_count].should eq(file.total_count)
+  end
+
+  it "serializes use_headers" do
+    serialized[:use_headers].should eq(file.use_headers)
+  end
+
+  it "serializes headers" do
+    serialized[:headers].should eq(file.headers)
+  end
+
+  it "serializes sample_data" do
+    serialized[:sample_data].should eq(file.sample_data)
+  end
+end

data/spec/models/massive/job_spec.rb

@@ -8,6 +8,8 @@ describe Massive::Job do
   let(:step) { process.steps.build }
   subject(:job) { step.jobs.build }
 
+  before { job.stub(:process).and_return(process) }
+
   describe ".perform" do
     before do
       Massive::Process.stub(:find_job).with(process.id, step.id, job.id).and_return(job)
@@ -20,9 +22,57 @@ describe Massive::Job do
   end
 
   describe ".queue" do
+    after { Massive::Job.queue_prefix(:massive_job) }
+
     it "should be massive_job" do
       Massive::Job.queue.should eq(:massive_job)
     end
+
+    it "should use queue_prefix" do
+      Massive::Job.queue_prefix(:my_job_queue)
+      Massive::Job.queue.should eq(:my_job_queue)
+    end
+
+    context "when Massive.split_jobs is set to 100" do
+      before { Massive.split_jobs = 100 }
+      after  { Massive.split_jobs = false }
+
+      it "should be massive_job_XXX where XXX is a random number" do
+        values = 10000.times.inject({}) do |memo, index|
+          match = Massive::Job.queue.to_s.match(/massive_job_(\d+)/)
+          memo[match[1].to_i] ||= 0
+          memo[match[1].to_i] += 1
+          memo
+        end
+
+        (1..100).each do |key|
+          expect(values.keys.sort).to include(key)
+        end
+      end
+
+      it "should use the queue prefix" do
+        Massive::Job.queue_prefix(:my_job_queue)
+        expect(Massive::Job.queue.to_s).to start_with('my_job_queue')
+      end
+
+      context "when Job split_jobs is set to 10" do
+        before { Massive::Job.split_jobs 200 }
+        after  { Massive::Job.split_jobs false }
+
+        it "should be massive_job_XXX where XXX is a random number between 1 and 10" do
+          values = 10000.times.inject({}) do |memo, index|
+            match = Massive::Job.queue.to_s.match(/massive_job_(\d+)/)
+            memo[match[1].to_i] ||= 0
+            memo[match[1].to_i] += 1
+            memo
+          end
+
+          (1..200).each do |key|
+            expect(values.keys.sort).to include(key)
+          end
+        end
+      end
+    end
   end
 
   describe "#enqueue" do
@@ -219,6 +269,20 @@ describe Massive::Job do
         rescue StandardError, SignalException
         end
       end
+
+      context "when it is configured to cancel when failed" do
+        before { Massive::Job.cancel_when_failed true }
+        after  { Massive::Job.cancel_when_failed false }
+
+        it "cancels the process" do
+          expect(process).to receive(:cancel)
+
+          begin
+            job.work
+          rescue StandardError, SignalException
+          end
+        end
+      end
     end
 
     context "when an error occurs" do

data/spec/models/massive/process_serializer_spec.rb

@@ -35,4 +35,18 @@ describe Massive::ProcessSerializer do
       serialized[:completed].should be_false
     end
   end
+
+  context "when it does not respond to file" do
+    it "does not serializes file" do
+      serialized[:file].should be_blank
+    end
+  end
+
+  context "when it responds to file" do
+    let(:process) { Massive::FileProcess.new }
+
+    it "serializes file" do
+      serialized[:file].should be_present
+    end
+  end
 end

data/spec/models/massive/process_spec.rb

@@ -1,81 +1,45 @@
 require "spec_helper"
 
 describe Massive::Process do
-  subject(:process) { Massive::Process.new }
+  subject(:process) { Massive::Process.create }
 
   describe "#enqueue_next" do
-    context "when there are steps" do
-      let!(:first_step) { process.steps.build }
-      let!(:second_step) { process.steps.build }
-      let!(:third_step) { process.steps.build }
+    context "when there is a next steps" do
+      let(:step) { process.steps.create }
 
       before do
-        first_step.stub(:reload).and_return(first_step)
-        second_step.stub(:reload).and_return(second_step)
-        third_step.stub(:reload).and_return(third_step)
+        process.stub(:next_step).and_return(step)
       end
 
-      context "and none of them are completed" do
-        it "enqueues the first step" do
-          first_step.should_receive(:enqueue)
-          process.enqueue_next
-        end
-
-        it "does not enqueue the other steps" do
-          second_step.should_not_receive(:enqueue)
-          third_step.should_not_receive(:enqueue)
-          process.enqueue_next
-        end
+      it "enqueues the step" do
+        step.should_receive(:enqueue)
+        process.enqueue_next
       end
+    end
 
-      context "and the first one is completed, but the second one is not" do
-        before { first_step.finished_at = Time.now }
-
-        it "does not enqueue the first step" do
-          first_step.should_not_receive(:enqueue)
-          process.enqueue_next
-        end
-
-        it "enqueues the second step" do
-          second_step.should_receive(:enqueue)
-          process.enqueue_next
-        end
-
-        it "does not enqueue the third step" do
-          third_step.should_not_receive(:enqueue)
-          process.enqueue_next
-        end
-      end
-
-      context "and the first one is enqueued" do
-        before { first_step.stub(:enqueued?).and_return(true) }
-
-        it "does not enqueue the next step" do
-          second_step.should_not_receive(:enqueue)
-          process.enqueue_next
-        end
+    context "when there is no next step" do
+      before do
+        process.stub(:next_step).and_return(nil)
       end
 
-      context "but all of them are completed" do
-        before do
-          process.steps.each do |step|
-            step.finished_at = Time.now
-          end
-        end
-
-        it "does not enqueue any of the steps" do
-          process.steps.each do |step|
-            step.should_not_receive(:enqueue)
-          end
-
+      it "does not raise error" do
+        expect {
           process.enqueue_next
-        end
+        }.to_not raise_error
       end
     end
   end
 
   describe "#next_step" do
-    let!(:step) { process.steps.build }
+    let!(:step) { process.steps.create }
+
+    before do
+      steps = double('Array')
+      process.stub(:steps).and_return(steps)
+      steps.stub(:not_completed).and_return(steps)
+      steps.stub(:not_started).and_return(steps)
+      steps.stub(:first).and_return(step)
+    end
 
     context "when the step is enqueued" do
       before { step.stub(:enqueued?).and_return(true) }
@@ -84,14 +48,14 @@ describe Massive::Process do
     end
 
     context "when the step is not enqueued" do
+      before { step.stub(:enqueued?).and_return(false) }
+
       its(:next_step) { should eq step }
     end
   end
 
   describe ".find_step" do
-    let!(:step) { process.steps.build }
-
-    before { process.save }
+    let!(:step) { process.steps.create }
 
     it "returns the step with id within the process" do
       Massive::Process.find_step(process.id, step.id).should eq(step)
@@ -99,10 +63,8 @@ describe Massive::Process do
   end
 
   describe ".find_job" do
-    let!(:step) { process.steps.build }
-    let!(:job)  { step.jobs.build }
-
-    before { process.save }
+    let!(:step) { process.steps.create }
+    let!(:job)  { step.jobs.create }
 
     it "returns the job with id within the step of the process" do
       Massive::Process.find_job(process.id, step.id, job.id).should eq(job)
@@ -110,8 +72,8 @@ describe Massive::Process do
   end
 
   describe "#processed_percentage" do
-    let(:step_1)  { process.steps.build(weight: 9) }
-    let(:step_2)  { process.steps.build            }
+    let(:step_1)  { process.steps.create(weight: 9) }
+    let(:step_2)  { process.steps.create            }
 
     context "when the process have not started" do
       before do
@@ -162,19 +124,19 @@ describe Massive::Process do
     end
 
     context "when the total weight of the steps is zero" do
-      let(:step_1)  { process.steps.build(weight: 0) }
-      let(:step_2)  { process.steps.build(weight: 0) }
+      let(:step_1)  { process.steps.create(weight: 0) }
+      let(:step_2)  { process.steps.create(weight: 0) }
 
       its(:processed_percentage) { should eq 0 }
     end
   end
 
   describe "#completed?" do
-    let!(:step_1)  { process.steps.build }
-    let!(:step_2)  { process.steps.build }
-
     before { process.save }
 
+    let!(:step_1)  { process.steps.create }
+    let!(:step_2)  { process.steps.create }
+
     context "when the steps are incompleted steps" do
       its(:completed?) { should be_false }
     end
@@ -190,8 +152,8 @@ describe Massive::Process do
   end
 
   describe "#failed?" do
-    let!(:step_1) { process.steps.build }
-    let!(:step_2) { process.steps.build }
+    let!(:step_1) { process.steps.create }
+    let!(:step_2) { process.steps.create }
 
     before { process.save }
 

data/spec/models/massive/step_spec.rb

@@ -7,6 +7,8 @@ describe Massive::Step do
   let(:process) { Massive::Process.new }
   subject(:step) { process.steps.build }
 
+  before { step.stub(:process).and_return(process) }
+
   describe ".perform" do
     before do
       Massive::Process.stub(:find_step).with(process.id, step.id).and_return(step)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: massive
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Vicente Mundim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-15 00:00:00.000000000 Z
+date: 2015-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -91,6 +91,7 @@ files:
 - lib/massive/file.rb
 - lib/massive/file_job.rb
 - lib/massive/file_process.rb
+- lib/massive/file_serializer.rb
 - lib/massive/file_step.rb
 - lib/massive/job.rb
 - lib/massive/locking.rb
@@ -113,6 +114,7 @@ files:
 - spec/models/massive/authenticators/s3_spec.rb
 - spec/models/massive/cancelling_spec.rb
 - spec/models/massive/file_job_spec.rb
+- spec/models/massive/file_serializer_spec.rb
 - spec/models/massive/file_spec.rb
 - spec/models/massive/file_step_spec.rb
 - spec/models/massive/job_spec.rb
@@ -160,6 +162,7 @@ test_files:
 - spec/models/massive/authenticators/s3_spec.rb
 - spec/models/massive/cancelling_spec.rb
 - spec/models/massive/file_job_spec.rb
+- spec/models/massive/file_serializer_spec.rb
 - spec/models/massive/file_spec.rb
 - spec/models/massive/file_step_spec.rb
 - spec/models/massive/job_spec.rb