rrrspec-server 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3dd5351ecddcebbc6fbc7f89e1102c06b02dcb21
-  data.tar.gz: 33b7df062b10d6f6d9099372c931d85663bb68bc
+  metadata.gz: 130e42b6ea51f45c286e7c21fe8aae49014eef81
+  data.tar.gz: 1563ac90134f3837f0c705a6cce802c6b854c478
 SHA512:
-  metadata.gz: c5bd9cf986be28aa07d504e771a2be2fef1260c7e0a76f9e6bf91b405c81331b51f0d5c7ca551b9080b4383d22de9391c7551e8343b03c8392a75130e7bb204b
-  data.tar.gz: b85287965ddcf34ee7f64e1dc857bc2106257fac8b927ea2bd1862249a5f935473a4bfe7a690aa013ab9d435b4272e4177e4da626e187790b61423b5403ef9a3
+  metadata.gz: 8cccda69622490863225f289edb743f1182ded6beb1e32cad0b2aaefe0b3dd7ea65070254496c78b9a71eccc8bb6e3191be9f356af68393c2594fe44ba7bcc00
+  data.tar.gz: bde828da11f815a64fed9de13cfee92cdd9a5b6c0357d905caafe01140ff7f441ca133f417d4b6f3ff6576ae53a058a532749735cb15312188fd4bf59736346f

data/lib/rrrspec/server.rb

@@ -10,4 +10,5 @@ require 'rrrspec/server/json_constructor'
 require 'rrrspec/server/log_file_persister'
 require 'rrrspec/server/persistent_models'
 require 'rrrspec/server/persister'
+require 'rrrspec/server/statistics_updater'
 require 'rrrspec/server/worker_runner'

data/lib/rrrspec/server/cli.rb

@@ -30,6 +30,7 @@ module RRRSpec
           Thread.fork { Dispatcher.work_loop }
           Thread.fork { Arbiter.work_loop }
           Thread.fork { Persister.work_loop }
+          Thread.fork { StatisticsUpdater.work_loop }
           Kernel.sleep
         end
       end

data/lib/rrrspec/server/persister.rb

@@ -32,8 +32,9 @@ module RRRSpec
             create_api_cache(taskset, RRRSpec.configuration.json_cache_path)
           end
           taskset.expire(PERSISTED_RESIDUE_SEC)
-          update_estimate_sec(taskset)
         end
+
+        StatisticsUpdaterQueue.enqueue(taskset)
       rescue
         RRRSpec.logger.error($!)
       end
@@ -45,6 +46,9 @@ module RRRSpec
         taskset_finished_at = taskset.finished_at
         return if taskset_finished_at.blank?
 
+        RRRSpec.logger.debug("Persisting taskset #{taskset.key}")
+        start = Time.now
+
         p_taskset = ActiveRecord::Base.transaction do
           h = taskset.to_h
           h.delete('tasks')
@@ -115,6 +119,8 @@ module RRRSpec
           Persistence::WorkerLog.import(p_worker_logs)
           p_worker_logs.each { |p_worker_log| p_worker_log.run_callbacks(:save) {} }
         end
+
+        RRRSpec.logger.info("Taskset #{taskset.key} persisted (#{Time.now - start} seconds taken)")
       end
 
       def create_api_cache(taskset, path)
@@ -126,29 +132,6 @@ module RRRSpec
         IO.write(json_path, json)
         Zlib::GzipWriter.open(json_path + ".gz") { |gz| gz.write(json) }
       end
-
-      ESTIMATION_FIELDS = [
-        "`spec_file`",
-        "avg(UNIX_TIMESTAMP(`trials`.`finished_at`)-UNIX_TIMESTAMP(`trials`.`started_at`)) as `avg`",
-        # "avg(`trials`.`finished_at`-`trials`.`started_at`) as `avg`",
-      ]
-
-      def update_estimate_sec(taskset)
-        p_obj = Persistence::Taskset.where(key: taskset.key).first
-        taskset_class = p_obj.taskset_class
-        query = Persistence::Task.joins(:trials).joins(:taskset).
-          select(ESTIMATION_FIELDS).
-          where('tasksets.taskset_class' => taskset_class).
-          where('trials.status' => ["passed", "pending"]).
-          group('spec_file')
-        estimation = {}
-        query.each do |row|
-          estimation[row.spec_file] = row.avg.to_i
-        end
-        unless estimation.empty?
-          TasksetEstimation.update_estimate_secs(taskset_class, estimation)
-        end
-      end
     end
   end
 end

data/lib/rrrspec/server/statistics_updater.rb

@@ -0,0 +1,62 @@
+module RRRSpec
+  module Server
+    module StatisticsUpdater
+      ESTIMATION_FIELDS = [
+        "`spec_file`",
+        "avg(UNIX_TIMESTAMP(`trials`.`finished_at`)-UNIX_TIMESTAMP(`trials`.`started_at`)) as `avg`",
+        # "avg(`trials`.`finished_at`-`trials`.`started_at`) as `avg`",
+      ]
+
+      module_function
+
+      def work_loop
+        loop { work }
+      end
+
+      def work
+        taskset, recalculate = StatisticsUpdaterQueue.dequeue
+        recalculate = true
+
+        ActiveRecord::Base.connection_pool.with_connection do
+          unless Persistence::Taskset.where(key: taskset.key).exists?
+            RRRSpec.logger.warn("StatisticsUpdater: Ignoreing unpersisted taskset: #{taskset.key}")
+          end
+
+          if recalculate
+            recalculate_estimate_sec taskset
+          else
+            update_estimate_sec taskset
+          end
+        end
+      rescue
+        RRRSpec.logger.error($!)
+      end
+
+      def update_estimate_sec(taskset)
+      end
+
+      def recalculate_estimate_sec(taskset)
+        RRRSpec.logger.debug("Calculating estimate sec for taskset #{taskset.key}")
+
+        start = Time.now 
+
+        p_obj = Persistence::Taskset.where(key: taskset.key).first
+        taskset_class = p_obj.taskset_class
+        query = Persistence::Task.joins(:trials).joins(:taskset).
+          select(ESTIMATION_FIELDS).
+          where('tasksets.taskset_class' => taskset_class).
+          where('trials.status' => ["passed", "pending"]).
+          group('spec_file')
+        estimation = {}
+        query.each do |row|
+          estimation[row.spec_file] = row.avg.to_i
+        end
+        unless estimation.empty?
+          TasksetEstimation.update_estimate_secs(taskset_class, estimation)
+        end
+
+        RRRSpec.logger.info("Recalculated estimate sec for taskset #{taskset.key} (total: #{Time.now - start} seconds)")
+      end
+    end
+  end
+end

data/lib/rrrspec/server/version.rb

@@ -1,5 +1,5 @@
 module RRRSpec
   module Server
-    VERSION = "0.2.1"
+    VERSION = "0.2.2"
   end
 end

data/spec/rrrspec/server/arbiter_spec.rb

@@ -29,10 +29,10 @@ module RRRSpec
           it 'cancels the taskset' do
             Arbiter.cancel(taskset)
             expect(taskset.status).to eq('cancelled')
-            expect(taskset.queue_empty?).to be_true
+            expect(taskset).to be_queue_empty
             expect(taskset.finished_at).not_to be_nil
             expect(ActiveTaskset.list).not_to include(taskset)
-            expect(PersisterQueue.empty?).to be_false
+            expect(PersisterQueue).not_to be_empty
           end
         end
 
@@ -43,7 +43,7 @@ module RRRSpec
             Arbiter.cancel(taskset)
             expect(taskset.status).not_to eq('cancelled')
             expect(ActiveTaskset.list).not_to include(taskset)
-            expect(PersisterQueue.empty?).to be_false
+            expect(PersisterQueue).not_to be_empty
           end
         end
       end
@@ -59,7 +59,7 @@ module RRRSpec
           it 'does nothing' do
             Arbiter.check(taskset)
             expect(ActiveTaskset.list).not_to include(taskset)
-            expect(PersisterQueue.empty?).to be_false
+            expect(PersisterQueue).not_to be_empty
           end
         end
 
@@ -90,7 +90,7 @@ module RRRSpec
                 expect(taskset.status).to eq('running')
                 expect(taskset.finished_at).to be_blank
                 expect(ActiveTaskset.list).to include(taskset)
-                expect(PersisterQueue.empty?).to be_true
+                expect(PersisterQueue).to be_empty
               end
             end
           end
@@ -106,7 +106,7 @@ module RRRSpec
                 expect(taskset.status).to eq("succeeded")
                 expect(taskset.finished_at).not_to be_blank
                 expect(ActiveTaskset.list).not_to include(taskset)
-                expect(PersisterQueue.empty?).to be_false
+                expect(PersisterQueue).not_to be_empty
               end
             end
 
@@ -118,7 +118,7 @@ module RRRSpec
                 expect(taskset.status).to eq("failed")
                 expect(taskset.finished_at).not_to be_nil
                 expect(ActiveTaskset.list).not_to include(taskset)
-                expect(PersisterQueue.empty?).to be_false
+                expect(PersisterQueue).not_to be_empty
               end
             end
           end
@@ -295,10 +295,10 @@ module RRRSpec
           it 'fails the taskset' do
             Arbiter.fail(taskset)
             expect(taskset.status).to eq('failed')
-            expect(taskset.queue_empty?).to be_true
+            expect(taskset).to be_queue_empty
             expect(taskset.finished_at).not_to be_nil
             expect(ActiveTaskset.list).not_to include(taskset)
-            expect(PersisterQueue.empty?).to be_false
+            expect(PersisterQueue).not_to be_empty
           end
         end
 
@@ -309,7 +309,7 @@ module RRRSpec
             Arbiter.fail(taskset)
             expect(taskset.status).not_to eq('failed')
             expect(ActiveTaskset.list).not_to include(taskset)
-            expect(PersisterQueue.empty?).to be_false
+            expect(PersisterQueue).not_to be_empty
           end
         end
       end
@@ -352,7 +352,7 @@ module RRRSpec
               expect(trial.status).to eq('error')
               expect(task1.status).to eq('failed')
               expect(taskset.tasks_left).not_to include(task1)
-              expect(taskset.queue_empty?).to be_true
+              expect(taskset).to be_queue_empty
               expect(taskset.succeeded_count).to eq(0)
               expect(taskset.failed_count).to eq(1)
             end
@@ -380,7 +380,7 @@ module RRRSpec
             Arbiter.trial(trial)
             expect(task1.status).to eq('passed')
             expect(taskset.tasks_left).not_to include(task1)
-            expect(taskset.queue_empty?).to be_true
+            expect(taskset).to be_queue_empty
             expect(taskset.succeeded_count).to eq(1)
             expect(taskset.failed_count).to eq(0)
           end
@@ -393,7 +393,7 @@ module RRRSpec
             Arbiter.trial(trial)
             expect(task1.status).to eq('pending')
             expect(taskset.tasks_left).not_to include(task1)
-            expect(taskset.queue_empty?).to be_true
+            expect(taskset).to be_queue_empty
             expect(taskset.succeeded_count).to eq(1)
             expect(taskset.failed_count).to eq(0)
           end
@@ -427,7 +427,7 @@ module RRRSpec
                   expect(trial.status).to eq('failed')
                   expect(task1.status).to eq('failed')
                   expect(taskset.tasks_left).not_to include(task1)
-                  expect(taskset.queue_empty?).to be_true
+                  expect(taskset).to be_queue_empty
                   expect(taskset.succeeded_count).to eq(0)
                   expect(taskset.failed_count).to eq(1)
                 end
@@ -438,7 +438,7 @@ module RRRSpec
                   Arbiter.trial(trial)
                   expect(task1.status).to be_blank
                   expect(taskset.tasks_left).to include(task1)
-                  expect(taskset.queue_empty?).to be_true
+                  expect(taskset).to be_queue_empty
                   expect(taskset.succeeded_count).to eq(0)
                   expect(taskset.failed_count).to eq(0)
                 end
@@ -474,7 +474,7 @@ module RRRSpec
               Arbiter.trial(trial)
               expect(task1.status).to eq('failed')
               expect(taskset.tasks_left).not_to include(task1)
-              expect(taskset.queue_empty?).to be_true
+              expect(taskset).to be_queue_empty
               expect(taskset.succeeded_count).to eq(0)
               expect(taskset.failed_count).to eq(1)
             end

data/spec/rrrspec/server/dispatcher_spec.rb

@@ -42,7 +42,7 @@ module RRRSpec
 
           it 'assignes worker upto the max_workers' do
             Dispatcher.work
-            expect(worker1.queue_empty?).to be_true
+            expect(worker1).to be_queue_empty
             workers = [worker1, worker2, worker3, worker4]
             expect(workers.count { |worker| worker.queue_empty? }).to eq(2)
           end

data/spec/rrrspec/server/persister_spec.rb

@@ -141,18 +141,6 @@ module RRRSpec
           end
         end
       end
-
-      describe '.update_estimate_sec' do
-        before { Persister.persist(@taskset) }
-
-        it 'udpates estimation of the time taken to finish the tasks' do
-          pending "sqlite3 doesn't have UNIT_TIMESTAMP function"
-          Persister.update_estimate_sec(@taskset)
-          expect(RRRSpec::TasksetEstimation.estimate_secs(@taskset.taskset_class)).to eq(
-            {"spec/test_spec.rb" => 0}
-          )
-        end
-      end
     end
   end
 end

data/spec/rrrspec/server/statistics_updater_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+module RRRSpec
+  module Server
+    describe StatisticsUpdater do
+      before do
+        RRRSpec.configuration = ServerConfiguration.new
+        RRRSpec.configuration.redis = @redis
+
+        @worker, @taskset, @task, @worker_log, @slave, @trial =
+          RRRSpec.finished_fullset
+      end
+
+      describe '.update_estimate_sec' do
+        before { Persister.persist(@taskset) }
+
+        xit 'updates estimation of the time taken to finish the tasks' do
+          pending "sqlite3 doesn't have UNIT_TIMESTAMP function"
+          Persister.update_estimate_sec(@taskset)
+          expect(RRRSpec::TasksetEstimation.estimate_secs(@taskset.taskset_class)).to eq(
+            {"spec/test_spec.rb" => 0}
+          )
+        end
+      end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rrrspec-server
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Masaya Suzuki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-22 00:00:00.000000000 Z
+date: 2014-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: database_cleaner
@@ -232,6 +232,7 @@ files:
 - lib/rrrspec/server/log_file_persister.rb
 - lib/rrrspec/server/persistent_models.rb
 - lib/rrrspec/server/persister.rb
+- lib/rrrspec/server/statistics_updater.rb
 - lib/rrrspec/server/version.rb
 - lib/rrrspec/server/worker_runner.rb
 - rrrspec-server.gemspec
@@ -241,6 +242,7 @@ files:
 - spec/rrrspec/server/log_file_persister_spec.rb
 - spec/rrrspec/server/persistent_models_spec.rb
 - spec/rrrspec/server/persister_spec.rb
+- spec/rrrspec/server/statistics_updater_spec.rb
 - spec/spec_helper.rb
 - tasks/db.rake
 homepage: ''
@@ -274,4 +276,5 @@ test_files:
 - spec/rrrspec/server/log_file_persister_spec.rb
 - spec/rrrspec/server/persistent_models_spec.rb
 - spec/rrrspec/server/persister_spec.rb
+- spec/rrrspec/server/statistics_updater_spec.rb
 - spec/spec_helper.rb