dronejob 1.6.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d5e8318926dc2e1172dd75fde843e138c07d957
-  data.tar.gz: 84d2f9eff533a50c1bbe198edf695d8f526a6216
+  metadata.gz: a310b9d28ba8888a684e9f92c46131ee23085858
+  data.tar.gz: 28b7ffcc75f4011b8355cd44c0a9672420f5e8d5
 SHA512:
-  metadata.gz: 727c3da9be4f677294e675ebcbc4c7f41ad513e861c20d86497a94af31db90e70ba324c61b2df5f4527cd744f144f6fa549a6b998cb576e76dd2f85d80a8d94c
-  data.tar.gz: d6f86e969fbb77d7324acd26436dcef4e1b9d1557f49de0a05d953b28e59ebc6a985e8f4193db64f15e20cecd5d38b9b00c88ce7e7feccb6310765f003d9ec54
+  metadata.gz: 9d3b332ca2191fbdec6fc587461aa156ad57ac01bee4bde8281e225aa03715e7a4a968fd78381b612f25b7a8bfcef92114a92c205704bbd9f0359a766f371541
+  data.tar.gz: f9b6f3f71e441f7d29f8ac3d8277be31f0d1e36661dcafafbdb930b18f6c3226137fe12035c8b966456b76ebcca4dfe469f52502f34352497f5c5312356aebfb

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    dronejob (1.6.0)
+    dronejob (2.0.0)
       actionpack (>= 4.0.2)
       activejob (>= 4.0.2)
+      activejob-google_cloud_pubsub (>= 0.7)
       bundler (>= 1.12.0, < 3.0.0)
       git (>= 1.3)
       google-cloud-logging (>= 1.2)
       pusher (>= 1.3)
-      sidekiq (>= 5.0)
       thor (>= 0.19)
       workspace (>= 1.0)
       workspace-archive (>= 1.0)
@@ -35,6 +35,11 @@ GEM
     activejob (5.2.3)
       activesupport (= 5.2.3)
       globalid (>= 0.3.6)
+    activejob-google_cloud_pubsub (0.7.1)
+      activejob
+      activesupport
+      concurrent-ruby
+      google-cloud-pubsub (>= 0.27.0)
     activesupport (5.2.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
@@ -46,7 +51,6 @@ GEM
     builder (3.2.3)
     coderay (1.1.2)
     concurrent-ruby (1.1.5)
-    connection_pool (2.2.2)
     crass (1.0.4)
     diff-lcs (1.3)
     erubi (1.8.0)
@@ -68,13 +72,18 @@ GEM
       google-gax (~> 1.3)
       googleapis-common-protos-types (>= 1.0.2)
       stackdriver-core (~> 1.3)
+    google-cloud-pubsub (0.30.2)
+      concurrent-ruby (~> 1.0)
+      google-cloud-core (~> 1.2)
+      google-gax (~> 1.0)
+      grpc-google-iam-v1 (~> 0.6.9)
     google-gax (1.5.0)
       google-protobuf (~> 3.2)
       googleapis-common-protos (>= 1.3.5, < 2.0)
       googleauth (>= 0.6.2, < 0.10.0)
       grpc (>= 1.7.2, < 2.0)
       rly (~> 0.2.3)
-    google-protobuf (3.7.1)
+    google-protobuf (3.7.1-universal-darwin)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
@@ -88,9 +97,12 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    grpc (1.20.0)
+    grpc (1.20.0-universal-darwin)
       google-protobuf (~> 3.7)
       googleapis-common-protos-types (~> 1.0.0)
+    grpc-google-iam-v1 (0.6.9)
+      googleapis-common-protos (>= 1.3.1, < 2.0)
+      grpc (~> 1.0)
     haml (5.1.0)
       temple (>= 0.8.0)
       tilt
@@ -130,8 +142,6 @@ GEM
       pusher-signature (~> 0.1.8)
     pusher-signature (0.1.8)
     rack (2.0.7)
-    rack-protection (2.0.5)
-      rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails-dom-testing (2.0.3)
@@ -140,7 +150,6 @@ GEM
     rails-html-sanitizer (1.0.4)
       loofah (~> 2.2, >= 2.2.2)
     rainbow (3.0.0)
-    redis (4.1.1)
     rly (0.2.3)
     rmagick (2.16.0)
     rspec (3.8.0)
@@ -165,11 +174,6 @@ GEM
       unicode-display_width (>= 1.4.0, < 1.7)
     ruby-progressbar (1.10.0)
     rubyzip (1.2.2)
-    sidekiq (5.2.7)
-      connection_pool (~> 2.2, >= 2.2.2)
-      rack (>= 1.5.0)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
     signet (0.11.0)
       addressable (~> 2.3)
       faraday (~> 0.9)

data/README.md

@@ -13,3 +13,7 @@ Install gems:
 List dronejob commands:
 
 	$ bundle exec dronejob
+
+Start server:
+
+	$ bundle exec dronejob server

data/dronejob.gemspec

@@ -17,10 +17,10 @@ Gem::Specification.new do |s|
   s.add_dependency "bundler", "< 3.0.0", ">= 1.12.0"
   s.add_runtime_dependency "actionpack", ">= 4.0.2"
   s.add_runtime_dependency "activejob", ">= 4.0.2"
+  s.add_runtime_dependency "activejob-google_cloud_pubsub", ">= 0.7"
   s.add_runtime_dependency "git", ">= 1.3"
   s.add_runtime_dependency "google-cloud-logging", ">= 1.2"
   s.add_runtime_dependency "pusher", ">= 1.3"
-  s.add_runtime_dependency "sidekiq", ">= 5.0"
   s.add_runtime_dependency "thor", ">= 0.19"
   s.add_runtime_dependency "workspace", ">= 1.0"
   s.add_runtime_dependency "workspace-archive", ">= 1.0"

data/lib/dronejob/command.rb

@@ -1,19 +1,16 @@
 require "dronejob"
 require "active_job"
+
 module Dronejob
   class Command < Thor
     Dronejob::Loader.workers(:base).each do |identifier, worker|
       desc identifier, "Queue #{identifier} worker"
 
-      # Pub/Sub options
-      method_option(:queue_adapter, type: :string, default: "sidekiq")
-
       # Worker options
       method_option(:run, aliases: "-r", type: :boolean, default: false)
       method_option(:break, aliases: "-b", type: :boolean, default: false)
       method_option(:skip, aliases: "-s", type: :array, default: [])
       method_option(:from, aliases: "-f", type: :string)
-      method_option(:listen, aliases: "-l", type: :boolean, default: false)
       method_option(:notify, aliases: "-n", type: :boolean, default: true)
       method_option(:jobs_path, type: :string)
       method_option(:output_path, type: :string)
@@ -24,7 +21,6 @@ module Dronejob
       end
 
       define_method(identifier) do
-        Dronejob::Base.queue_adapter = options.queue_adapter
         Dronejob::Base.jobs_path = options.jobs_path if options.jobs_path
         Dronejob::Base.output_path = options.output_path if options.output_path
 
@@ -32,28 +28,8 @@ module Dronejob
         if options.run
           worker.perform_now(options)
         else
-          Sidekiq.configure_client do |config|
-            config.redis = { url: Dronejob::Base.options.redis_url }
-          end
           job = worker.perform_later(options.to_hash)
-
-          # Listen for notifications
-          if options.listen
-            if options.run or !options.notify
-              Dronejob::Base.log("error", "Unable to listen to a jobs that '--run' instantly, or with '--no-notify' set")
-            else
-              Dronejob::Base.subscribe(Dronejob::Base.status_topic, true) do |data|
-                if data["job_id"] == job.job_id
-                  Dronejob::Base.log("info", "status: #{data['detail'].to_json}")
-                  if data["detail"]["phase"] == "complete"
-                    Dronejob::Base.log("info", "job completed")
-                    Kernel.exit
-                  end
-                end
-              end
-            end
-          end
-
+          Dronejob::Base.log("info", "Enqueued job #{job.job_id}")
         end
       end
     end
@@ -109,8 +85,18 @@ module Dronejob
       job_identifier = dronejob_config[:job_identifier]
       worker = Dronejob::Loader.workers[job_identifier]
       job_params["from"] = options.from
-      Dronejob::Base.queue_adapter = job_params["queue_adapter"]
       worker.perform_now(job_params)
     end
+
+    desc :server, "Start Dronejob Server"
+    def server
+      server = Server.new(Dronejob::Base.options)
+      begin
+        server.start
+      rescue Interrupt
+        server.stop
+        exit
+      end
+    end
   end
 end

data/lib/dronejob/modules/log.rb

@@ -30,7 +30,7 @@ module Dronejob
       end
 
       class_methods do
-        def log(level, message, job)
+        def log(level, message, job = nil)
           if !logger.nil?
             if option(:dronejob_log) == "google"
               message = "[#{job.job_id}] #{message}" if level != :error
@@ -38,15 +38,17 @@ module Dronejob
               # data[:phase] = job.phase.to_s if job.phase
               # data[:user_id] = job.param(:user_id) if job.param(:user_id)
               data[:phase_title] = job.phase_config[:title].to_s if job.phase_config&.key?(:title)
-              if job.phase_config&.key?(:progress)
+              if job&.phase_config&.key?(:progress)
                 data[:phase_progress] = job.phase_config[:progress]
               end
               logger.send(level, data.compact)
             else
               logger.send(level, message)
             end
-          else
+          elsif job
             puts "#{level}: [#{job.phase}] #{message}"
+          else
+            puts "#{level}: #{message}"
           end
         end
       end

data/lib/dronejob/modules/queue.rb

@@ -1,17 +1,25 @@
 require "pusher"
+
 module Dronejob
   module Modules
     module Queue
       extend ActiveSupport::Concern
 
       included do
-        attr_accessor :queue_adapter
         queue_as option(:dronejob_queue) || :dronejob
         set_callback :phase, :before, :notify_before_phase
         set_callback :job, :after, :notify_after_job
       end
 
       class_methods do
+        def pubsub
+          @pubsub ||= Google::Cloud::Pubsub.new(project_id: option(:google_cloud_project_id), credentials: option(:google_cloud_service_account))
+        end
+
+        def queue_adapter
+          @queue_adapter ||= ActiveJob::GoogleCloudPubsub::Adapter.new(async: false, logger: logger, pubsub: pubsub)
+        end
+
         def notify(value = true)
           @notify = value
         end

data/lib/dronejob/server.rb

@@ -0,0 +1,21 @@
+module Dronejob
+  class Dronejob::Server
+    attr_reader :worker, :queue, :min_threads, :max_threads
+
+    def initialize(options)
+      @queue = options.dronejob_queue || "drone"
+      @min_threads = options.min_threads || 1
+      @max_threads = options.max_threads || 1
+    end
+
+    def start
+      @worker = ActiveJob::GoogleCloudPubsub::Worker.new(queue: @queue, pubsub: Dronejob::Base.pubsub, min_threads: self.min_threads, max_threads: self.max_threads)
+      Dronejob::Base.log("info", "Starting server for queue #{@queue} ...", nil)
+      @worker.run
+    end
+
+    def stop
+      Dronejob::Base.log("info", "Stopping server ...", nil)
+    end
+  end
+end

data/lib/dronejob/version.rb

@@ -1,3 +1,3 @@
 module Dronejob
-  VERSION = "1.6.0"
+  VERSION = "2.0.0"
 end

data/lib/dronejob.rb

@@ -1,6 +1,8 @@
+require 'activejob-google_cloud_pubsub'
 require "google/cloud/logging"
 require "active_support"
 require 'active_support/rails'
+require "dronejob/server"
 require "dronejob/version"
 require "psych"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dronejob
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Tobias Strebitzer
@@ -59,61 +59,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 4.0.2
 - !ruby/object:Gem::Dependency
-  name: git
+  name: activejob-google_cloud_pubsub
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: google-cloud-logging
+  name: git
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: pusher
+  name: google-cloud-logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
-  name: sidekiq
+  name: pusher
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -253,7 +253,6 @@ files:
 - Gemfile
 - Gemfile.lock
 - README.md
-- app.rb
 - bin/dronejob
 - dronejob.gemspec
 - lib/dronejob.rb
@@ -272,6 +271,7 @@ files:
 - lib/dronejob/modules/workspace.rb
 - lib/dronejob/railtie.rb
 - lib/dronejob/railties/dronejob.rake
+- lib/dronejob/server.rb
 - lib/dronejob/version.rb
 homepage: https://github.com/MagLoft/dronejob
 licenses:

data/app.rb

@@ -1,6 +0,0 @@
-require "dronejob"
-Dronejob::Loader.load
-Dronejob::Base.queue_adapter = "sidekiq"
-Sidekiq.configure_server do |config|
-  config.redis = { url: Dronejob::Base.options.redis_url }
-end