cloudtasker 0.7.0 → 0.9.1

data/gemfiles/rails_5.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    cloudtasker (0.2.0)
+    cloudtasker (0.9.1)
       activesupport
       fugit
       google-cloud-tasks
@@ -69,14 +69,24 @@ GEM
     erubi (1.9.0)
     et-orbi (1.2.2)
       tzinfo
-    faraday (0.17.0)
+    faraday (0.17.1)
       multipart-post (>= 1.2, < 3)
+    faraday-http-cache (2.0.0)
+      faraday (~> 0.8)
     fugit (1.3.3)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.1)
+    github_changelog_generator (1.15.0)
+      activesupport
+      faraday-http-cache
+      multi_json
+      octokit (~> 4.6)
+      rainbow (>= 2.2.1)
+      rake (>= 10.0)
+      retriable (~> 3.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-cloud-tasks (1.3.1)
+    google-cloud-tasks (1.3.3)
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
       googleapis-common-protos-types (>= 1.0.4, < 2.0)
@@ -87,7 +97,7 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.10.1)
+    google-protobuf (3.11.3-universal-darwin)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
@@ -101,8 +111,8 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.12)
-    grpc (1.25.0)
-      google-protobuf (~> 3.8)
+    grpc (1.27.0-universal-darwin)
+      google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.9)
       googleapis-common-protos (>= 1.3.1, < 2.0)
@@ -112,14 +122,14 @@ GEM
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     jwt (2.2.1)
-    loofah (2.3.1)
+    loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    memoist (0.16.1)
+    memoist (0.16.2)
     method_source (0.9.2)
     mimemagic (0.3.3)
     mini_mime (1.0.2)
@@ -128,10 +138,12 @@ GEM
     multi_json (1.14.1)
     multipart-post (2.1.1)
     nio4r (2.5.2)
-    nokogiri (1.10.5)
+    nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
+    octokit (4.14.0)
+      sawyer (~> 0.8.0, >= 0.5.3)
     os (1.0.1)
-    parallel (1.19.0)
+    parallel (1.19.1)
     parser (2.6.5.0)
       ast (~> 2.4.0)
     public_suffix (4.0.1)
@@ -166,6 +178,7 @@ GEM
     rainbow (3.0.0)
     rake (10.5.0)
     redis (4.1.3)
+    retriable (3.1.2)
     rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -195,10 +208,13 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.36.0)
+    rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     safe_yaml (1.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
     signet (0.12.0)
       addressable (~> 2.3)
       faraday (~> 0.9)
@@ -233,6 +249,7 @@ DEPENDENCIES
   appraisal
   bundler (~> 2.0)
   cloudtasker!
+  github_changelog_generator
   rails (= 5.2)
   rake (~> 10.0)
   rspec (~> 3.0)

data/gemfiles/rails_6.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    cloudtasker (0.2.0)
+    cloudtasker (0.9.1)
       activesupport
       fugit
       google-cloud-tasks
@@ -82,14 +82,24 @@ GEM
     erubi (1.9.0)
     et-orbi (1.2.2)
       tzinfo
-    faraday (0.17.0)
+    faraday (0.17.1)
       multipart-post (>= 1.2, < 3)
+    faraday-http-cache (2.0.0)
+      faraday (~> 0.8)
     fugit (1.3.3)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.1)
+    github_changelog_generator (1.15.0)
+      activesupport
+      faraday-http-cache
+      multi_json
+      octokit (~> 4.6)
+      rainbow (>= 2.2.1)
+      rake (>= 10.0)
+      retriable (~> 3.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-cloud-tasks (1.3.1)
+    google-cloud-tasks (1.3.3)
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
       googleapis-common-protos-types (>= 1.0.4, < 2.0)
@@ -100,7 +110,7 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.10.1-universal-darwin)
+    google-protobuf (3.11.3-universal-darwin)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
@@ -114,8 +124,8 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.12)
-    grpc (1.25.0-universal-darwin)
-      google-protobuf (~> 3.8)
+    grpc (1.27.0-universal-darwin)
+      google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.9)
       googleapis-common-protos (>= 1.3.1, < 2.0)
@@ -125,14 +135,14 @@ GEM
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     jwt (2.2.1)
-    loofah (2.3.1)
+    loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    memoist (0.16.1)
+    memoist (0.16.2)
     method_source (0.9.2)
     mimemagic (0.3.3)
     mini_mime (1.0.2)
@@ -141,10 +151,12 @@ GEM
     multi_json (1.14.1)
     multipart-post (2.1.1)
     nio4r (2.5.2)
-    nokogiri (1.10.5)
+    nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
+    octokit (4.14.0)
+      sawyer (~> 0.8.0, >= 0.5.3)
     os (1.0.1)
-    parallel (1.19.0)
+    parallel (1.19.1)
     parser (2.6.5.0)
       ast (~> 2.4.0)
     public_suffix (4.0.1)
@@ -181,6 +193,7 @@ GEM
     rainbow (3.0.0)
     rake (10.5.0)
     redis (4.1.3)
+    retriable (3.1.2)
     rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -210,10 +223,13 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.36.0)
+    rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     safe_yaml (1.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
     signet (0.12.0)
       addressable (~> 2.3)
       faraday (~> 0.9)
@@ -240,7 +256,7 @@ GEM
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
-    zeitwerk (2.2.1)
+    zeitwerk (2.2.2)
 
 PLATFORMS
   ruby
@@ -249,6 +265,7 @@ DEPENDENCIES
   appraisal
   bundler (~> 2.0)
   cloudtasker!
+  github_changelog_generator
   rails (= 6.0)
   rake (~> 10.0)
   rspec (~> 3.0)

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'
 require 'cloudtasker/authenticator'
@@ -47,5 +48,4 @@ module Cloudtasker
   end
 end
 
-require 'cloudtasker/railtie' if defined?(Rails)
 require 'cloudtasker/engine' if defined?(::Rails::Engine)

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -9,15 +9,28 @@ module Cloudtasker
       #
       # Create the queue configured in Cloudtasker if it does not already exist.
       #
+      # @param [String] queue_name The relative name of the queue.
+      #
       # @return [Google::Cloud::Tasks::V2beta3::Queue] The queue
       #
-      def self.setup_queue
-        client.get_queue(queue_path)
+      def self.setup_queue(**opts)
+        # Build full queue path
+        queue_name = opts[:name] || Cloudtasker::Config::DEFAULT_JOB_QUEUE
+        full_queue_name = queue_path(queue_name)
+
+        # Try to get existing queue
+        client.get_queue(full_queue_name)
       rescue Google::Gax::RetryError
+        # Extract options
+        concurrency = (opts[:concurrency] || Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY).to_i
+        retries = (opts[:retries] || Cloudtasker::Config::DEFAULT_QUEUE_RETRIES).to_i
+
+        # Create queue on 'not found' error
         client.create_queue(
           client.location_path(config.gcp_project_id, config.gcp_location_id),
-          name: queue_path,
-          retry_config: { max_attempts: -1 }
+          name: full_queue_name,
+          retry_config: { max_attempts: retries },
+          rate_limits: { max_concurrent_dispatches: concurrency }
         )
       end
 
@@ -42,13 +55,15 @@ module Cloudtasker
       #
       # Return the fully qualified path for the Cloud Task queue.
       #
+      # @param [String] queue_name The relative name of the queue.
+      #
       # @return [String] The queue path.
       #
-      def self.queue_path
+      def self.queue_path(queue_name)
         client.queue_path(
           config.gcp_project_id,
           config.gcp_location_id,
-          config.gcp_queue_id
+          [config.gcp_queue_prefix, queue_name].join('-')
         )
       end
 
@@ -67,6 +82,29 @@ module Cloudtasker
         Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
       end
 
+      #
+      # Format the job payload sent to Cloud Tasks.
+      #
+      # @param [Hash] hash The worker payload.
+      #
+      # @return [Hash] The Cloud Task payloadd.
+      #
+      def self.format_task_payload(payload)
+        payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
+
+        # Format schedule time to Google Protobuf timestamp
+        payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
+
+        # Encode job content to support UTF-8. Google Cloud Task
+        # expect content to be ASCII-8BIT compatible (binary)
+        payload[:http_request][:headers] ||= {}
+        payload[:http_request][:headers][Cloudtasker::Config::CONTENT_TYPE_HEADER] = 'text/json'
+        payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
+        payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
+
+        payload
+      end
+
       #
       # Find a task by id.
       #
@@ -89,13 +127,13 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
       #
       def self.create(payload)
-        # Format payload
-        payload = payload.merge(
-          schedule_time: format_schedule_time(payload[:schedule_time])
-        ).compact
+        payload = format_task_payload(payload)
+
+        # Extract relative queue name
+        relative_queue = payload.delete(:queue)
 
         # Create task
-        resp = client.create_task(queue_path, payload)
+        resp = client.create_task(queue_path(relative_queue), payload)
         resp ? new(resp) : nil
       rescue Google::Gax::RetryError
         nil
@@ -121,6 +159,20 @@ module Cloudtasker
         @gcp_task = gcp_task
       end
 
+      #
+      # Return the relative queue (queue name minus prefix) the task is in.
+      #
+      # @return [String] The relative queue name
+      #
+      def relative_queue
+        gcp_task
+          .name
+          .match(%r{/queues/([^/]+)})
+          &.captures
+          &.first
+          &.sub("#{self.class.config.gcp_queue_prefix}-", '')
+      end
+
       #
       # Return a hash description of the task.
       #
@@ -131,7 +183,8 @@ module Cloudtasker
           id: gcp_task.name,
           http_request: gcp_task.to_h[:http_request],
           schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
-          retries: gcp_task.to_h[:response_count]
+          retries: gcp_task.to_h[:response_count],
+          queue: relative_queue
         }
       end
     end

data/lib/cloudtasker/backend/memory_task.rb

@@ -7,7 +7,7 @@ module Cloudtasker
     # Manage local tasks pushed to memory.
     # Used for testing.
     class MemoryTask
-      attr_reader :id, :http_request, :schedule_time
+      attr_reader :id, :http_request, :schedule_time, :queue
 
       #
       # Return the task queue. A worker class name
@@ -116,10 +116,11 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
+        @queue = queue
       end
 
       #
@@ -149,7 +150,8 @@ module Cloudtasker
         {
           id: id,
           http_request: http_request,
-          schedule_time: schedule_time.to_i
+          schedule_time: schedule_time.to_i,
+          queue: queue
         }
       end
 

data/lib/cloudtasker/backend/redis_task.rb

@@ -7,17 +7,17 @@ module Cloudtasker
   module Backend
     # Manage local tasks pushed to Redis
     class RedisTask
-      attr_reader :id, :http_request, :schedule_time, :retries
+      attr_reader :id, :http_request, :schedule_time, :retries, :queue
 
       RETRY_INTERVAL = 20 # seconds
 
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -48,20 +48,26 @@ module Cloudtasker
       #
       # Reeturn all tasks ready to process.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks ready to process.
       #
-      def self.ready_to_process
-        all.select { |e| e.schedule_time <= Time.now }
+      def self.ready_to_process(queue = nil)
+        list = all.select { |e| e.schedule_time <= Time.now }
+        list = list.select { |e| e.queue == queue } if queue
+        list
       end
 
       #
       # Retrieve and remove a task from the queue.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Cloudtasker::Backend::RedisTask] A task ready to process.
       #
-      def self.pop
+      def self.pop(queue = nil)
         redis.with_lock('cloudtasker/server') do
-          ready_to_process.first&.tap(&:destroy)
+          ready_to_process(queue).first&.tap(&:destroy)
         end
       end
 
@@ -110,11 +116,12 @@ module Cloudtasker
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       # @param [Integer] retries The number of times the job failed.
       #
-      def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @retries = retries || 0
+        @queue = queue || Cloudtasker::Config::DEFAULT_JOB_QUEUE
       end
 
       #
@@ -136,7 +143,8 @@ module Cloudtasker
           id: id,
           http_request: http_request,
           schedule_time: schedule_time.to_i,
-          retries: retries
+          retries: retries,
+          queue: queue
         }
       end
 
@@ -155,10 +163,13 @@ module Cloudtasker
       # @param [Integer] interval The delay in seconds before retrying the task
       #
       def retry_later(interval, is_error: true)
-        redis.write(gid,
-                    retries: is_error ? retries + 1 : retries,
-                    http_request: http_request,
-                    schedule_time: (Time.now + interval).to_i)
+        redis.write(
+          gid,
+          retries: is_error ? retries + 1 : retries,
+          http_request: http_request,
+          schedule_time: (Time.now + interval).to_i,
+          queue: queue
+        )
       end
 
       #