cloudtasker 0.8.0 → 0.9.2

data/gemfiles/rails_6.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    cloudtasker (0.2.0)
+    cloudtasker (0.9.2)
       activesupport
       fugit
       google-cloud-tasks
@@ -73,23 +73,33 @@ GEM
       rake
       thor (>= 0.14.0)
     ast (2.4.0)
-    builder (3.2.3)
-    concurrent-ruby (1.1.5)
+    builder (3.2.4)
+    concurrent-ruby (1.1.6)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.5)
+    crass (1.0.6)
     diff-lcs (1.3)
     erubi (1.9.0)
     et-orbi (1.2.2)
       tzinfo
-    faraday (0.17.0)
+    faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
+    faraday-http-cache (2.0.0)
+      faraday (~> 0.8)
     fugit (1.3.3)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.1)
+    github_changelog_generator (1.15.0)
+      activesupport
+      faraday-http-cache
+      multi_json
+      octokit (~> 4.6)
+      rainbow (>= 2.2.1)
+      rake (>= 10.0)
+      retriable (~> 3.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-cloud-tasks (1.3.1)
+    google-cloud-tasks (1.4.0)
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
       googleapis-common-protos-types (>= 1.0.4, < 2.0)
@@ -100,56 +110,59 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.10.1-universal-darwin)
+    google-protobuf (3.11.4)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
       grpc (~> 1.0)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
-    googleauth (0.10.0)
-      faraday (~> 0.12)
+    googleauth (0.11.0)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.12)
-    grpc (1.25.0-universal-darwin)
-      google-protobuf (~> 3.8)
+    grpc (1.27.0)
+      google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.9)
       googleapis-common-protos (>= 1.3.1, < 2.0)
       grpc (~> 1.0)
-    hashdiff (1.0.0)
-    i18n (1.7.0)
+    hashdiff (1.0.1)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     jwt (2.2.1)
-    loofah (2.3.1)
+    loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    memoist (0.16.1)
+    memoist (0.16.2)
     method_source (0.9.2)
-    mimemagic (0.3.3)
+    mimemagic (0.3.4)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.13.0)
+    minitest (5.14.0)
     multi_json (1.14.1)
     multipart-post (2.1.1)
     nio4r (2.5.2)
-    nokogiri (1.10.5)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
+    octokit (4.16.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
     os (1.0.1)
-    parallel (1.19.0)
-    parser (2.6.5.0)
+    parallel (1.19.1)
+    parser (2.7.0.4)
       ast (~> 2.4.0)
-    public_suffix (4.0.1)
+    public_suffix (4.0.3)
     raabro (1.1.6)
-    rack (2.0.7)
+    rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (6.0.0)
@@ -179,19 +192,20 @@ GEM
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
-    rake (10.5.0)
+    rake (13.0.1)
     redis (4.1.3)
+    retriable (3.1.2)
     rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
     rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-rails (3.9.0)
@@ -202,7 +216,7 @@ GEM
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
+    rspec-support (3.9.2)
     rubocop (0.76.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
@@ -210,13 +224,16 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.36.0)
+    rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     safe_yaml (1.0.5)
-    signet (0.12.0)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    signet (0.13.0)
       addressable (~> 2.3)
-      faraday (~> 0.9)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     sprockets (4.0.0)
@@ -226,21 +243,21 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sqlite3 (1.4.1)
-    thor (0.20.3)
+    sqlite3 (1.4.2)
+    thor (1.0.1)
     thread_safe (0.3.6)
     timecop (0.9.1)
-    tzinfo (1.2.5)
+    tzinfo (1.2.6)
       thread_safe (~> 0.1)
-    unicode-display_width (1.6.0)
-    webmock (3.7.6)
+    unicode-display_width (1.6.1)
+    webmock (3.8.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
-    zeitwerk (2.2.1)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
@@ -249,12 +266,13 @@ DEPENDENCIES
   appraisal
   bundler (~> 2.0)
   cloudtasker!
+  github_changelog_generator
   rails (= 6.0)
-  rake (~> 10.0)
+  rake (>= 12.3.3)
   rspec (~> 3.0)
   rspec-rails
   rubocop (= 0.76.0)
-  rubocop-rspec
+  rubocop-rspec (= 1.37.0)
   sqlite3
   timecop
   webmock

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'
 require 'cloudtasker/authenticator'

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -82,6 +82,29 @@ module Cloudtasker
         Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
       end
 
+      #
+      # Format the job payload sent to Cloud Tasks.
+      #
+      # @param [Hash] hash The worker payload.
+      #
+      # @return [Hash] The Cloud Task payloadd.
+      #
+      def self.format_task_payload(payload)
+        payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
+
+        # Format schedule time to Google Protobuf timestamp
+        payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
+
+        # Encode job content to support UTF-8. Google Cloud Task
+        # expect content to be ASCII-8BIT compatible (binary)
+        payload[:http_request][:headers] ||= {}
+        payload[:http_request][:headers][Cloudtasker::Config::CONTENT_TYPE_HEADER] = 'text/json'
+        payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
+        payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
+
+        payload
+      end
+
       #
       # Find a task by id.
       #
@@ -104,10 +127,7 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
       #
       def self.create(payload)
-        # Format payload
-        payload = payload.merge(
-          schedule_time: format_schedule_time(payload[:schedule_time])
-        ).compact
+        payload = format_task_payload(payload)
 
         # Extract relative queue name
         relative_queue = payload.delete(:queue)
@@ -126,7 +146,7 @@ module Cloudtasker
       #
       def self.delete(id)
         client.delete_task(id)
-      rescue Google::Gax::RetryError
+      rescue Google::Gax::RetryError, GRPC::NotFound
         nil
       end
 

data/lib/cloudtasker/backend/redis_task.rb

@@ -121,7 +121,7 @@ module Cloudtasker
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @retries = retries || 0
-        @queue = queue
+        @queue = queue || Cloudtasker::Config::DEFAULT_JOB_QUEUE
       end
 
       #
@@ -163,10 +163,13 @@ module Cloudtasker
       # @param [Integer] interval The delay in seconds before retrying the task
       #
       def retry_later(interval, is_error: true)
-        redis.write(gid,
-                    retries: is_error ? retries + 1 : retries,
-                    http_request: http_request,
-                    schedule_time: (Time.now + interval).to_i)
+        redis.write(
+          gid,
+          retries: is_error ? retries + 1 : retries,
+          http_request: http_request,
+          schedule_time: (Time.now + interval).to_i,
+          queue: queue
+        )
       end
 
       #

data/lib/cloudtasker/cloud_task.rb

@@ -48,6 +48,8 @@ module Cloudtasker
     # @return [Cloudtasker::CloudTask] The created task.
     #
     def self.create(payload)
+      raise MaxTaskSizeExceededError if payload.to_json.bytesize > Config::MAX_TASK_SIZE
+
       resp = backend.create(payload)&.to_h
       resp ? new(resp) : nil
     end

data/lib/cloudtasker/config.rb

@@ -9,9 +9,21 @@ module Cloudtasker
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
+    # Max Cloud Task size in bytes
+    MAX_TASK_SIZE = 100 * 1024 # 100 KB
+
     # Retry header in Cloud Task responses
     RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
 
+    # Content-Transfer-Encoding header in Cloud Task responses
+    ENCODING_HEADER = 'Content-Transfer-Encoding'
+
+    # Content Type
+    CONTENT_TYPE_HEADER = 'Content-Type'
+
+    # Authorization header
+    AUTHORIZATION_HEADER = 'Authorization'
+
     # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
@@ -101,8 +113,14 @@ module Cloudtasker
     def processor_host=(val)
       @processor_host = val
 
+      # Check if Rails supports host filtering
+      return unless val &&
+                    defined?(Rails) &&
+                    Rails.application.config.respond_to?(:hosts) &&
+                    Rails.application.config.hosts&.any?
+
       # Add processor host to the list of authorized hosts
-      Rails.application.config.hosts << val.gsub(%r{https?://}, '') if val && defined?(Rails)
+      Rails.application.config.hosts << val.gsub(%r{https?://}, '')
     end
 
     #

data/lib/cloudtasker/max_task_size_exceeded_error.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle Cloud Task size quota
+  # See: https://cloud.google.com/appengine/quotas#Task_Queue
+  #
+  class MaxTaskSizeExceededError < StandardError
+    MSG = 'The size of Cloud Tasks must not exceed 100KB'
+
+    def initialize(msg = MSG)
+      super
+    end
+  end
+end

data/lib/cloudtasker/redis_client.rb

@@ -8,13 +8,17 @@ module Cloudtasker
     # Suffix added to cache keys when locking them
     LOCK_KEY_PREFIX = 'cloudtasker/lock'
 
+    def self.client
+      @client ||= Redis.new(Cloudtasker.config.redis || {})
+    end
+
     #
     # Return the underlying redis client.
     #
     # @return [Redis] The redis client.
     #
     def client
-      @client ||= Redis.new(Cloudtasker.config.redis || {})
+      @client ||= self.class.client
     end
 
     #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.8.0'
+  VERSION = '0.9.2'
 end

data/lib/cloudtasker/worker.rb

@@ -35,7 +35,7 @@ module Cloudtasker
     def self.from_hash(hash)
       # Symbolize metadata keys and stringify job arguments
       payload = JSON.parse(hash.to_json, symbolize_names: true)
-      payload[:job_args] = JSON.parse(hash[:job_args].to_json)
+      payload[:job_args] = JSON.parse(payload[:job_args].to_json)
 
       # Extract worker parameters
       klass_name = payload&.dig(:worker)

data/lib/cloudtasker/worker_handler.rb

@@ -42,8 +42,8 @@ module Cloudtasker
           http_method: 'POST',
           url: Cloudtasker.config.processor_url,
           headers: {
-            'Content-Type' => 'application/json',
-            'Authorization' => "Bearer #{Authenticator.verification_token}"
+            Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
+            Cloudtasker::Config::AUTHORIZATION_HEADER => "Bearer #{Authenticator.verification_token}"
           },
           body: worker_payload.to_json
         },

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.2
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-27 00:00:00.000000000 Z
+date: 2020-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -126,16 +126,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +168,16 @@ dependencies:
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -264,10 +264,10 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- _config.yml
 - app/controllers/cloudtasker/application_controller.rb
 - app/controllers/cloudtasker/worker_controller.rb
 - bin/console
@@ -320,6 +320,7 @@ files:
 - lib/cloudtasker/engine.rb
 - lib/cloudtasker/invalid_worker_error.rb
 - lib/cloudtasker/local_server.rb
+- lib/cloudtasker/max_task_size_exceeded_error.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
 - lib/cloudtasker/redis_client.rb