cloudtasker 0.8.2 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4c3ce66208b1b45f5ec361f098ac3ceb1db562f2b4af45b3d42ef6ce54b6aac
-  data.tar.gz: 52ed1b155fea5ce6c307498f058b4b57d115b97e73ac7d0e1ab685c10c62672d
+  metadata.gz: 79a19719ba7b8f5572cc36b390d2c9c6a84149911dda847cd6f3dda6f3f9aa86
+  data.tar.gz: ae5c92c86c5363caaf5143f576c181a3a500c661c93120ff2d7545bebf05cc85
 SHA512:
-  metadata.gz: a3a4276e4494fc8912480a06cfe81f283340b516ec45cc9a504ffe02c733b42c3dfe3e14e9d132aacf93ff5d4f25534459d85467426497bc46383fbfc4c7fa08
-  data.tar.gz: ce32054e9e9f6fe496bcd3c36b6e15d012720834f71c237da1cbe82119a6a09d23c4b1463c11044261e890f3e1262fc94ed6dd42e86f72bfb340f79fed6cbbba
+  metadata.gz: 626cbfb8be6e316a50e76b9d082fb709c0b0a637752ed888e07a5af9ba0804bac01afa2a853d8b2d72743579afafda5e0c92ef32e6f0ec6f7a8ab1a36435c74a
+  data.tar.gz: b9dda9cef3b3b0278760c2fcda32e3e0abbfa1f571dd08e7636af640d3aedb447f29c097b7ccc04316a4392306dbdc96b243de715a527f29a7dd70fc13d9997c

data/.github/workflows/test.yml

@@ -0,0 +1,45 @@
+name: Test
+
+on:
+  push:
+    branches: [ master, 0.9-stable ]
+  pull_request:
+    branches: [ master, 0.9-stable ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby:
+          - '2.5.x'
+          - '2.6.x'
+        appraisal:
+          - 'google-cloud-tasks-1.0'
+          - 'google-cloud-tasks-1.1'
+          - 'google-cloud-tasks-1.2'
+          - 'google-cloud-tasks-1.3'
+          - 'rails-5.2'
+          - 'rails-6.0'
+          - 'semantic_logger-3.4'
+          - 'semantic_logger-4.6'
+          - 'semantic_logger-4.7.0'
+          - 'semantic_logger-4.7.2'
+    steps:
+      - name: Setup System
+        run: sudo apt-get install libsqlite3-dev
+      - uses: actions/checkout@v2
+      - uses: zhulik/redis-action@1.1.0
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        env:
+            APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rubocop
+          bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/.rubocop.yml

@@ -2,7 +2,7 @@ require: rubocop-rspec
 
 AllCops:
   Exclude:
-    - 'gemfiles/vendor/**/*'
+    - 'gemfiles/**/*'
     - 'vendor/**/*'
 
 Metrics/ClassLength:

data/Appraisals

@@ -23,3 +23,19 @@ end
 appraise 'rails-6.0' do
   gem 'rails', '6.0'
 end
+
+appraise 'semantic_logger-3.4' do
+  gem 'semantic_logger', '3.4.1'
+end
+
+appraise 'semantic_logger-4.6' do
+  gem 'semantic_logger', '4.6.1'
+end
+
+appraise 'semantic_logger-4.7.0' do
+  gem 'semantic_logger', '4.7.0'
+end
+
+appraise 'semantic_logger-4.7.2' do
+  gem 'semantic_logger', '4.7.2'
+end

data/CHANGELOG.md

@@ -1,5 +1,51 @@
 # Changelog
 
+## [v0.9.4](https://github.com/keypup-io/cloudtasker/tree/v0.9.4) (2020-10-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.9.4)
+
+**Fixed bugs:**
+- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
+
+## [v0.9.3](https://github.com/keypup-io/cloudtasker/tree/v0.9.3) (2020-06-25)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.2...v0.9.3)
+
+**Fixed bugs:**
+- Google Cloud Tasks: lock version to `~> 1.0` (Google recently released a v2 which changes its bindings completely). An [issue](https://github.com/keypup-io/cloudtasker/issues/11) has been raised to upgrade Cloudtasker to `google-cloud-tasks` `v2`.
+
+## [v0.9.2](https://github.com/keypup-io/cloudtasker/tree/v0.9.2) (2020-03-04)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.1...v0.9.2)
+
+**Fixed bugs:**
+- Cloud Task: ignore "not found" errors when trying to delete an already deleted task.
+
+## [v0.9.1](https://github.com/keypup-io/cloudtasker/tree/v0.9.1) (2020-02-11)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.0...v0.9.1)
+
+**Fixed bugs:**
+- Cloud Task: raise `Cloudtasker::MaxTaskSizeExceededError` if job payload exceeds 100 KB. This is mainly to have production parity in development when running the local processing server.
+
+## [v0.9.0](https://github.com/keypup-io/cloudtasker/tree/v0.9.0) (2020-01-23)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.2...v0.9.0)
+
+**Fixed bugs:**
+- Cloud Task: Base64 encode task body to support UTF-8 characters (e.g. emojis).
+- Redis: Restrict to one connection (class level) to avoid too many DNS lookups
+
+**Migration**
+For Sinatra applications please update your Cloudtasker controller according to [this diff](https://github.com/keypup-io/cloudtasker/commit/311fa8f9beec91fbae012164a25b2ee6e261a2e4#diff-c2a0ea6c6e6c31c749d2e1acdc574f0f).
+
+## [v0.8.2](https://github.com/keypup-io/cloudtasker/tree/v0.8.2) (2019-12-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.1...v0.8.2)
+
+**Fixed bugs:**
+- Config: do not add processor host to `Rails.application.config.hosts` if originally empty.
+
 ## [v0.8.1](https://github.com/keypup-io/cloudtasker/tree/v0.8.1) (2019-12-03)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.0...v0.8.1)

data/README.md

@@ -635,7 +635,9 @@ If you enqueue this worker by omitting the second argument `MyWorker.perform_asy
 - The `time_at` argument will be ignored by the `unique-job` extension, meaning that job uniqueness will be only based on the `user_id` argument.
 
 ### Handling big job payloads
-Keep in mind that jobs are pushed to Google Cloud Tasks via API and then delivered to your application via API as well. Therefore any excessive job payload will slow down the enqueuing of jobs and create additional processing when receiving the job.
+Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into accounts Cloudtasker authentication headers and meta information this leave ~85 KB of free space for JSONified job arguments.
+
+Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
 If you feel that a job payload is going to get big, prefer to store the payload using a datastore (e.g. Redis) and pass a reference to the job to retrieve the payload inside your job `perform` method.
 

data/app/controllers/cloudtasker/worker_controller.rb

@@ -16,9 +16,6 @@ module Cloudtasker
     # Run a worker from a Cloud Task payload
     #
     def run
-      # Build payload
-      payload = JSON.parse(request.body.read).merge(job_retries: job_retries)
-
       # Process payload
       WorkerHandler.execute_from_payload!(payload)
       head :no_content
@@ -37,6 +34,27 @@ module Cloudtasker
 
     private
 
+    #
+    # Parse the request body and return the actual job
+    # payload.
+    #
+    # @return [Hash] The job payload
+    #
+    def payload
+      @payload ||= begin
+        # Get raw body
+        content = request.body.read
+
+        # Decode content if the body is Base64 encoded
+        if request.headers[Cloudtasker::Config::ENCODING_HEADER].to_s.downcase == 'base64'
+          content = Base64.decode64(content)
+        end
+
+        # Return content parsed as JSON and add job retries count
+        JSON.parse(content).merge(job_retries: job_retries)
+      end
+    end
+
     #
     # Extract the number of times this task failed at runtime.
     #

data/cloudtasker.gemspec

@@ -32,17 +32,18 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport'
   spec.add_dependency 'fugit'
-  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'google-cloud-tasks', '~> 1.0'
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
 
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'github_changelog_generator'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '0.76.0'
-  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'rubocop-rspec', '1.37.0'
+  spec.add_development_dependency 'semantic_logger'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'webmock'
 

data/gemfiles/google_cloud_tasks_1.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.0'
+gem "google-cloud-tasks", "1.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.1.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.1'
+gem "google-cloud-tasks", "1.1"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.2'
+gem "google-cloud-tasks", "1.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.3.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.3'
+gem "google-cloud-tasks", "1.3"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '5.2'
+gem "rails", "5.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_6.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '6.0'
+gem "rails", "6.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/semantic_logger_3.4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "3.4.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.6.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.6.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.0"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# This file was generated by Appraisal
+
+source 'https://rubygems.org'
+
+gem 'semantic_logger', '4.7.2'
+
+gemspec path: '../'

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'
 require 'cloudtasker/authenticator'

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -82,6 +82,29 @@ module Cloudtasker
         Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
       end
 
+      #
+      # Format the job payload sent to Cloud Tasks.
+      #
+      # @param [Hash] hash The worker payload.
+      #
+      # @return [Hash] The Cloud Task payloadd.
+      #
+      def self.format_task_payload(payload)
+        payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
+
+        # Format schedule time to Google Protobuf timestamp
+        payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
+
+        # Encode job content to support UTF-8. Google Cloud Task
+        # expect content to be ASCII-8BIT compatible (binary)
+        payload[:http_request][:headers] ||= {}
+        payload[:http_request][:headers][Cloudtasker::Config::CONTENT_TYPE_HEADER] = 'text/json'
+        payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
+        payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
+
+        payload
+      end
+
       #
       # Find a task by id.
       #
@@ -104,10 +127,7 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
       #
       def self.create(payload)
-        # Format payload
-        payload = payload.merge(
-          schedule_time: format_schedule_time(payload[:schedule_time])
-        ).compact
+        payload = format_task_payload(payload)
 
         # Extract relative queue name
         relative_queue = payload.delete(:queue)
@@ -126,7 +146,7 @@ module Cloudtasker
       #
       def self.delete(id)
         client.delete_task(id)
-      rescue Google::Gax::RetryError
+      rescue Google::Gax::RetryError, GRPC::NotFound
         nil
       end
 

data/lib/cloudtasker/cloud_task.rb

@@ -48,6 +48,8 @@ module Cloudtasker
     # @return [Cloudtasker::CloudTask] The created task.
     #
     def self.create(payload)
+      raise MaxTaskSizeExceededError if payload.to_json.bytesize > Config::MAX_TASK_SIZE
+
       resp = backend.create(payload)&.to_h
       resp ? new(resp) : nil
     end

data/lib/cloudtasker/config.rb

@@ -9,9 +9,21 @@ module Cloudtasker
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
+    # Max Cloud Task size in bytes
+    MAX_TASK_SIZE = 100 * 1024 # 100 KB
+
     # Retry header in Cloud Task responses
     RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
 
+    # Content-Transfer-Encoding header in Cloud Task responses
+    ENCODING_HEADER = 'Content-Transfer-Encoding'
+
+    # Content Type
+    CONTENT_TYPE_HEADER = 'Content-Type'
+
+    # Authorization header
+    AUTHORIZATION_HEADER = 'Authorization'
+
     # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'

data/lib/cloudtasker/max_task_size_exceeded_error.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle Cloud Task size quota
+  # See: https://cloud.google.com/appengine/quotas#Task_Queue
+  #
+  class MaxTaskSizeExceededError < StandardError
+    MSG = 'The size of Cloud Tasks must not exceed 100KB'
+
+    def initialize(msg = MSG)
+      super
+    end
+  end
+end

data/lib/cloudtasker/redis_client.rb

@@ -8,13 +8,17 @@ module Cloudtasker
     # Suffix added to cache keys when locking them
     LOCK_KEY_PREFIX = 'cloudtasker/lock'
 
+    def self.client
+      @client ||= Redis.new(Cloudtasker.config.redis || {})
+    end
+
     #
     # Return the underlying redis client.
     #
     # @return [Redis] The redis client.
     #
     def client
-      @client ||= Redis.new(Cloudtasker.config.redis || {})
+      @client ||= self.class.client
     end
 
     #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.8.2'
+  VERSION = '0.9.4'
 end

data/lib/cloudtasker/worker_handler.rb

@@ -42,8 +42,8 @@ module Cloudtasker
           http_method: 'POST',
           url: Cloudtasker.config.processor_url,
           headers: {
-            'Content-Type' => 'application/json',
-            'Authorization' => "Bearer #{Authenticator.verification_token}"
+            Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
+            Cloudtasker::Config::AUTHORIZATION_HEADER => "Bearer #{Authenticator.verification_token}"
           },
           body: worker_payload.to_json
         },

data/lib/cloudtasker/worker_logger.rb

@@ -141,7 +141,8 @@ module Cloudtasker
     # @param [Proc] &block Optional context block.
     #
     def log_message(level, msg, &block)
-      payload_block = block || log_block
+      # Merge log-specific context into worker-specific context
+      payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.4
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-05 00:00:00.000000000 Z
+date: 2020-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -42,16 +42,16 @@ dependencies:
   name: google-cloud-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jwt
   requirement: !ruby/object:Gem::Requirement
@@ -126,16 +126,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -166,6 +166,20 @@ dependencies:
         version: 0.76.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.37.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.37.0
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -256,10 +270,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -296,6 +310,11 @@ files:
 - gemfiles/rails_5.2.gemfile.lock
 - gemfiles/rails_6.0.gemfile
 - gemfiles/rails_6.0.gemfile.lock
+- gemfiles/semantic_logger_3.4.gemfile
+- gemfiles/semantic_logger_4.6.gemfile
+- gemfiles/semantic_logger_4.7.0.gemfile
+- gemfiles/semantic_logger_4.7.2.gemfile
+- gemfiles/semantic_logger_4.7.gemfile
 - lib/cloudtasker.rb
 - lib/cloudtasker/authentication_error.rb
 - lib/cloudtasker/authenticator.rb
@@ -320,6 +339,7 @@ files:
 - lib/cloudtasker/engine.rb
 - lib/cloudtasker/invalid_worker_error.rb
 - lib/cloudtasker/local_server.rb
+- lib/cloudtasker/max_task_size_exceeded_error.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
 - lib/cloudtasker/redis_client.rb
@@ -367,8 +387,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 3.0.0
 signing_key: 
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)

data/.travis.yml

@@ -1,16 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.5.5
-services:
-  - redis-server
-before_install: gem install bundler -v 2.0.2
-before_script: bundle exec rubocop
-gemfile:
-  - gemfiles/google_cloud_tasks_1.0.gemfile
-  - gemfiles/google_cloud_tasks_1.1.gemfile
-  - gemfiles/google_cloud_tasks_1.2.gemfile
-  - gemfiles/google_cloud_tasks_1.3.gemfile
-  - gemfiles/rails_5.2.gemfile
-  - gemfiles/rails_6.0.gemfile