cloudtasker 0.8.1 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3459c464d92cbd6e9c920a0f67a96ba143bf5746af3e8dde4fe13cade0a75ffb
-  data.tar.gz: 42313e898816af4d2eb670622569e1545e13eb54f2de1acaee2f4cddb923e101
+  metadata.gz: 4f2c05ceac2d50370d52f7135f242f949b264abfaec1f31cb755a2afac6d35dc
+  data.tar.gz: e64432a75782ebc4fa87edfea9b3f683f3a518a1680372a644b77b5c4df52839
 SHA512:
-  metadata.gz: 7ce1fcf26ec35f3e21a8f932f5f530c54a136193458944535097e84186c23fdf61306059fbe7449af38c3747ba48b0aad380cf706c3d4d27ffa70eae36c2f042
-  data.tar.gz: ad8c3a5529ef16e699a5db45594b19ee72eaaa3ff22315e463fc68e7548b961ce399ce7a44af784270cf7cac02b057cc5f31bb41188eabbb6e29224fbc0e87ef
+  metadata.gz: 7a3a8faf52c7a91e9739d867f4fda9786e7399942b4e341fcc297940bd6ed9c1b7810e3109c38b13e555db5532794af3f1f5aa0dc7c1384678e222d1162f5796
+  data.tar.gz: dc2ac72de941e8b8ba617b5f1cfc85a535102b86a4476af2d1c5c8bd93953e3773077714620f8593ada54f57675532233a5a244c912b9d36407bd51dfac02db6

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Test
+
+on:
+  push:
+    branches: [ master, 0.9-stable ]
+  pull_request:
+    branches: [ master, 0.9-stable ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby:
+          - '2.5.x'
+          - '2.6.x'
+        appraisal:
+          - 'google-cloud-tasks-1.0'
+          - 'google-cloud-tasks-1.1'
+          - 'google-cloud-tasks-1.2'
+          - 'google-cloud-tasks-1.3'
+          - 'rails-5.2'
+          - 'rails-6.0'
+    steps:
+      - name: Setup System
+        run: sudo apt-get install libsqlite3-dev
+      - uses: actions/checkout@v2
+      - uses: zhulik/redis-action@1.1.0
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        env:
+            APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rubocop
+          bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## [v0.9.3](https://github.com/keypup-io/cloudtasker/tree/v0.9.3) (2020-06-25)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.2...v0.9.3)
+
+**Fixed bugs:**
+- Google Cloud Tasks: lock version to `~> 1.0` (Google recently released a v2 which changes its bindings completely). An [issue](https://github.com/keypup-io/cloudtasker/issues/11) has been raised to upgrade Cloudtasker to `google-cloud-tasks` `v2`.
+
+## [v0.9.2](https://github.com/keypup-io/cloudtasker/tree/v0.9.2) (2020-03-04)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.1...v0.9.2)
+
+**Fixed bugs:**
+- Cloud Task: ignore "not found" errors when trying to delete an already deleted task.
+
+## [v0.9.1](https://github.com/keypup-io/cloudtasker/tree/v0.9.1) (2020-02-11)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.0...v0.9.1)
+
+**Fixed bugs:**
+- Cloud Task: raise `Cloudtasker::MaxTaskSizeExceededError` if job payload exceeds 100 KB. This is mainly to have production parity in development when running the local processing server.
+
+## [v0.9.0](https://github.com/keypup-io/cloudtasker/tree/v0.9.0) (2020-01-23)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.2...v0.9.0)
+
+**Fixed bugs:**
+- Cloud Task: Base64 encode task body to support UTF-8 characters (e.g. emojis).
+- Redis: Restrict to one connection (class level) to avoid too many DNS lookups
+
+**Migration**
+For Sinatra applications please update your Cloudtasker controller according to [this diff](https://github.com/keypup-io/cloudtasker/commit/311fa8f9beec91fbae012164a25b2ee6e261a2e4#diff-c2a0ea6c6e6c31c749d2e1acdc574f0f).
+
+## [v0.8.2](https://github.com/keypup-io/cloudtasker/tree/v0.8.2) (2019-12-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.1...v0.8.2)
+
+**Fixed bugs:**
+- Config: do not add processor host to `Rails.application.config.hosts` if originally empty.
+
+## [v0.8.1](https://github.com/keypup-io/cloudtasker/tree/v0.8.1) (2019-12-03)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.0...v0.8.1)
+
+**Fixed bugs:**
+- Local dev server: ensure job queue name is kept when taks is retried
+- Rails/Controller: bypass Rails munge logic to preserve nil values inside job arguments.
+
 ## [v0.8.0](https://github.com/keypup-io/cloudtasker/tree/v0.8.0) (2019-11-27)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.7.0...v0.8.0)

data/README.md

@@ -635,7 +635,9 @@ If you enqueue this worker by omitting the second argument `MyWorker.perform_asy
 - The `time_at` argument will be ignored by the `unique-job` extension, meaning that job uniqueness will be only based on the `user_id` argument.
 
 ### Handling big job payloads
-Keep in mind that jobs are pushed to Google Cloud Tasks via API and then delivered to your application via API as well. Therefore any excessive job payload will slow down the enqueuing of jobs and create additional processing when receiving the job.
+Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into accounts Cloudtasker authentication headers and meta information this leave ~85 KB of free space for JSONified job arguments.
+
+Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
 If you feel that a job payload is going to get big, prefer to store the payload using a datastore (e.g. Redis) and pass a reference to the job to retrieve the payload inside your job `perform` method.
 

data/app/controllers/cloudtasker/worker_controller.rb

@@ -16,9 +16,6 @@ module Cloudtasker
     # Run a worker from a Cloud Task payload
     #
     def run
-      # Build payload
-      payload = JSON.parse(request.body.read).merge(job_retries: job_retries)
-
       # Process payload
       WorkerHandler.execute_from_payload!(payload)
       head :no_content
@@ -37,6 +34,27 @@ module Cloudtasker
 
     private
 
+    #
+    # Parse the request body and return the actual job
+    # payload.
+    #
+    # @return [Hash] The job payload
+    #
+    def payload
+      @payload ||= begin
+        # Get raw body
+        content = request.body.read
+
+        # Decode content if the body is Base64 encoded
+        if request.headers[Cloudtasker::Config::ENCODING_HEADER].to_s.downcase == 'base64'
+          content = Base64.decode64(content)
+        end
+
+        # Return content parsed as JSON and add job retries count
+        JSON.parse(content).merge(job_retries: job_retries)
+      end
+    end
+
     #
     # Extract the number of times this task failed at runtime.
     #

data/cloudtasker.gemspec

@@ -32,17 +32,17 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport'
   spec.add_dependency 'fugit'
-  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'google-cloud-tasks', '~> 1.0'
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
 
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'github_changelog_generator'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '0.76.0'
-  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'rubocop-rspec', '1.37.0'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'webmock'
 

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'
 require 'cloudtasker/authenticator'

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -82,6 +82,29 @@ module Cloudtasker
         Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
       end
 
+      #
+      # Format the job payload sent to Cloud Tasks.
+      #
+      # @param [Hash] hash The worker payload.
+      #
+      # @return [Hash] The Cloud Task payloadd.
+      #
+      def self.format_task_payload(payload)
+        payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
+
+        # Format schedule time to Google Protobuf timestamp
+        payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
+
+        # Encode job content to support UTF-8. Google Cloud Task
+        # expect content to be ASCII-8BIT compatible (binary)
+        payload[:http_request][:headers] ||= {}
+        payload[:http_request][:headers][Cloudtasker::Config::CONTENT_TYPE_HEADER] = 'text/json'
+        payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
+        payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
+
+        payload
+      end
+
       #
       # Find a task by id.
       #
@@ -104,10 +127,7 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
       #
       def self.create(payload)
-        # Format payload
-        payload = payload.merge(
-          schedule_time: format_schedule_time(payload[:schedule_time])
-        ).compact
+        payload = format_task_payload(payload)
 
         # Extract relative queue name
         relative_queue = payload.delete(:queue)
@@ -126,7 +146,7 @@ module Cloudtasker
       #
       def self.delete(id)
         client.delete_task(id)
-      rescue Google::Gax::RetryError
+      rescue Google::Gax::RetryError, GRPC::NotFound
         nil
       end
 

data/lib/cloudtasker/cloud_task.rb

@@ -48,6 +48,8 @@ module Cloudtasker
     # @return [Cloudtasker::CloudTask] The created task.
     #
     def self.create(payload)
+      raise MaxTaskSizeExceededError if payload.to_json.bytesize > Config::MAX_TASK_SIZE
+
       resp = backend.create(payload)&.to_h
       resp ? new(resp) : nil
     end

data/lib/cloudtasker/config.rb

@@ -9,9 +9,21 @@ module Cloudtasker
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
+    # Max Cloud Task size in bytes
+    MAX_TASK_SIZE = 100 * 1024 # 100 KB
+
     # Retry header in Cloud Task responses
     RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
 
+    # Content-Transfer-Encoding header in Cloud Task responses
+    ENCODING_HEADER = 'Content-Transfer-Encoding'
+
+    # Content Type
+    CONTENT_TYPE_HEADER = 'Content-Type'
+
+    # Authorization header
+    AUTHORIZATION_HEADER = 'Authorization'
+
     # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
@@ -102,7 +114,10 @@ module Cloudtasker
       @processor_host = val
 
       # Check if Rails supports host filtering
-      return unless val && defined?(Rails) && Rails.application.config.respond_to?(:hosts)
+      return unless val &&
+                    defined?(Rails) &&
+                    Rails.application.config.respond_to?(:hosts) &&
+                    Rails.application.config.hosts&.any?
 
       # Add processor host to the list of authorized hosts
       Rails.application.config.hosts << val.gsub(%r{https?://}, '')

data/lib/cloudtasker/max_task_size_exceeded_error.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle Cloud Task size quota
+  # See: https://cloud.google.com/appengine/quotas#Task_Queue
+  #
+  class MaxTaskSizeExceededError < StandardError
+    MSG = 'The size of Cloud Tasks must not exceed 100KB'
+
+    def initialize(msg = MSG)
+      super
+    end
+  end
+end

data/lib/cloudtasker/redis_client.rb

@@ -8,13 +8,17 @@ module Cloudtasker
     # Suffix added to cache keys when locking them
     LOCK_KEY_PREFIX = 'cloudtasker/lock'
 
+    def self.client
+      @client ||= Redis.new(Cloudtasker.config.redis || {})
+    end
+
     #
     # Return the underlying redis client.
     #
     # @return [Redis] The redis client.
     #
     def client
-      @client ||= Redis.new(Cloudtasker.config.redis || {})
+      @client ||= self.class.client
     end
 
     #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.8.1'
+  VERSION = '0.9.3'
 end

data/lib/cloudtasker/worker_handler.rb

@@ -42,8 +42,8 @@ module Cloudtasker
           http_method: 'POST',
           url: Cloudtasker.config.processor_url,
           headers: {
-            'Content-Type' => 'application/json',
-            'Authorization' => "Bearer #{Authenticator.verification_token}"
+            Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
+            Cloudtasker::Config::AUTHORIZATION_HEADER => "Bearer #{Authenticator.verification_token}"
           },
           body: worker_payload.to_json
         },

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.3
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-03 00:00:00.000000000 Z
+date: 2020-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -42,16 +42,16 @@ dependencies:
   name: google-cloud-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jwt
   requirement: !ruby/object:Gem::Requirement
@@ -126,16 +126,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +168,16 @@ dependencies:
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -256,10 +256,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -320,6 +320,7 @@ files:
 - lib/cloudtasker/engine.rb
 - lib/cloudtasker/invalid_worker_error.rb
 - lib/cloudtasker/local_server.rb
+- lib/cloudtasker/max_task_size_exceeded_error.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
 - lib/cloudtasker/redis_client.rb
@@ -367,8 +368,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 3.0.0
 signing_key: 
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)

data/.travis.yml

@@ -1,16 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.5.5
-services:
-  - redis-server
-before_install: gem install bundler -v 2.0.2
-before_script: bundle exec rubocop
-gemfile:
-  - gemfiles/google_cloud_tasks_1.0.gemfile
-  - gemfiles/google_cloud_tasks_1.1.gemfile
-  - gemfiles/google_cloud_tasks_1.2.gemfile
-  - gemfiles/google_cloud_tasks_1.3.gemfile
-  - gemfiles/rails_5.2.gemfile
-  - gemfiles/rails_6.0.gemfile