asyncapi-client 0.6.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2c4aa942b90ab8794c0eaeb572810d4277581df5
-  data.tar.gz: 186bd9cf4144c883d8f532e18caa8bd5143c2755
+SHA256:
+  metadata.gz: 5fd1ef047e069d51524a6d6fa1a935dd21291f9287314f9c0cecedd06a4443be
+  data.tar.gz: 2103154fa8afb427645b952aa63e1503819ea961d38d0899e5efe5c01ed4b972
 SHA512:
-  metadata.gz: 6553230b72635076b4fee02bb0a47fd04b50821c6ddae57f3819bc353f33289c151eadbfe5fbcd8be0a58cb8e36f07d28209e7684ffee3aef7c6c82876c5e1a1
-  data.tar.gz: a4358536d0282459c8830a2009b761d60d008ea5dfdfa6c7be598edd75c5effa8629b18ccfd583c18f323b24974c2beedbeab90caf94c4a6c28f065e0e66ed0a
+  metadata.gz: a012d3e92b0955d0d35e520441eb2d17798740366c410e319a33f595826ab1368ceb197a4ae27995d7610b2aceb5a644820b96488b0551203506e1f535a36d65
+  data.tar.gz: 41a1a7a745dc6e67fb9097dedb1f97c4f3cf03e3ab97378567f0bbc7023bf7283cc83fd36fef2c0a6f6ebc08a556f9bee1eaf48200d13123038faa6de90c2bf2

data/README.md

@@ -73,9 +73,11 @@ There is a feed of all jobs that can be accessed via `/asyncapi/client/v1/jobs.j
 
 ## Expiry
 
-To make space in the database, old jobs must be deleted. By default, jobs older than 10 days will be deleted in both the Asyncapi Client and Asyncapi Server. Asyncapi Client is responsible for deleting the jobs it no longer needs a response from on the server.
+To make space in the database, old jobs must be deleted. By default, jobs older than 4 days will be deleted in both the Asyncapi Client and Asyncapi Server. Asyncapi Client is responsible for deleting the jobs it no longer needs a response from on the server.
 
-By default, jobs 10 days old and older will be deleted. You can change this setting by putting this in an initializer:
+**Important:** keep in mind that this setting may conflict with the `successful_jobs_deletion_after` setting with jobs in `success` state, which normally are deleted after they reach this state (see [Successful jobs automatic deletion](#successful-jobs-automatic-deletion) for more information).
+
+By default, jobs 4 days old and older will be deleted. You can change this setting by putting this in an initializer:
 
 ```ruby
 Asyncapi::Client.expiry_threshold = 5.days
@@ -89,6 +91,16 @@ The cleaner job is run every day at "0 0 * * *". If you want to change the frequ
 Asyncapi::Client.clean_job_cron = "30 2 * * *"
 ```
 
+## Successful jobs automatic deletion
+
+After a job completes successfully, we schedule another job to delete it after a brief period of time to avoid having a lot of records on the table. By default, a successful job is deleted from the table after 2 minutes, ignoring the `expiry_threshold` configuration option. You can change this setting using the `successful_jobs_deletion_after` configuration option in an initializer if you want to extend or shorten the wait time for deletion:
+
+```ruby
+Asyncapi::Client.successful_jobs_deletion_after = 3.days
+```
+
+**Important:** if you want to ignore `successful_jobs_deletion_after` setting in favor of `expiry_threshold`, set `successful_jobs_deletion_after` to a value greater than `expiry_threshold`.
+
 # Installation
 
 **Note**: if you're using the `protected_attributes`, also see the "Optional" section below.
@@ -156,7 +168,7 @@ require "asyncapi/client/factories"
 # Development
 
 ```
-rake app:db:schema:load
+rake db:migrate && rake db:migrate RAILS_ENV=test
 rspec spec
 ```
 

data/app/controllers/asyncapi/client/v1/jobs_controller.rb

@@ -1,6 +1,7 @@
 module Asyncapi::Client
   module V1
     class JobsController < Asyncapi::Client.parent_controller
+      include Rails::Pagination
 
       def index
         jobs = Job.all
@@ -12,7 +13,7 @@ module Asyncapi::Client
           UpdateJob.execute(job: job, params: job_params)
           render json: job
         else
-          render nothing: true, status: 403
+          head :forbidden
         end
       end
 

data/app/models/asyncapi/client/job.rb

@@ -21,7 +21,7 @@ module Asyncapi::Client
         transitions from: :fresh, to: :queued
       end
 
-      event :succeed do
+      event :succeed, after: :schedule_for_deletion do
         transitions from: :queued, to: :success
       end
 
@@ -40,6 +40,10 @@ module Asyncapi::Client
 
     scope :expired, -> { where(arel_table[:expired_at].lt(Time.now)) }
     scope :with_time_out, -> { where(arel_table[:time_out_at].not_eq(nil)) }
+    scope :stale, -> (stale_duration = 5) do
+      where(arel_table[:updated_at].lteq(stale_duration.minutes.ago)).
+      where(status: statuses[:queued])
+    end
     scope :for_time_out, -> do
       where(arel_table[:time_out_at].lt(Time.now)).
       where(status: [statuses[:queued], statuses[:fresh]])
@@ -70,18 +74,15 @@ module Asyncapi::Client
       }
       args[:time_out_at] = time_out.from_now if time_out
       job = create(args)
-      JobPostWorker.perform_async(job.id, url)
+      ActiveRecord::Base.after_transaction do
+        JobPostWorker.perform_async(job.id, url)
+      end
     end
 
     def url
       Asyncapi::Client::Engine.routes.url_helpers.v1_job_url(self)
     end
 
-    def body=(body)
-      json = body.is_a?(Hash) ? body.to_json : body
-      write_attribute :body, json
-    end
-
     [:on_success, :on_error, :on_queue, :on_time_out, :on_queue_error].each do |attr|
       define_method("#{attr}=") do |klass|
         write_attribute attr, klass.to_s
@@ -90,6 +91,13 @@ module Asyncapi::Client
 
     private
 
+    def schedule_for_deletion
+      if success?
+        # delete in a couple of minutes, giving time for the success to be broadcasted
+        JobCleanerWorker.perform_in(Asyncapi::Client.successful_jobs_deletion_after, self.id)
+      end
+    end
+
     def set_expired_at
       self.expired_at ||= Asyncapi::Client.expiry_threshold.from_now
     end

data/app/services/asyncapi/client/update_job.rb

@@ -9,7 +9,9 @@ module Asyncapi::Client
       if may_transition?(job, to: status)
         transition(job, to: status)
         if job.status_changed? && job.save
-          JobStatusWorker.perform_async(job.id)
+          ActiveRecord::Base.after_transaction do
+            JobStatusWorker.perform_async(job.id)
+          end
         else
           job.save
         end

data/app/workers/asyncapi/client/cleaner_worker.rb

@@ -6,9 +6,7 @@ module Asyncapi
       sidekiq_options retry: false
 
       def perform
-        Job.expired.find_each do |job|
-          JobCleanerWorker.perform_async(job.id)
-        end
+        Job.expired.pluck(:id).each{ |job_id| JobCleanerWorker.perform_async(job_id) }
       end
 
     end

data/app/workers/asyncapi/client/job_cleaner_worker.rb

@@ -7,7 +7,7 @@ module Asyncapi
 
       def perform(job_id)
         if job = Job.find_by(id: job_id)
-          destroy_remote job
+          destroy_remote(job)
           job.destroy
         end
       end
@@ -15,12 +15,46 @@ module Asyncapi
       private
 
       def destroy_remote(job)
-        Typhoeus.delete(job.server_job_url, {
-          params: { secret: job.secret },
-          headers: job.headers,
-        })
+        errors = validate_remote_job_info(job)
+        if errors.empty?
+          Typhoeus.delete(job.server_job_url, {
+            params: { secret: job.secret },
+            headers: job.headers,
+          })
+        else
+          log_remote_error_for(job, errors)
+        end
+      end
+
+      def validate_remote_job_info(job)
+        errors = []
+        errors << "server_job_url is invalid"         unless url_ok_for?(job)
+        errors << "authorization headers are not present" unless headers_ok_for?(job)
+        errors << "secret is not present"                unless job.secret.present?
+        errors
+      end
+
+      def url_ok_for?(job)
+        uri = URI.parse(job.server_job_url)
+        uri.is_a?(URI::HTTP) && !uri.host.nil?
+      rescue URI::InvalidURIError
+        false
       end
 
+      def headers_ok_for?(job)
+        job.headers.is_a?(Hash) && job.headers[:AUTHORIZATION]
+      end
+
+      def log_remote_error_for(job, errors)
+        if defined?(G5::Logger::Log)
+          G5::Logger::Log.send(:warn, {
+            origin: "#{self.class.name}#destroy_remote",
+            external_parent_id: "#{job.id}",
+            message: "Not enough info to delete expired remote job: #{errors.join(", ")}",
+            error: "Unable to delete remote job",
+          })
+        end
+      end
     end
   end
 end

data/app/workers/asyncapi/client/job_post_worker.rb

@@ -21,12 +21,16 @@ module Asyncapi::Client
         job.assign_attributes(job_params_from(response))
         job.enqueue
         if job.save!
-          JobStatusWorker.perform_async(job.id)
+          ActiveRecord::Base.after_transaction do
+            JobStatusWorker.perform_async(job.id)
+          end
         end
       else
         job.fail_queue
         if job.update_attributes!(message: response.body, response_code: response.response_code)
-          JobStatusWorker.perform_async(job.id)
+          ActiveRecord::Base.after_transaction do
+            JobStatusWorker.perform_async(job.id)
+          end
         end
       end
     end
@@ -38,6 +42,7 @@ module Asyncapi::Client
     end
 
     def server_params_from(job, params)
+      params = params.to_json if params.is_a? Hash
       {
         job: {
           callback_url: job.url,

data/app/workers/asyncapi/client/job_time_out_worker.rb

@@ -12,8 +12,10 @@ module Asyncapi::Client
     private
 
     def time_out_job(job)
-      job.update_attributes(status: :timed_out)
-      JobStatusWorker.perform_async(job.id)
+      job.update(status: :timed_out)
+      ActiveRecord::Base.after_transaction do
+        JobStatusWorker.perform_async(job.id)
+      end
     end
 
   end

data/db/migrate/20141104030959_create_asyncapi_client_jobs.rb

@@ -1,4 +1,4 @@
-class CreateAsyncapiClientJobs < ActiveRecord::Migration
+class CreateAsyncapiClientJobs < ActiveRecord::Migration[4.2]
   def change
     create_table :asyncapi_client_jobs do |t|
       t.string   "server_job_url"

data/db/migrate/20141119011011_add_callback_params_to_asyncapi_client_jobs.rb

@@ -1,4 +1,4 @@
-class AddCallbackParamsToAsyncapiClientJobs < ActiveRecord::Migration
+class AddCallbackParamsToAsyncapiClientJobs < ActiveRecord::Migration[4.2]
   def change
     add_column :asyncapi_client_jobs, :callback_params, :text
   end

data/db/migrate/20141212064041_add_secret_to_asyncapi_client_job.rb

@@ -1,4 +1,4 @@
-class AddSecretToAsyncapiClientJob < ActiveRecord::Migration
+class AddSecretToAsyncapiClientJob < ActiveRecord::Migration[4.2]
   def change
     add_column :asyncapi_client_jobs, :secret, :string
   end

data/db/migrate/20150202062211_add_expired_at_to_asyncapi_client_job.rb

@@ -1,4 +1,4 @@
-class AddExpiredAtToAsyncapiClientJob < ActiveRecord::Migration
+class AddExpiredAtToAsyncapiClientJob < ActiveRecord::Migration[4.2]
   def change
     add_column :asyncapi_client_jobs, :expired_at, :datetime
   end

data/db/migrate/20150202062320_populate_asyncapi_client_job_expired_at.rb

@@ -1,4 +1,4 @@
-class PopulateAsyncapiClientJobExpiredAt < ActiveRecord::Migration
+class PopulateAsyncapiClientJobExpiredAt < ActiveRecord::Migration[4.2]
   def change
     Asyncapi::Client::Job.find_each do |job|
       job.update_attributes(expired_at: Asyncapi::Client.expiry_threshold.from_now)

data/db/migrate/20150610053320_add_on_time_out_to_asyncapi_client_job.rb

@@ -1,4 +1,4 @@
-class AddOnTimeOutToAsyncapiClientJob < ActiveRecord::Migration
+class AddOnTimeOutToAsyncapiClientJob < ActiveRecord::Migration[4.2]
   def change
     add_column :asyncapi_client_jobs, :on_time_out, :string
   end

data/db/migrate/20150612082965_add_time_out_index_to_asyncapi_client_job.rb

@@ -1,4 +1,4 @@
-class AddTimeOutIndexToAsyncapiClientJob < ActiveRecord::Migration
+class AddTimeOutIndexToAsyncapiClientJob < ActiveRecord::Migration[4.2]
   def change
     add_index :asyncapi_client_jobs, :time_out_at
   end

data/db/migrate/20150630004215_add_response_code_to_asyncapi_client_jobs.rb

@@ -1,4 +1,4 @@
-class AddResponseCodeToAsyncapiClientJobs < ActiveRecord::Migration
+class AddResponseCodeToAsyncapiClientJobs < ActiveRecord::Migration[4.2]
   def change
     add_column :asyncapi_client_jobs, :response_code, :integer
   end

data/db/migrate/20150703001225_add_on_queue_error_to_asyncapi_client_jobs.rb

@@ -1,4 +1,4 @@
-class AddOnQueueErrorToAsyncapiClientJobs < ActiveRecord::Migration
+class AddOnQueueErrorToAsyncapiClientJobs < ActiveRecord::Migration[4.2]
   def change
     add_column :asyncapi_client_jobs, :on_queue_error, :string
   end

data/db/migrate/20190218200630_add_expired_at_index_to_asyncapi_client_jobs.rb

@@ -0,0 +1,11 @@
+class AddExpiredAtIndexToAsyncapiClientJobs < ActiveRecord::Migration[5.0]
+  disable_ddl_transaction!
+
+  def change
+    opts = {}
+    if !!(ActiveRecord::Base.connection_config[:adapter] =~ /postgresql/i)
+      opts[:algorithm] = :concurrently
+    end
+    add_index :asyncapi_client_jobs, :expired_at, opts
+  end
+end

data/db/migrate/20190304200630_remove_expired_at_index_on_asyncapi_client_jobs.rb

@@ -0,0 +1,5 @@
+class RemoveExpiredAtIndexOnAsyncapiClientJobs < ActiveRecord::Migration[5.0]
+  def change
+    remove_index :asyncapi_client_jobs, :expired_at
+  end
+end

data/lib/asyncapi/client.rb

@@ -3,15 +3,17 @@ require "sidekiq-cron"
 require "api-pagination"
 require "typhoeus"
 require 'aasm'
+require 'ar_after_transaction'
 require "asyncapi/client/engine"
 require "securerandom"
 
 module Asyncapi
   module Client
 
-    CONFIG_ATTRS = %i[parent_controller expiry_threshold clean_job_cron]
+    CONFIG_ATTRS = %i[parent_controller expiry_threshold clean_job_cron successful_jobs_deletion_after]
     mattr_accessor(*CONFIG_ATTRS)
-    self.expiry_threshold = 10.days
+    self.expiry_threshold = 4.days
+    self.successful_jobs_deletion_after = 2.minutes
     self.clean_job_cron = "0 0 * * *"
 
     def self.configure

data/lib/asyncapi/client/version.rb

@@ -1,5 +1,5 @@
 module Asyncapi
   module Client
-    VERSION = "0.6.0"
+    VERSION = "0.11.0".freeze
   end
 end

metadata

@@ -1,31 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: asyncapi-client
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.11.0
 platform: ruby
 authors:
 - G5
 - Marc Ignacio
 - Ramon Tayag
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-26 00:00:00.000000000 Z
+date: 2021-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -111,19 +111,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: ar_after_transaction
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sprockets
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.2
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
@@ -253,6 +281,8 @@ files:
 - db/migrate/20150612082965_add_time_out_index_to_asyncapi_client_job.rb
 - db/migrate/20150630004215_add_response_code_to_asyncapi_client_jobs.rb
 - db/migrate/20150703001225_add_on_queue_error_to_asyncapi_client_jobs.rb
+- db/migrate/20190218200630_add_expired_at_index_to_asyncapi_client_jobs.rb
+- db/migrate/20190304200630_remove_expired_at_index_on_asyncapi_client_jobs.rb
 - lib/asyncapi-client.rb
 - lib/asyncapi/client.rb
 - lib/asyncapi/client/engine.rb
@@ -264,7 +294,7 @@ homepage: https://github.com/G5/asyncapi-client
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -279,9 +309,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: Asynchronous API communication
 test_files: []