bridge_cache 0.3.14 → 0.3.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eea899b543665aad075f716c90a25364439044cbd691ab481b084729bf915a56
-  data.tar.gz: 1dac6755245cb45fa1c0f8923e56ff6fd01eb6dc5328abed7c28e3dffa476f67
+  metadata.gz: c515d7e48aed8e9e9d05b6cec9d956074704206846744fca065335b23a389609
+  data.tar.gz: 30c0dee61a1f5dd8120a2c495ad8b625fe7e1da3ba270bba101b64a6be3987ec
 SHA512:
-  metadata.gz: 8211bf6a5a48b13b50822a451c150167a718d96f994550ff69c9fca71f6d3f493cffab97d7db53109d89f4ff90f0646e94b15c0ef510567b8f4e71a665cf48e9
-  data.tar.gz: c3779fc81eb4a86ddde1a8ae13e7c9ba3162c7e773df8c53a1802170100725a5bfaf3a2c99d92b56e4a38c6e4e89d93a7c56b5ad9795b50f64c27a2e7ad35edf
+  metadata.gz: 908acbd97a28b594ae5295900064a94cf8a905c7ef19ef066b7e92e11c39baa86297cee0704903dc70c5bdee19b68195c182322a85e690b34ba9b0d1c640ea62
+  data.tar.gz: b6ad05dc8cf847bfbbea1d262abea61131368b865827e316055a263bf3e23114875f6c78bbe0c6976c241d6833b2a586dc61b6624344b9adf77cf877f02ebddb

data/Rakefile

@@ -14,7 +14,7 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-APP_RAKEFILE = File.expand_path('../test/dummy/Rakefile', __FILE__)
+APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
 load 'rails/tasks/engine.rake'
 
 load 'rails/tasks/statistics.rake'

data/app/lib/bridge_cache/data/bridge_model.rb

@@ -1,92 +1,100 @@
-module BridgeCache::Data
-  module BridgeModel
-    EXCLUDED_COLUMNS = Set['id', 'created_at', 'updated_at']
-    OVERRIDDEN_COLUMNS = { 'bridge_created_at' => 'created_at'.freeze, 'bridge_updated_at' => 'updated_at'.freeze }
+# rubocop:disable Naming/PredicateName,Metrics/MethodLength
+module BridgeCache
+  module Data
+    module BridgeModel
+      EXCLUDED_COLUMNS = Set['id', 'created_at', 'updated_at']
+      OVERRIDDEN_COLUMNS = { 'bridge_created_at' => 'created_at'.freeze,
+                             'bridge_updated_at' => 'updated_at'.freeze }.freeze
+
+      def adjusted_table_name
+        if BridgeCache.use_internal_database
+          table_name
+        else
+          to_s.demodulize.underscore.pluralize
+        end
+      end
 
-    def adjusted_table_name
-      if BridgeCache.use_internal_database
-        table_name
-      else
-        to_s.demodulize.underscore.pluralize
+      def self.extended(base)
+        base.class_eval do
+          scope :in_domain, ->(domain_id) { for_domain(domain_id) }
+          scope :active, -> { is_active? }
+          self.table_name = adjusted_table_name
+        end
       end
-    end
 
-    def self.extended(base)
-      base.class_eval do
-        scope :in_domain, ->(domain_id) { for_domain(domain_id) }
-        scope :active, -> { is_active? }
-        self.table_name = adjusted_table_name
+      def format_import_row(row)
+        row
       end
-    end
 
-    def format_import_row(row)
-      row
-    end
+      def import_from_csv(file_path)
+        BridgeCache::Plugins::CSVDump.dump_to_table(self, file_path)
+      end
 
-    def import_from_csv(file_path)
-      BridgeCache::Plugins::CSVDump.dump_to_table(self, file_path)
-    end
+      def create_from_csv_row(row)
+        BridgeCache::Plugins::CSVDump.dump_row(self, row)
+      end
 
-    def create_from_csv_row(row)
-      BridgeCache::Plugins::CSVDump.dump_row(self, row)
-    end
+      def cleanup(row_ids)
+        # Implement this method in your model if you want to do any sort of post creation cleanup.
+        # See tagging.rb for an example.
+      end
 
-    def cleanup(row_ids)
-      # Implement this method in your model if you want to do any sort of post creation cleanup.
-      # See tagging.rb for an example.
-    end
+      def webhook_updated(message)
+        create_or_update_from_webhook(message)
+      end
 
-    def webhook_updated(message)
-      create_or_update_from_webhook(message)
-    end
+      def webhook_created(message)
+        create_or_update_from_webhook(message)
+      end
 
-    def webhook_created(message)
-      create_or_update_from_webhook(message)
-    end
+      def webhook_deleted(message)
+        message.payload_class.where(bridge_id: message.resource_object['id']).destroy_all
+      end
 
-    def webhook_deleted(message)
-      message.payload_class.where(bridge_id: message.resource_object['id']).destroy_all
-    end
+      def webhook_completed(_message)
+        raise 'Method not implemented'
+      end
 
-    def webhook_completed(_message)
-      raise 'Method not implemented'
-    end
+      def for_domain(domain_id)
+        where(domain_id: domain_id)
+      end
 
-    def for_domain(domain_id)
-      where(domain_id: domain_id)
-    end
+      def uniq_constraint_name
+        "#{table_name.downcase}_bridge_uniq"
+      end
 
-    def uniq_constraint_name
-      "#{table_name.downcase}_bridge_uniq"
-    end
+      def unique_column_names
+        %w[bridge_id]
+      end
 
-    def unique_column_names
-      %w[bridge_id]
-    end
+      def is_active?
+        where(deleted_at: nil) if column_names.include? 'deleted_at'
+      end
 
-    def is_active?
-      where(deleted_at: nil) if column_names.include? 'deleted_at'
-    end
+      def csv_mapping
+        columns = column_names.as_json
+        mapping = {}
 
-    def csv_mapping
-      columns = column_names.as_json
-      mapping = {}
-      columns.each do |column|
-        next if EXCLUDED_COLUMNS.include? column
-        if OVERRIDDEN_COLUMNS.key?(column)
-          mapping[OVERRIDDEN_COLUMNS[column]] = column
-        else
-          mapping[column] = column
+        columns.each do |column|
+          next if EXCLUDED_COLUMNS.include? column
+
+          if OVERRIDDEN_COLUMNS.key?(column)
+            mapping[OVERRIDDEN_COLUMNS[column]] = column
+          else
+            mapping[column] = column
+          end
         end
+
+        mapping['bridge_id'] = 'bridge_id'
+        mapping
       end
-      mapping['bridge_id'] = 'bridge_id'
-      mapping
-    end
 
-    private
+      private
 
-    def create_or_update_from_webhook(message)
-      message.payload_class.create_from_csv_row(message.resource_object)
+      def create_or_update_from_webhook(message)
+        message.payload_class.create_from_csv_row(message.resource_object)
+      end
     end
   end
 end
+# rubocop:enable Naming/PredicateName,Metrics/MethodLength

data/app/lib/bridge_cache/jobs/cleanup_job.rb

@@ -1,7 +1,9 @@
-module BridgeCache::Jobs
-  class CleanupJob < ActiveJob::Base
-    def perform(model, row_ids)
-      "BridgeCache::#{model.camelcase}".constantize.cleanup(row_ids)
+module BridgeCache
+  module Jobs
+    class CleanupJob < ActiveJob::Base
+      def perform(model, row_ids)
+        "BridgeCache::#{model.camelcase}".constantize.cleanup(row_ids)
+      end
     end
   end
 end

data/app/lib/bridge_cache/jobs/import_check.rb

@@ -1,22 +1,38 @@
-module BridgeCache::Jobs
-  class ImportCheck < ActiveJob::Base
-    queue_as :default
-    include BridgeCache::Utils::WorkerHelper
+# rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+module BridgeCache
+  module Jobs
+    class ImportCheck < ActiveJob::Base
+      queue_as :default
+      include BridgeCache::Utils::WorkerHelper
 
-    ATTEMPTS_THRESHOLD = 30
+      def max_import_attempts
+        ENV['BRIDGE_CACHE_MAX_IMPORT_ATTEMPTS'].to_i || 30
+      end
+
+      def remote_data
+        @remote_data ||= BridgeBlueprint::RemoteData.new(@account_settings[:url], @account_settings[:api_key],
+                                                         @account_settings[:api_secret])
+      end
+
+      def perform(account_settings, attempts, chain = [], retries = 0)
+        @account_settings = account_settings.with_indifferent_access
+
+        if attempts > max_import_attempts
+          Rails.logger.fatal("Bridge data dump failed to download after #{(max_import_attempts * 30) / 60} minutes")
+        elsif remote_data.status == BridgeBlueprint::Constants::STATUS_COMPLETE
+          BridgeCache::Jobs::ImportData.set(queue: queue_name).perform_later(@account_settings, chain)
+        elsif remote_data.status == BridgeBlueprint::Constants::STATUS_PENDING
+          BridgeCache::Jobs::ImportCheck.set(queue: queue_name, wait: 30.seconds).perform_later(@account_settings,
+                                                                                                attempts + 1, chain)
+        elsif remote_data.status == BridgeBlueprint::Constants::STATUS_FAILED ||
+              remote_data.status == BridgeBlueprint::Constants::STATUS_NOT_FOUND
+
+          raise 'Bridge data dump download failed' if retries < 1
 
-    def perform(account_settings, attempts, chain = [])
-      account_settings = account_settings.with_indifferent_access
-      remote_data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key], account_settings[:api_secret])
-      if attempts > ATTEMPTS_THRESHOLD
-        Rails.logger.fatal("Bridge data dump failed to download after #{(ATTEMPTS_THRESHOLD * 30) / 60} minutes")
-      elsif remote_data.status == BridgeBlueprint::Constants::STATUS_COMPLETE
-        BridgeCache::Jobs::ImportData.set(queue: queue_name).perform_later(account_settings, chain)
-      elsif remote_data.status == BridgeBlueprint::Constants::STATUS_PENDING
-        BridgeCache::Jobs::ImportCheck.set(queue: queue_name, wait: 30.seconds).perform_later(account_settings, attempts + 1, chain)
-      elsif remote_data.status == BridgeBlueprint::Constants::STATUS_FAILED || remote_data.status == BridgeBlueprint::Constants::STATUS_NOT_FOUND
-        raise 'Bridge data dump download failed'
+          BridgeCache::Jobs::ImportStart.set(queue: queue_name).perform_later(@account_settings, chain, retries - 1)
+        end
       end
     end
   end
 end
+# rubocop:enable Metrics/AbcSize,Metrics/MethodLength

data/app/lib/bridge_cache/jobs/import_data.rb

@@ -1,23 +1,30 @@
-module BridgeCache::Jobs
-  class ImportData < ActiveJob::Base
-    include BridgeCache::Utils::WorkerHelper
-    queue_as :default
+# rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+module BridgeCache
+  module Jobs
+    class ImportData < ActiveJob::Base
+      include BridgeCache::Utils::WorkerHelper
+      queue_as :default
 
-    def perform(account_settings, chain = [])
-      account_settings = account_settings.with_indifferent_access
-      remote_data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key], account_settings[:api_secret])
-      Dir.mktmpdir do |dir|
-        path = "#{dir}/data_dump"
-        remote_data.store_file(path)
-        import = BridgeBlueprint::DataDump.new(path)
-        models = account_settings[:models] || BridgeCache.resolved_models(account_settings[:is_subaccount])
-        models.each do |model|
-          ids = []
-          ids << BridgeCache::Plugins::CSVDump.bulk_import(import, model)
-          BridgeCache::Jobs::CleanupJob.set(queue: queue_name).perform_later(model, ids)
+      def perform(account_settings, chain = [])
+        account_settings = account_settings.with_indifferent_access
+        remote_data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key],
+                                                      account_settings[:api_secret])
+        Dir.mktmpdir do |dir|
+          path = "#{dir}/data_dump"
+          remote_data.store_file(path)
+          import = BridgeBlueprint::DataDump.new(path)
+          models = account_settings[:models] || BridgeCache.resolved_models(account_settings[:is_subaccount])
+
+          models.each do |model|
+            ids = []
+            ids << BridgeCache::Plugins::CSVDump.bulk_import(import, model)
+            BridgeCache::Jobs::CleanupJob.set(queue: queue_name).perform_later(model, ids)
+          end
         end
+
+        BridgeCache::Utils::WorkerHelper.invoke_next(chain)
       end
-      BridgeCache::Utils::WorkerHelper.invoke_next(chain)
     end
   end
 end
+# rubocop:enable Metrics/AbcSize,Metrics/MethodLength

data/app/lib/bridge_cache/jobs/import_row.rb

@@ -1,9 +1,11 @@
-module BridgeCache::Jobs
-  class ImportRow < ActiveJob::Base
-    queue_as :default
+module BridgeCache
+  module Jobs
+    class ImportRow < ActiveJob::Base
+      queue_as :default
 
-    def perform(model, row)
-      "BridgeCache::#{model.camelcase}".constantize.create_from_csv_row(row)
+      def perform(model, row)
+        "BridgeCache::#{model.camelcase}".constantize.create_from_csv_row(row)
+      end
     end
   end
 end

data/app/lib/bridge_cache/jobs/import_start.rb

@@ -1,17 +1,17 @@
-module BridgeCache::Jobs
-  class ImportStart < ActiveJob::Base
-    queue_as :default
+module BridgeCache
+  module Jobs
+    class ImportStart < ActiveJob::Base
+      queue_as :default
 
-    def perform(account_settings, chain = [])
-      data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key], account_settings[:api_secret])
-      data.start_data_report
-      BridgeCache::Jobs::ImportCheck
-        .set(queue: queue_name, wait: 30.seconds)
-        .perform_later(
-          account_settings,
-          0,
-          chain
-        )
+      def perform(account_settings, chain = [], retries = 0)
+        data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key],
+                                               account_settings[:api_secret])
+        data.start_data_report
+
+        BridgeCache::Jobs::ImportCheck
+          .set(queue: queue_name, wait: 30.seconds)
+          .perform_later(account_settings, 0, chain, retries)
+      end
     end
   end
 end

data/app/lib/bridge_cache/plugins/csv_dump.rb

@@ -1,97 +1,107 @@
 require 'csv'
 
-module BridgeCache::Plugins
-  class CSVDump
-    MAX_ROW_INTERVAL = 5000
-
-    def self.bulk_import(iterator, model)
-      ids = []
-      rows = []
-      klass = "BridgeCache::#{model.camelcase}".constantize
-
-      csv_column_names = klass.csv_mapping.keys
-      database_column_names = klass.csv_mapping.values
-
-      iterator.each_row(model.pluralize) do |row|
-        row = remove_bad_columns(klass, BridgeCache::Plugins::DataTransform.set_bridge_id(row).to_h)
-        row = klass.format_import_row(row)
-        rows << csv_column_names.map { |column| row[column] }
-        ids << row['bridge_id'] if row['bridge_id'].present? # Some CSV's do not have an ID column
-
-        if rows.length >= BridgeCache.batch_size
-          perform_bulk_import(klass, database_column_names, rows)
-          rows = []
+# rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+module BridgeCache
+  module Plugins
+    class CSVDump
+      MAX_ROW_INTERVAL = 5000
+
+      def self.bulk_import(iterator, model)
+        ids = []
+        rows = []
+        klass = "BridgeCache::#{model.camelcase}".constantize
+
+        csv_column_names = klass.csv_mapping.keys
+        database_column_names = klass.csv_mapping.values
+
+        iterator.each_row(model.pluralize) do |row|
+          row = remove_bad_columns(klass, BridgeCache::Plugins::DataTransform.set_bridge_id(row).to_h)
+          row = klass.format_import_row(row)
+          rows << csv_column_names.map { |column| row[column] }
+          ids << row['bridge_id'] if row['bridge_id'].present? # Some CSV's do not have an ID column
+
+          if rows.length >= BridgeCache.batch_size
+            perform_bulk_import(klass, database_column_names, rows)
+            rows = []
+          end
         end
+
+        perform_bulk_import(klass, database_column_names, rows)
+        ids
       end
 
-      perform_bulk_import(klass, database_column_names, rows)
-      ids
-    end
+      def self.dump_to_table(clazz, file_path)
+        count = 1
+        total = 0
+        rows = []
 
-    def self.dump_to_table(clazz, file_path)
-      count = 1
-      total = 0
-      rows = []
-      CSV.foreach(file_path, headers: true) do |_row|
-        total += 1
-      end
-      CSV.foreach(file_path, headers: true) do |row|
-        rows << initialze_row(clazz, row) if count < MAX_ROW_INTERVAL
-        if count % MAX_ROW_INTERVAL == 0 || count == total
-          dump_rows(rows)
-          count = 0
-          rows = []
+        CSV.foreach(file_path, headers: true) do |_row|
+          total += 1
+        end
+
+        CSV.foreach(file_path, headers: true) do |row|
+          rows << initialze_row(clazz, row) if count < MAX_ROW_INTERVAL
+
+          if (count % MAX_ROW_INTERVAL).zero? || count == total
+            dump_rows(rows)
+
+            count = 0
+            rows = []
+          end
+
+          count += 1
         end
-        count += 1
       end
-    end
 
-    def self.dump_row(clazz, row)
-      instance = initialze_row(clazz, row)
-      dump_rows([instance])
-    end
+      def self.dump_row(clazz, row)
+        instance = initialze_row(clazz, row)
+        dump_rows([instance])
+      end
 
-    def self.perform_bulk_import(klass, columns, rows)
-      return if rows.empty?
-      columns = columns.dup
-      klass.import(columns, rows, validate: false, on_duplicate_key_update: {
-                     conflict_target: klass.unique_column_names,
-                     condition: condition_sql(klass, columns),
-                     columns: columns
-                   })
-    end
+      def self.perform_bulk_import(klass, columns, rows)
+        return if rows.empty?
 
-    # This method generates SQL that looks like:
-    # (users.sis_id, users.email) IS DISTINCT FROM (EXCLUDED.sis_id, EXCLUDED.email)
-    #
-    # This prevents activerecord-import from setting the `updated_at` column for
-    # rows that haven't actually changed. This allows you to query for rows that have changed
-    # by doing something like:
-    #
-    # started_at = Time.now
-    # run_the_users_sync!
-    # changed = User.where("updated_at >= ?", started_at)
-    def self.condition_sql(klass, columns)
-      columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(', ')
-      excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(', ')
-      "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
-    end
+        columns = columns.dup
+        klass.import(columns, rows, validate: false, on_duplicate_key_update: {
+                       conflict_target: klass.unique_column_names,
+                       condition: condition_sql(klass, columns),
+                       columns: columns
+                     })
+      end
 
-    def self.initialze_row(clazz, row)
-      instance = clazz.find_or_create_by(bridge_id: row['id'])
-      instance.assign_attributes(remove_bad_columns(clazz, BridgeCache::Plugins::DataTransform.set_bridge_id(row).to_h))
-      instance
-    end
+      # This method generates SQL that looks like:
+      # (users.sis_id, users.email) IS DISTINCT FROM (EXCLUDED.sis_id, EXCLUDED.email)
+      #
+      # This prevents activerecord-import from setting the `updated_at` column for
+      # rows that haven't actually changed. This allows you to query for rows that have changed
+      # by doing something like:
+      #
+      # started_at = Time.now
+      # run_the_users_sync!
+      # changed = User.where("updated_at >= ?", started_at)
+      def self.condition_sql(klass, columns)
+        columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(', ')
+        excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(', ')
+        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+      end
 
-    def self.dump_rows(rows)
-      rows.each do |row|
-        row.save! if row.changed?
+      def self.initialze_row(clazz, row)
+        instance = clazz.find_or_create_by(bridge_id: row['id'])
+        instance.assign_attributes(remove_bad_columns(clazz,
+                                                      BridgeCache::Plugins::DataTransform.set_bridge_id(row).to_h))
+        instance
+      end
+
+      def self.dump_rows(rows)
+        rows.each do |row|
+          row.save! if row.changed?
+        end
       end
-    end
 
-    def self.remove_bad_columns(clazz, row)
-      row = row.delete_if { |key, _value| !clazz.column_names.include?(key) }
-      row
+      def self.remove_bad_columns(clazz, row)
+        row.delete_if { |key, _value| !clazz.column_names.include?(key) }
+      end
     end
   end
 end
+# rubocop:enable Metrics/MethodLength,Metrics/AbcSize