bridge_cache 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: b65a0aeee23bdfaa1534a911ed6bffcebb63121fc21459950792bafdd7afacdb
-  data.tar.gz: 699d173dbfbaec7ef7c99e6d940dce75a5f3259d8b4ff4e526461f42e3a7e00e
+SHA1:
+  metadata.gz: f3c52f75722394bef7332035eb20cd3d7aeec83e
+  data.tar.gz: 85c863da50ae24857c04c7fbcbfb0b1a944eb13c
 SHA512:
-  metadata.gz: d9dd46fe8b82bb9143cd2ad974537b96b0f0297db2efd9f00269cfddc7ab8ff98dd1fda9c8f8c6153397b7d9aab8d214a6f949bc516f3a8da13f9952eee787ad
-  data.tar.gz: 627d7257a6630b936f946e3d12c36a2942d81047ed6f33c6992de0afa64356ce9d86edfd8ac496570e8fc1aa54471fd90b6bcd4c9dab453f72a80a0a2a9bd842
+  metadata.gz: af984e81a83c4237feedd11712b4e6020768a8c0a12668b6cda2288ef6bdb62ed1deea31457f3c13671b7e043d290a578b33e449cdbc13a1aa9ff4bc8c131197
+  data.tar.gz: a35ff760b4d414b480d7d8ebf8078a84865481f66735680219c569022ba1cfcbbdb8c864362ed77b481dd7dcfe91709364ef0e6990e756fe559aade72d2e2aab

data/Rakefile

@@ -14,14 +14,11 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+APP_RAKEFILE = File.expand_path('../test/dummy/Rakefile', __FILE__)
 load 'rails/tasks/engine.rake'
 
-
 load 'rails/tasks/statistics.rake'
 
-
-
 Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
@@ -33,5 +30,4 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = false
 end
 
-
 task default: :test

data/app/lib/bridge_cache/data/bridge_model.rb

@@ -1,28 +1,32 @@
 module BridgeCache::Data
   module BridgeModel
-
+    EXCLUDED_COLUMNS = %w[created_at updated_at].freeze
     def adjusted_table_name
       if BridgeCache.use_internal_database
         table_name
       else
-        self.to_s.demodulize.underscore.pluralize
+        to_s.demodulize.underscore.pluralize
       end
     end
 
     def self.extended(base)
       base.class_eval do
-        scope :in_domain, -> (domain_id) {for_domain(domain_id)}
-        scope :active, -> {is_active?}
-        self.table_name = self.adjusted_table_name
+        scope :in_domain, ->(domain_id) { for_domain(domain_id) }
+        scope :active, -> { is_active? }
+        self.table_name = adjusted_table_name
       end
     end
 
+    def format_import_row(row)
+      row
+    end
+
     def import_from_csv(file_path)
-      BridgeCache::Plugins::CSVDump::dump_to_table(self, file_path)
+      BridgeCache::Plugins::CSVDump.dump_to_table(self, file_path)
     end
 
     def create_from_csv_row(row)
-      BridgeCache::Plugins::CSVDump::dump_row(self, row)
+      BridgeCache::Plugins::CSVDump.dump_row(self, row)
     end
 
     def cleanup(row_ids)
@@ -42,18 +46,28 @@ module BridgeCache::Data
       message.payload_class.where(bridge_id: message.resource_object['id']).destroy_all
     end
 
-    def webhook_completed(message)
-      raise "Method not implemented"
+    def webhook_completed(_message)
+      raise 'Method not implemented'
     end
 
     def for_domain(domain_id)
       where(domain_id: domain_id)
     end
 
+    def uniq_constraint_name
+      "#{table_name.downcase}_bridge_uniq"
+    end
+
     def is_active?
-      if self.column_names.include? 'deleted_at'
-        where(deleted_at: nil)
-      end
+      where(deleted_at: nil) if column_names.include? 'deleted_at'
+    end
+
+    def csv_mapping
+      columns = column_names.as_json
+      mapping = {}
+      columns.each { |column| mapping[column] = column unless EXCLUDED_COLUMNS.include?(column) }
+      mapping['bridge_id'] = 'bridge_id'
+      mapping
     end
 
     private

data/app/lib/bridge_cache/jobs/import_check.rb

@@ -5,14 +5,15 @@ module BridgeCache::Jobs
 
     ATTEMPTS_THRESHOLD = 30
 
-    def perform(bridge_base_url, bridge_api_key, bridge_api_secret, models, attempts, chain = [], global_options = {})
-      remote_data = BridgeBlueprint::RemoteData.new(bridge_base_url, bridge_api_key, bridge_api_secret)
+    def perform(account_settings, attempts, chain = [])
+      account_settings = account_settings.with_indifferent_access
+      remote_data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key], account_settings[:api_secret])
       if attempts > ATTEMPTS_THRESHOLD
         Rails.logger.fatal("Bridge data dump failed to download after #{(ATTEMPTS_THRESHOLD * 30) / 60} minutes")
       elsif remote_data.status == BridgeBlueprint::Constants::STATUS_COMPLETE
-        BridgeCache::Jobs::ImportData.set(queue: self.queue_name).perform_later(bridge_base_url, bridge_api_key, bridge_api_secret, models, chain, global_options)
+        BridgeCache::Jobs::ImportData.set(queue: queue_name).perform_later(account_settings, chain)
       elsif remote_data.status == BridgeBlueprint::Constants::STATUS_PENDING
-        BridgeCache::Jobs::ImportCheck.set(queue: self.queue_name, wait: 30.seconds).perform_later(bridge_base_url, bridge_api_key, bridge_api_secret, models, attempts + 1, chain, global_options)
+        BridgeCache::Jobs::ImportCheck.set(queue: queue_name, wait: 30.seconds).perform_later(account_settings, attempts + 1, chain)
       elsif remote_data.status == BridgeBlueprint::Constants::STATUS_FAILED || remote_data.status == BridgeBlueprint::Constants::STATUS_NOT_FOUND
         raise 'Bridge data dump download failed'
       end

data/app/lib/bridge_cache/jobs/import_data.rb

@@ -3,28 +3,20 @@ module BridgeCache::Jobs
     include BridgeCache::Utils::WorkerHelper
     queue_as :default
 
-    def perform(bridge_base_url, bridge_api_key, bridge_api_secret, models, chain = [], global_options = {})
-      begin
-        remote_data = BridgeBlueprint::RemoteData.new(bridge_base_url, bridge_api_key, bridge_api_secret)
-        Dir.mktmpdir do |dir|
-          path = "#{dir}/data_dump"
-          remote_data.store_file(path)
-          import = BridgeBlueprint::DataDump.new(path)
-          models.each do |model|
-            row_ids = []
-
-            import.each_row(model.pluralize) do |row|
-              BridgeCache::Jobs::ImportRow.set(queue: self.queue_name).perform_later(model, row.to_h)
-              row_ids << row['id'].to_i
-            end
-
-            BridgeCache::Jobs::CleanupJob.set(queue: self.queue_name).perform_later(model, row_ids)
-          end
+    def perform(account_settings, chain = [])
+      account_settings = account_settings.with_indifferent_access
+      remote_data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key], account_settings[:api_secret])
+      Dir.mktmpdir do |dir|
+        path = "#{dir}/data_dump"
+        remote_data.store_file(path)
+        import = BridgeBlueprint::DataDump.new(path)
+        BridgeCache.resolved_models(account_settings[:is_subaccount]).each do |model|
+          ids = []
+          ids << BridgeCache::Plugins::CSVDump.bulk_import(import, model)
+          BridgeCache::Jobs::CleanupJob.set(queue: queue_name).perform_later(model, ids)
         end
-        BridgeCache::Utils::WorkerHelper.invoke_next(chain)
-      rescue => e
-        Rails.logger.debug(e)
       end
+      BridgeCache::Utils::WorkerHelper.invoke_next(chain)
     end
   end
 end

data/app/lib/bridge_cache/jobs/import_row.rb

@@ -2,7 +2,7 @@ module BridgeCache::Jobs
   class ImportRow < ActiveJob::Base
     queue_as :default
 
-    def perform(model, row, chain = [])
+    def perform(model, row)
       "BridgeCache::#{model.camelcase}".constantize.create_from_csv_row(row)
     end
   end

data/app/lib/bridge_cache/jobs/import_start.rb

@@ -2,25 +2,16 @@ module BridgeCache::Jobs
   class ImportStart < ActiveJob::Base
     queue_as :default
 
-    def perform(bridge_base_url, bridge_api_key, bridge_api_secret, models, chain = [], global_options = {})
-      begin
-        data = BridgeBlueprint::RemoteData.new(bridge_base_url, bridge_api_key, bridge_api_secret)
-        models = (['domain', 'affiliated_sub_account'] + models).uniq unless global_options[:subaccount]
-        data.start_data_report
-        BridgeCache::Jobs::ImportCheck
-          .set(queue: self.queue_name, wait: 30.seconds)
-          .perform_later(
-            bridge_base_url,
-            bridge_api_key,
-            bridge_api_secret,
-            models,
-            0,
-            chain,
-            global_options
-          )
-      rescue => e
-        Rails.logger.debug(e)
-      end
+    def perform(account_settings, chain = [])
+      data = BridgeBlueprint::RemoteData.new(account_settings[:url], account_settings[:api_key], account_settings[:api_secret])
+      data.start_data_report
+      BridgeCache::Jobs::ImportCheck
+        .set(queue: queue_name, wait: 30.seconds)
+        .perform_later(
+          account_settings,
+          0,
+          chain
+        )
     end
   end
 end

data/app/lib/bridge_cache/plugins/csv_dump.rb

@@ -2,22 +2,43 @@ require 'csv'
 
 module BridgeCache::Plugins
   class CSVDump
-
     MAX_ROW_INTERVAL = 5000
 
+    def self.bulk_import(iterator, model)
+      ids = []
+      rows = []
+      klass = "BridgeCache::#{model.camelcase}".constantize
+
+      csv_column_names = klass.csv_mapping.keys
+      database_column_names = klass.csv_mapping.values
+
+      iterator.each_row(model.pluralize) do |row|
+        row = remove_bad_columns(klass, BridgeCache::Plugins::DataTransform.set_bridge_id(row).to_h)
+        row = klass.format_import_row(row)
+        rows << csv_column_names.map { |column| row[column] }
+        ids << row['bridge_id']
+
+        if rows.length >= BridgeCache.batch_size
+          perform_bulk_import(klass, database_column_names, rows)
+          rows = []
+        end
+      end
+
+      perform_bulk_import(klass, database_column_names, rows)
+      ids
+    end
+
     def self.dump_to_table(clazz, file_path)
       count = 1
       total = 0
       rows = []
-      CSV.foreach(file_path, headers: true) do |row|
+      CSV.foreach(file_path, headers: true) do |_row|
         total += 1
       end
       CSV.foreach(file_path, headers: true) do |row|
-        if count < MAX_ROW_INTERVAL
-          rows << initialze_row(clazz, row)
-        end
+        rows << initialze_row(clazz, row) if count < MAX_ROW_INTERVAL
         if count % MAX_ROW_INTERVAL == 0 || count == total
-          dump_rows(clazz, rows)
+          dump_rows(rows)
           count = 0
           rows = []
         end
@@ -27,27 +48,52 @@ module BridgeCache::Plugins
 
     def self.dump_row(clazz, row)
       instance = initialze_row(clazz, row)
-      dump_rows(clazz, [instance])
+      dump_rows([instance])
     end
 
     private
 
+    def self.perform_bulk_import(klass, columns, rows)
+      return if rows.empty?
+      columns = columns.dup
+      klass.import(columns, rows, validate: false, on_duplicate_key_update: {
+                     constraint_name: klass.uniq_constraint_name,
+                     condition: condition_sql(klass, columns),
+                     columns: columns
+                   })
+    end
+
+    # This method generates SQL that looks like:
+    # (users.sis_id, users.email) IS DISTINCT FROM (EXCLUDED.sis_id, EXCLUDED.email)
+    #
+    # This prevents activerecord-import from setting the `updated_at` column for
+    # rows that haven't actually changed. This allows you to query for rows that have changed
+    # by doing something like:
+    #
+    # started_at = Time.now
+    # run_the_users_sync!
+    # changed = User.where("updated_at >= ?", started_at)
+    def self.condition_sql(klass, columns)
+      columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(', ')
+      excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(', ')
+      "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+    end
+
     def self.initialze_row(clazz, row)
       instance = clazz.find_or_create_by(bridge_id: row['id'])
-      instance.assign_attributes(remove_bad_columns(clazz, BridgeCache::Plugins::DataTransform::set_bridge_id(row).to_h))
+      instance.assign_attributes(remove_bad_columns(clazz, BridgeCache::Plugins::DataTransform.set_bridge_id(row).to_h))
       instance
     end
 
-    def self.dump_rows(clazz, rows)
+    def self.dump_rows(rows)
       rows.each do |row|
         row.save! if row.changed?
       end
     end
 
     def self.remove_bad_columns(clazz, row)
-      row = row.delete_if {|key, value| !clazz.column_names.include?(key)}
+      row = row.delete_if { |key, _value| !clazz.column_names.include?(key) }
       row
     end
-
   end
 end

data/app/lib/bridge_cache/plugins/data_transform.rb

@@ -1,6 +1,5 @@
 module BridgeCache::Plugins
   class DataTransform
-
     def self.set_bridge_id(row)
       if row.key?('id')
         row['bridge_id'] = row['id'] if row.key?('id')

data/app/lib/bridge_cache/utils/worker_helper.rb

@@ -5,7 +5,7 @@ module BridgeCache::Utils
       job = job_chain.shift
       if job
         job_class = Object.const_get(job[:job].classify)
-        job_class.perform_later(*(job[:params].concat(params)))
+        job_class.perform_later(*job[:params].concat(params))
       end
     end
   end

data/app/lib/bridge_cache/webhooks/message.rb

@@ -1,6 +1,5 @@
 module BridgeCache::Webhooks
   class Message
-
     attr_reader :payload
 
     def initialize(payload)

data/app/models/bridge_cache/bridge_base_model.rb

@@ -9,7 +9,7 @@ module BridgeCache
 
     def bridge_id
       if self.class.column_names.include? BRIDGE_ID
-         self[:bridge_id]
+        self[:bridge_id]
       else
         id
       end
@@ -19,9 +19,8 @@ module BridgeCache
       if BridgeCache.use_internal_database
         table_name
       else
-        self.to_s.demodulize.underscore.pluralize
+        to_s.demodulize.underscore.pluralize
       end
     end
-
   end
 end

data/app/models/bridge_cache/course_template.rb

@@ -14,9 +14,8 @@ module BridgeCache
 
     def self.for_domain(domain_id)
       includes(:domains)
-      .where('bridge_cache_affiliated_sub_accounts.domain_id = ? OR bridge_cache_course_templates.sub_account_id = ?', domain_id, domain_id)
-      .references(:domains)
+        .where('bridge_cache_affiliated_sub_accounts.domain_id = ? OR bridge_cache_course_templates.sub_account_id = ?', domain_id, domain_id)
+        .references(:domains)
     end
-
   end
 end

data/app/models/bridge_cache/custom_field.rb

@@ -4,6 +4,5 @@ module BridgeCache
 
     belongs_to :domain, foreign_key: :domain_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Domain.name, optional: true
     has_many :learner_custom_field_values, primary_key: BridgeCache.primary_key, foreign_key: :custom_field_id, class_name: BridgeCache::LearnerCustomFieldValue.name
-
   end
 end

data/app/models/bridge_cache/domain.rb

@@ -16,6 +16,5 @@ module BridgeCache
     belongs_to :owner, class_name: BridgeCache::User.name
 
     serialize :config
-
   end
 end

data/app/models/bridge_cache/enrollment.rb

@@ -8,6 +8,5 @@ module BridgeCache
 
     belongs_to :user, foreign_key: :user_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::User.name, optional: true
     belongs_to :course_template, foreign_key: :course_template_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::CourseTemplate.name, optional: true
-
   end
 end

data/app/models/bridge_cache/enrollment_profile.rb

@@ -11,9 +11,19 @@ module BridgeCache
     end
 
     def parse_json
-      json = BridgeCache::Plugins::DataTransform::valid_json?(self.config)
+      self.parse_config(self.config)
       self.config = json if json.present?
     end
 
+    def self.parse_config(config)
+      json = BridgeCache::Plugins::DataTransform::valid_json?(config)
+    end
+
+    def self.format_import_row(row)
+      json = parse_config(row['config'])
+      row['config'] = json if json.present?
+      row
+    end
+
   end
 end

data/app/models/bridge_cache/grant.rb

@@ -4,6 +4,5 @@ module BridgeCache
 
     belongs_to :user, foreign_key: :user_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::User.name, optional: true
     belongs_to :domain, foreign_key: :domain_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Domain.name, optional: true
-
   end
 end

data/app/models/bridge_cache/group.rb

@@ -9,6 +9,5 @@ module BridgeCache
     has_many :group_enrollments, foreign_key: :group_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::GroupEnrollment.name
     has_many :memberships, foreign_key: :group_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Membership.name
     belongs_to :domain, foreign_key: :domain_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Domain.name, optional: true
-
   end
 end

data/app/models/bridge_cache/group_enrollment.rb

@@ -3,11 +3,10 @@ module BridgeCache
     extend BridgeCache::Data::BridgeModel
 
     def self.for_domain(domain_id)
-      joins(:course_template).where(bridge_cache_course_templates: {domain_id: domain_id})
+      joins(:course_template).where(bridge_cache_course_templates: { domain_id: domain_id })
     end
 
     belongs_to :group, foreign_key: :group_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Group.name, optional: true
     belongs_to :course_template, foreign_key: :course_template_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::CourseTemplate.name, optional: true
-
   end
 end

data/app/models/bridge_cache/learner_custom_field_value.rb

@@ -3,11 +3,10 @@ module BridgeCache
     extend BridgeCache::Data::BridgeModel
 
     def self.for_domain(domain_id)
-      joins(:user).where(bridge_cache_users: {domain_id: domain_id})
+      joins(:user).where(bridge_cache_users: { domain_id: domain_id })
     end
 
     belongs_to :user, foreign_key: :user_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::User.name, optional: true
     belongs_to :custom_field, foreign_key: :custom_field_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::CustomField.name, optional: true
-
   end
 end

data/app/models/bridge_cache/live_course.rb

@@ -11,9 +11,8 @@ module BridgeCache
 
     def self.for_domain(domain_id)
       includes(:domains)
-      .where('bridge_cache_affiliated_sub_accounts.domain_id = ? OR bridge_cache_live_courses.sub_account_id = ?', domain_id, domain_id)
-      .references(:domains)
+        .where('bridge_cache_affiliated_sub_accounts.domain_id = ? OR bridge_cache_live_courses.sub_account_id = ?', domain_id, domain_id)
+        .references(:domains)
     end
-
   end
 end

data/app/models/bridge_cache/live_course_enrollment.rb

@@ -9,6 +9,5 @@ module BridgeCache
     belongs_to :user, foreign_key: :user_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::User.name, optional: true
     belongs_to :live_course, foreign_key: :live_course_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::LiveCourse.name, optional: true
     has_many :live_course_session_registrations, foreign_key: :live_course_enrollment_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::LiveCourseSessionRegistration.name
-
   end
 end

data/app/models/bridge_cache/live_course_session.rb

@@ -10,6 +10,5 @@ module BridgeCache
     belongs_to :sub_account, foreign_key: :sub_account_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Domain.name, optional: true
     belongs_to :live_course, foreign_key: :live_course_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::LiveCourse.name, optional: true
     has_many :live_course_session_registrations, foreign_key: :live_course_session_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::LiveCourseSessionRegistration.name
-
   end
 end

data/app/models/bridge_cache/live_course_session_registration.rb

@@ -10,6 +10,5 @@ module BridgeCache
     belongs_to :domain, foreign_key: :domain_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Domain.name, optional: true
     belongs_to :live_course_enrollment, primary_key: BridgeCache.primary_key, class_name: BridgeCache::LiveCourseEnrollment.name, optional: true
     belongs_to :live_course_session, primary_key: BridgeCache.primary_key, class_name: BridgeCache::LiveCourseSession.name, optional: true
-
   end
 end

data/app/models/bridge_cache/membership.rb

@@ -6,8 +6,7 @@ module BridgeCache
     belongs_to :user, foreign_key: :group_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::User.name, optional: true
 
     def self.for_domain(domain_id)
-      joins(:group).where(bridge_cache_groups: {domain_id: domain_id})
+      joins(:group).where(bridge_cache_groups: { domain_id: domain_id })
     end
-
   end
 end

data/app/models/bridge_cache/program.rb

@@ -12,9 +12,8 @@ module BridgeCache
 
     def self.for_domain(domain_id)
       includes(:domains)
-      .where('bridge_cache_affiliated_sub_accounts.domain_id = ? OR bridge_cache_programs.sub_account_id = ?', domain_id, domain_id)
-      .references(:domains)
+        .where('bridge_cache_affiliated_sub_accounts.domain_id = ? OR bridge_cache_programs.sub_account_id = ?', domain_id, domain_id)
+        .references(:domains)
     end
-
   end
 end

data/app/models/bridge_cache/program_enrollment.rb

@@ -8,6 +8,5 @@ module BridgeCache
     def self.for_domain(domain_id)
       joins(:program).merge(BridgeCache::Program.for_domain(domain_id))
     end
-
   end
 end

data/app/models/bridge_cache/program_group_enrollment.rb

@@ -4,6 +4,5 @@ module BridgeCache
 
     belongs_to :group, foreign_key: :group_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Group.name, optional: true
     belongs_to :program, foreign_key: :program_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Program.name, optional: true
-
   end
 end

data/app/models/bridge_cache/program_item.rb

@@ -7,6 +7,5 @@ module BridgeCache
     def self.for_domain(domain_id)
       joins(:program).merge(BridgeCache::Program.for_domain(domain_id))
     end
-
   end
 end

data/app/models/bridge_cache/tag.rb

@@ -3,6 +3,5 @@ module BridgeCache
     extend BridgeCache::Data::BridgeModel
 
     has_many :taggings, foreign_key: :tag_id, primary_key: BridgeCache.primary_key, class_name: BridgeCache::Tagging.name
-
   end
 end

data/app/models/bridge_cache/user.rb

@@ -24,6 +24,5 @@ module BridgeCache
     def manager
       domain.owner
     end
-
   end
 end

data/db/migrate/20171019195503_create_live_course_enrollments.rb

@@ -3,7 +3,7 @@ class CreateLiveCourseEnrollments < ActiveRecord::Migration[5.1]
     return unless BridgeCache.use_internal_database
     create_table :bridge_cache_live_course_enrollments do |t|
       t.integer :bridge_id, limit: 8
-      t.integer :live_course_id,limit: 8
+      t.integer :live_course_id, limit: 8
       t.integer :user_id, limit: 8
       t.datetime :created_at
       t.datetime :updated_at

data/db/migrate/20171031165231_add_indexes.rb

@@ -92,6 +92,5 @@ class AddIndexes < ActiveRecord::Migration[5.1]
     add_index :bridge_cache_live_course_sessions, :domain_id
     add_index :bridge_cache_live_course_sessions, :sub_account_id
     add_index :bridge_cache_live_course_sessions, :parent_id
-
   end
 end

data/db/migrate/20180406204906_add_uniq_constraints.rb

@@ -0,0 +1,26 @@
+class AddUniqConstraints < ActiveRecord::Migration[5.1]
+  def change
+    add_unique_constraint :bridge_cache_affiliated_sub_accounts, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_affiliated_sub_accounts_bridge_uniq'
+    add_unique_constraint :bridge_cache_course_templates, %i[bridge_id sub_account_id], deferrable: false, name: 'bridge_cache_course_templates_bridge_uniq'
+    add_unique_constraint :bridge_cache_custom_fields, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_custom_fields_bridge_uniq'
+    add_unique_constraint :bridge_cache_domains, [:bridge_id], deferrable: false, name: 'bridge_cache_domains_bridge_uniq'
+    add_unique_constraint :bridge_cache_enrollment_profiles, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_enrollment_profiles_bridge_uniq'
+    add_unique_constraint :bridge_cache_enrollments, [:bridge_id], deferrable: false, name: 'bridge_cache_enrollments_bridge_uniq'
+    add_unique_constraint :bridge_cache_grants, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_grants_bridge_uniq'
+    add_unique_constraint :bridge_cache_groups, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_groups_bridge_uniq'
+    add_unique_constraint :bridge_cache_group_enrollments, [:bridge_id], deferrable: false, name: 'bridge_cache_group_enrollments_bridge_uniq'
+    add_unique_constraint :bridge_cache_learner_custom_field_values, [:bridge_id], deferrable: false, name: 'bridge_cache_learner_custom_field_values_bridge_uniq'
+    add_unique_constraint :bridge_cache_live_course_enrollments, [:bridge_id], deferrable: false, name: 'bridge_cache_live_course_enrollments_bridge_uniq'
+    add_unique_constraint :bridge_cache_live_course_session_registrations, [:bridge_id], deferrable: false, name: 'bridge_cache_live_course_session_registrations_bridge_uniq'
+    add_unique_constraint :bridge_cache_live_course_sessions, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_live_course_sessions_bridge_uniq'
+    add_unique_constraint :bridge_cache_live_courses, %i[bridge_id domain_id], deferrable: false, name: 'bridge_cache_live_courses_bridge_uniq'
+    add_unique_constraint :bridge_cache_memberships, [:bridge_id], deferrable: false, name: 'bridge_cache_memberships_bridge_uniq'
+    add_unique_constraint :bridge_cache_program_enrollments, [:bridge_id], deferrable: false, name: 'bridge_cache_program_enrollments_bridge_uniq'
+    add_unique_constraint :bridge_cache_program_group_enrollments, [:bridge_id], deferrable: false, name: 'bridge_cache_program_group_enrollments_bridge_uniq'
+    add_unique_constraint :bridge_cache_program_items, [:bridge_id], deferrable: false, name: 'bridge_cache_program_items_bridge_uniq'
+    add_unique_constraint :bridge_cache_programs, [:bridge_id], deferrable: false, name: 'bridge_cache_programs_bridge_uniq'
+    add_unique_constraint :bridge_cache_tags, [:bridge_id], deferrable: false, name: 'bridge_cache_tags_bridge_uniq'
+    add_unique_constraint :bridge_cache_taggings, [:bridge_id], deferrable: false, name: 'bridge_cache_taggings_bridge_uniq'
+    add_unique_constraint :bridge_cache_users, %i[bridge_id sub_account_id], deferrable: false, name: 'bridge_cache_users_bridge_uniq'
+  end
+end

data/lib/bridge_cache.rb

@@ -1,26 +1,58 @@
-require "bridge_cache/engine"
+require 'bridge_cache/engine'
 require 'bridge_blueprint'
+require 'active_record'
+require 'activerecord-import'
+require 'rein'
 
 module BridgeCache
+  class << self
+    WRITER_METHODS = %w[use_internal_database batch_size is_subaccount].freeze
+    READ_METHODS = %w[models].freeze
+    READ_WRITE_METHODS = %w[].freeze
+    DEFAULT_BATCH_SIZE = 10_000
+    DEFAULT_MODELS = %(domain affiliated_sub_account).freeze
+
+    attr_writer(*WRITER_METHODS)
+    attr_accessor(*READ_WRITE_METHODS)
+    attr_reader(*READ_METHODS)
+
+    def configure
+      yield self if block_given?
+    end
+
+    def use_internal_database
+      return @use_internal_database if defined?(@use_internal_database)
+
+      @use_internal_database = true
+    end
+
+    def batch_size
+      @batch_size || DEFAULT_BATCH_SIZE
+    end
 
-    class << self
+    def models=(models)
+      @models = merged_required_models(models)
+    end
 
-      WRITER_METHODS = [:use_internal_database]
+    def models
+      return @models if defined?(@models)
 
-      attr_writer(*WRITER_METHODS)
+      @models = merged_required_models([])
+    end
 
-      def configure
-        yield self if block_given?
-      end
+    def primary_key
+      @use_internal_database ? :bridge_id : :id
+    end
 
-      def use_internal_database
-        return @use_internal_database if defined?(@use_internal_database)
+    def resolved_models(remove_root_models)
+      return @models unless remove_root_models
+      @models.reject { |model| DEFAULT_MODELS.include?(model) }
+    end
 
-        @use_internal_database = true
-      end
+    private
 
-      def primary_key
-        @use_internal_database ? :bridge_id : :id
-      end
+    def merged_required_models(models)
+      (%w[domain affiliated_sub_account] + models).uniq
     end
+  end
 end

data/lib/bridge_cache/engine.rb

@@ -5,19 +5,18 @@ module BridgeCache
     config.autoload_once_paths += Dir["#{config.root}/lib/**/"]
 
     config.generators do |g|
-     g.test_framework :rspec
-     g.fixture_replacement :factory_girl, :dir => 'spec/factories'
-   end
+      g.test_framework :rspec
+      g.fixture_replacement :factory_girl, dir: 'spec/factories'
+    end
 
-   initializer :append_migrations do |app|
-     unless app.root.to_s.match root.to_s
-       config.paths["db/migrate"].expanded.each do |expanded_path|
-         app.config.paths["db/migrate"] << expanded_path
-       end
-       # Apartment will modify this, but it doesn't fully support engine migrations, so we'll reset it here
-       ActiveRecord::Migrator.migrations_paths = Rails.application.paths['db/migrate'].to_a
-     end
-   end
-   
+    initializer :append_migrations do |app|
+      unless app.root.to_s.match root.to_s
+        config.paths['db/migrate'].expanded.each do |expanded_path|
+          app.config.paths['db/migrate'] << expanded_path
+        end
+        # Apartment will modify this, but it doesn't fully support engine migrations, so we'll reset it here
+        ActiveRecord::Migrator.migrations_paths = Rails.application.paths['db/migrate'].to_a
+      end
+    end
   end
 end

data/lib/bridge_cache/version.rb

@@ -1,3 +1,3 @@
 module BridgeCache
-  VERSION = "0.2.8"
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,75 +1,104 @@
 --- !ruby/object:Gem::Specification
 name: bridge_cache
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - jshaffer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-26 00:00:00.000000000 Z
+date: 2018-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 0.20.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 0.20.2
 - !ruby/object:Gem::Dependency
   name: bridge_blueprint
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.04
+        version: 0.0.06
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.04
+        version: 0.0.06
 - !ruby/object:Gem::Dependency
-  name: actionmailer
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.1.4
-  type: :development
+        version: '5.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.1.4
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: rein
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: actionmailer
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 9.1.0
+        version: 5.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 9.1.0
+        version: 5.1.4
 - !ruby/object:Gem::Dependency
   name: actionpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.4
@@ -77,41 +106,59 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.4
 - !ruby/object:Gem::Dependency
-  name: rspec-rails
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.7.2
+        version: 1.6.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.7.2
+        version: 1.6.2
 - !ruby/object:Gem::Dependency
-  name: factory_bot_rails
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.2
+        version: 9.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.2
+        version: 9.1.0
 - !ruby/object:Gem::Dependency
   name: combustion
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.0
@@ -119,41 +166,59 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.0
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.16.0
+        version: 1.6.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.16.0
+        version: 1.6.2
 - !ruby/object:Gem::Dependency
-  name: database_cleaner
+  name: factory_bot_rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: 4.8.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: 4.8.2
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.21.0
@@ -161,13 +226,39 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.21.0
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.2
 - !ruby/object:Gem::Dependency
   name: shoulda-matchers
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.2
@@ -175,10 +266,16 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.2
-description: A mountable engine for bridge data storage
+description: |-
+  A mountable engine for bridge data storage,
+                    can either be used for direct database connection
+                    or as a localized data storage mechanism
 email:
 - jshaffer@instructure.com
 executables: []
@@ -186,7 +283,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - MIT-LICENSE
-- README.rdoc
 - Rakefile
 - app/assets/javascripts/bridge_cache/application.js
 - app/assets/stylesheets/bridge_cache/application.css
@@ -253,6 +349,7 @@ files:
 - db/migrate/20171031165231_add_indexes.rb
 - db/migrate/20180316205458_create_bridge_cache_enrollment_profiles.rb
 - db/migrate/20180320190443_add_index_to_enrollment_profile.rb
+- db/migrate/20180406204906_add_uniq_constraints.rb
 - lib/bridge_cache.rb
 - lib/bridge_cache/engine.rb
 - lib/bridge_cache/version.rb
@@ -277,8 +374,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: A mountable engine for bridge data storage
 test_files: []
+has_rdoc: 

data/README.rdoc

@@ -1,36 +0,0 @@
-# BridgeCache
-
-https://www.getbridge.com/
-
-Bridge Cache is a data dump engine used to cache remote bridge data into a local database.
-
-The application mimics the bridge data model and will copy csv rows directly into the local
-data store without any extra configuration
-
-## Setup
- gem install 'bridge_cache'
-
- rake db:migrate
-
- Enable Bridge data dumps through the administrative interface in bridge, or have your bridge
- admin enable it for you.
-
-## Usage
-
-  To Import an entire csv
-  BridgeCache::User.import_from_csv(path_to_csv)
-
-  Import a list of models from a data dump automatically.  This feature requires a queue manager
-  like sidekiq or delayed jobs
-
-  BridgeCache::Jobs::ImportStart.perform_later(
-    bridge_base_url, (https://yourbridgeurl.bridgeapp.com)
-    bridge_api_key, (API key generated through bridge admin interface)
-    bridge_api_secret, (API secret generated through bridge admin interface)
-    [
-      'user',
-      'group',
-      'domain',
-      'course_template'
-    ],
-    [])