inst_data_shipper 0.1.0.beta1 → 0.1.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf2f1cdd4b4181e945c5f36e7680ed0a054429dc191197fafbee60de9598305b
-  data.tar.gz: 5fb781dc8aa17bf7d672fdfc8942d70365edb68fd324d8aeab70297270af0b1e
+  metadata.gz: 146f5b93819d7950f9bd256a99eb690a63d453b86be4ac6ac7cf4c5901724cdd
+  data.tar.gz: 2410298ebb3b1ddc565ca70d49a274a129e83087d461dcfae4d4981979795ea5
 SHA512:
-  metadata.gz: 9212cd9c647193aa7256f15f6da12cd4ee3c56a12e011ac269a1d801d15e0cb7182a71c2fa8d8e2d2ea808aff73ff4f7c974c3720db54414eb43c24658ca554f
-  data.tar.gz: c4cb69ad7ea635833aa5051dec5a8c14f3aa13e2b11dd3e8fbdd4d12c2a9d63ac9dbb5b235915da0d80898d0b048d5677fb807d6947911e3e10a87956b8ee1fc
+  metadata.gz: c6dc93902e0ef7a114d2434d3901c677021d57603b07c1122d72bd2f953184b207d0e57f5441fafb7035a00bf28f2e5035d34cd0edfb73da6d2f93d93874f344
+  data.tar.gz: 2e7babf6a2ed86f9a2e5769bfb393549fb07b771b700513efd640ba2b98ebc3eeadec99578e4c828738903ef121f8e3bebe8490f6f75de0ce9afac43ac28b8fa

data/db/migrate/{20240301090836_create_canvas_sync_sync_batches.rb → 20240301090836_create_inst_data_shipper_dump_batches.rb}

@@ -1,4 +1,4 @@
-class InstDataShipperDumpBatches < CanvasSync::MiscHelper::MigrationClass
+class CreateInstDataShipperDumpBatches < CanvasSync::MiscHelper::MigrationClass
   def change
     create_table :inst_data_shipper_dump_batches do |t|
       t.datetime :started_at
@@ -6,6 +6,8 @@ class InstDataShipperDumpBatches < CanvasSync::MiscHelper::MigrationClass
       t.string :status
 
       t.string :job_class
+      t.string :genre
+
       t.string :exception
       t.text :backtrace
       t.text :metadata

data/lib/inst_data_shipper/basic_dumper.rb

@@ -21,7 +21,7 @@ module InstDataShipper
 
     def table_schemas
       pointer = @schema_pointer || batch_context[:schema_pointer]
-      safe_constantize(pointer)
+      pointer.constantize
     end
   end
 end

data/lib/inst_data_shipper/concerns/hooks.rb

@@ -9,21 +9,35 @@ module InstDataShipper
       end
 
       def hook(name, prepend: false, &block)
+        _assert_hook_defined(name)
+        @hooks ||= {}
+        @hooks[name] ||= []
         hooks = @hooks[name]
         prepend ? hooks.unshift(block) : hooks << block
       end
+
+      def _assert_hook_defined(name)
+        return true if @hooks&.key?(name)
+        return if superclass.respond_to?(:_assert_hook_defined) && superclass._assert_hook_defined(name)
+        raise ArgumentError, "Hook #{name} is not defined"
+      end
+
+      def _list_hooks(name)
+        list = []
+        list.push(*superclass._list_hooks(name)) if superclass.respond_to?(:_list_hooks)
+        list.push(*@hooks[name]) if (@hooks || {})[name]
+        list
+      end
     end
 
     def run_hook(name, *args, **kwargs)
-      hooks = @hooks[name]
-      hooks.each do |blk|
+      self.class._list_hooks(name).each do |blk|
         instance_exec(*args, **kwargs, &blk)
       end
     end
 
     def run_hook_safe(name, *args, **kwargs)
-      hooks = @hooks[name]
-      hooks.each do |blk|
+      self.class._list_hooks(name).each do |blk|
         instance_exec(*args, **kwargs, &blk)
       rescue StandardError
       end

data/lib/inst_data_shipper/data_sources/canvas_reports.rb

@@ -27,11 +27,25 @@ module InstDataShipper
         _in_canvas_report_pool(:_import_canvas_report, *args, **kwargs)
       end
 
-      def import_canvas_report_by_terms(target_table, report_name, terms: [], params: {}, **kwargs)
+      def import_canvas_report_by_terms(*args, **kwargs)
+        _in_canvas_report_pool(:_import_canvas_report_by_terms, *args, **kwargs)
+      end
+
+      def import_existing_report(report, **kwargs)
+        delayed(:_process_canvas_report, report: report, **kwargs)
+      end
+
+      private
+
+      def _import_canvas_report_by_terms(report_name, terms: [], params: {}, **kwargs)
         term_ids = (terms || []).map do |term|
           term.is_a?(Term) ? term.canvas_id : term
         end
 
+        table_def = lookup_table_schema!(kwargs[:schema_name], report_name)
+
+        _resolve_report_incremenal_parameters(table_def, params)
+
         Sidekiq::Batch.new.tap do |b|
           b.description = "Term Scoped #{report_name} Runners"
           b.context = {
@@ -40,19 +54,21 @@ module InstDataShipper
           b.jobs do
             terms_query = term_ids.present? ? Term.where(canvas_id: term_ids) : Term
             terms_query.find_each do |t|
-              import_canvas_report(target_table, report_name, params: { **params, enrollment_term_id: t.canvas_id }, **kwargs)
+              _in_canvas_report_pool(:_trigger_canvas_report, report_name, params: { **params, enrollment_term_id: t.canvas_id }, **kwargs)
             end
           end
         end
       end
 
-      def import_existing_report(table, report)
-        delayed(:_process_canvas_report, table, report: report)
-      end
+      def _import_canvas_report(report_name, params: {}, **kwargs)
+        table_def = lookup_table_schema!(kwargs[:schema_name], report_name)
 
-      private
+        _resolve_report_incremenal_parameters(table_def, params)
+
+        _trigger_canvas_report(report_name, params: params, **kwargs)
+      end
 
-      def _import_canvas_report(target_table, report_name, retry_count: 3, params: {}, **kwargs)
+      def _trigger_canvas_report(report_name, retry_count: 3, params: {}, **kwargs)
         report = canvas_sync_client.start_report(
           'self', report_name,
           parameters: params,
@@ -61,15 +77,13 @@ module InstDataShipper
         CanvasSync::Jobs::CanvasProcessWaiter.perform_later(
           "/api/v1/accounts/self/reports/#{report_name}/#{report[:id]}",
           {
-            instance_of: origin_class,
-            method: :_process_canvas_report,
-            args: [target_table],
+            job: Jobs::AsyncCaller,
+            args: [origin_class, :_process_canvas_report],
             kwargs: kwargs,
           },
           on_failure: {
-            instance_of: origin_class,
-            method: :_handle_failed_canvas_report,
-            args: [target_table, report_name, kwargs],
+            job: Jobs::AsyncCaller,
+            args: [origin_class, :_handle_failed_canvas_report, report_name, kwargs],
             kwargs: { retry_count: retry_count },
           },
           status_key: :status,
@@ -79,18 +93,18 @@ module InstDataShipper
 
       def _in_canvas_report_pool(mthd, *args, **kwargs)
         pool = CanvasSync::JobBatches::Pool.from_pid(batch_context[:report_processor_pool])
-        AsyncCaller.call_from_pool(pool, self.class, mthd, *args, **kwargs)
+        Jobs::AsyncCaller.call_from_pool(pool, self.class, mthd, *args, **kwargs)
       end
 
-      def _process_canvas_report(table, report:)
-        table_def = table_schemas.find { |t| t[:warehouse_name].to_s == table }
+      def _process_canvas_report(report:, schema_name: nil)
+        table_def = lookup_table_schema!(schema_name, report[:report])
 
-        IO.copy_stream(URI.parse(report['attachment']['url']).open, "#{working_dir}/#{table}.csv")
+        IO.copy_stream(URI.parse(report['attachment']['url']).open, "#{working_dir}/temp_report.csv")
 
         inner_block = ->(file) {
-          CSV.foreach("#{working_dir}/#{table}.csv", headers: true) do |m|
+          CSV.foreach("#{working_dir}/temp_report.csv", headers: true) do |m|
             file << table_def[:columns].map do |c|
-              c[:transformer].present? ? m.instance_exec(&c[:transformer]) : m[c[:local_name].to_s]
+              instance_exec(m, &c[:block])
             end
           end
         }
@@ -98,13 +112,36 @@ module InstDataShipper
         upload_data(table_def, extra: report['id'], &inner_block)
       end
 
-      def _handle_failed_canvas_report(table, report_name, kwargs, retry_count:, report:) # rubocop:disable Lint/UnusedMethodArgument
+      def _resolve_report_incremenal_parameters(table_def, params)
+        if table_is_incremental?(table_def)
+          inc = table_def[:incremental]
+          scope = inc[:scope]
+
+          if scope != false
+            scope ||= "updated_after"
+
+            if scope.is_a?(Proc)
+              scope = instance_exec(params, &scope)
+              if scope.is_a?(Hash) && scope != params
+                params.merge!(scope)
+              end
+            elsif scope.is_a?(String) || scope.is_a?(Symbol)
+              params[scope] = incremental_since
+            end
+          end
+        end
+
+        params
+      end
+
+      def _handle_failed_canvas_report(report_name, kwargs, retry_count:, report:)
         if retry_count.positive?
           tbid = batch_context[:report_bid] || batch_context[:root_bid]
           Sidekiq::Batch.new(tbid).jobs do
-            import_canvas_report(table, report_name, retry_count: retry_count - 1, **kwargs.symbolize_keys)
+            _in_canvas_report_pool(:_trigger_canvas_report, report_name, retry_count: retry_count - 1, **kwargs.symbolize_keys)
           end
         else
+          # TODO Allow marking the table as incomplete. Destination code can then decide how to handle incomplete tables since (eg) incremental imports wouldn't mind too much
           cleanup_fatal_error!
         end
       end

data/lib/inst_data_shipper/data_sources/local_tables.rb

@@ -12,22 +12,43 @@ module InstDataShipper
 
       private
 
-      def _import_local_table(table_name)
-        table_def = table_schemas.find { |t| t[:model].to_s == table_name }
-        model = table_def[:model]
+      def _import_local_table(model, schema_name: nil)
+        model = model.safe_constantize if model.is_a?(String)
+
+        table_def = lookup_table_schema!(schema_name, { model: model })
 
         inner_block = ->(file) {
-          query = model
-          query = query.includes(table_def[:includes]) if table_def[:includes].present?
-          model.find_each do |m|
+          query = model.all
+          query = _resolve_model_query(query, table_def[:query])
+
+          if table_is_incremental?(table_def)
+            query = _resolve_model_query(query, table_def.dig(:incremental, :scope), string: ->(r, c) { r.where("? > ?", c, incremental_since) })
+          end
+
+          query.find_each do |m|
             file << table_def[:columns].map do |c|
-              c[:transformer].present? ? m.instance_exec(&c[:transformer]) : m[c[:local_name].to_s]
+              instance_exec(m, &c[:block])
             end
           end
         }
 
         upload_data(table_def, &inner_block)
       end
+
+      def _resolve_model_query(relation, query, string: nil)
+        return relation if query.nil?
+
+        if query.is_a?(Symbol)
+          relation.send(query)
+        elsif query.is_a?(Proc)
+          instance_exec(relation, &query)
+        elsif query.is_a?(String) && string.present?
+          instance_exec(relation, query, &string)
+        else
+          raise "Invalid query: #{query.inspect}"
+        end
+      end
+
     end
   end
 end

data/lib/inst_data_shipper/destinations/base.rb

@@ -50,7 +50,7 @@ module InstDataShipper
       end
 
       def user_config
-        config[:extra]
+        config[:user_config]
       end
 
       def group_key
@@ -62,11 +62,11 @@ module InstDataShipper
       def parse_configuration(uri)
         if block_given?
           parsed = URI.parse(uri)
+          cparsed = ConfigURI.new(parsed)
           cfg = {
-            params: parsed.query.present? ? Rack::Utils.parse_nested_query(parsed.query) : {},
-            extra: (parsed.fragment.present? && parsed.fragment.match?(/^\w+=/) && Rack::Utils.parse_nested_query(parsed.fragment)).presence || parsed.fragment || nil,
+            user_config: cparsed.hash_params,
           }
-          yield parsed, cfg
+          yield cparsed, cfg
           cfg
         else
           raise NotImplementedError
@@ -100,5 +100,28 @@ module InstDataShipper
       end
 
     end
+
+    class ConfigURI
+      def initialize(uri)
+        @uri = uri
+      end
+
+      # delegate_missing_to :uri
+      delegate :scheme, :user, :password, :host, :hostname, :port, :path, :query, :fragment, to: :uri
+
+      def params
+        @params ||= (query.present? ? Rack::Utils.parse_nested_query(query).with_indifferent_access : {}).freeze
+      end
+
+      def hash_params
+        @hash_params ||= ((fragment.present? && fragment.match?(/^\w+=/) && Rack::Utils.parse_nested_query(fragment).with_indifferent_access).presence || fragment || nil)&.freeze
+      end
+
+      private
+
+      def uri
+        @uri
+      end
+    end
   end
 end

data/lib/inst_data_shipper/destinations/hosted_data.rb

@@ -1,3 +1,5 @@
+require "faraday_middleware"
+
 module InstDataShipper
   module Destinations
     class HostedData < Base
@@ -15,7 +17,7 @@ module InstDataShipper
       end
 
       def chunk_data(generator, table:, extra: nil)
-        warehouse_name = table_def[:warehouse_name]
+        warehouse_name = table[:warehouse_name]
 
         super(generator) do |batch, idx|
           bits = [warehouse_name, extra, idx].compact
@@ -36,18 +38,18 @@ module InstDataShipper
 
       def upload_data_chunk(table_def, chunk)
         hosted_data_client.put("api/v1/custom_dumps/#{hd_dump_id}/", artifacts: {
-          table_def[:warehouse_name] => [Faraday::UploadIO.new(chunk, 'application/gzip')],
+          table_name(table_def) => [Faraday::UploadIO.new(chunk, 'application/gzip')],
         })
       end
 
       def finalize_dump
         hosted_data_client.put("api/v1/custom_dumps/#{hd_dump_id}/", start_import: true) if hd_dump_id.present?
-        redis.delete(rk(:state))
+        redis.del(rk(:state))
       end
 
       def cleanup_fatal_error
         hosted_data_client.delete("api/v1/custom_dumps/#{hd_dump_id}/", reason: 'Failure during extraction or transformation') if hd_dump_id.present?
-        redis.delete(rk(:state))
+        redis.del(rk(:state))
       end
 
       # TODO Support/allow single-table fatal errors?
@@ -59,28 +61,25 @@ module InstDataShipper
       end
 
       def convert_schema
-        table_prefix = config[:table_prefix]
-        table_prefix = table_prefix.present? ? "#{table_prefix}_" : nil
-
         definititions = {}
         table_schemas.each do |ts|
           ts = ts.dup
+          tname = table_name(ts)
 
-          table_name = ts[:warehouse_name]
-          table_name = table_prefix + table_name if table_prefix.present?
-
-          definititions[ts[:warehouse_name]] = {
+          definititions[tname] = {
             dw_type: 'dimension',
             description: ts[:description],
-            incremental: !!ts[:incremental],
-            incremental_on: ts[:incremental] && ts[:incremental] != true ? ts[:incremental] : nil,
+            incremental: dumper.table_is_incremental?(ts),
+            incremental_on: ts.dig(:incremental, :on),
             # indexed_columns
-            tableName: table_name,
+            tableName: tname,
             columns: ts[:columns].map do |col|
+              coltype = col[:type]
+              coltype ||= ts[:model].column_for_attribute(col[:from]).sql_type if col[:from].is_a?(String)
               {
                 name: col[:warehouse_name],
                 description: col[:description],
-                type: col[:type] || ts[:model].column_for_attribute(col[:local_name]).sql_type,
+                type: coltype,
               }
             end,
           }
@@ -92,6 +91,14 @@ module InstDataShipper
         }
       end
 
+      def table_name(table_def)
+        table_prefix = config[:table_prefix]
+        table_prefix = table_prefix.present? ? "#{table_prefix}_" : nil
+        table_name = table_def[:warehouse_name]
+        table_name = table_prefix + table_name if table_prefix.present?
+        table_name
+      end
+
       def hosted_data_client
         @hosted_data_client ||= begin
           token = config[:token]
@@ -102,6 +109,8 @@ module InstDataShipper
             host = tok_content['host']
           end
 
+          host = "https://#{host}" unless host.include?('://')
+
           Faraday.new(url: host) do |faraday|
             faraday.request                     :multipart
             faraday.request                     :json
@@ -117,14 +126,16 @@ module InstDataShipper
 
       def parse_configuration(uri)
         super do |parsed_uri, cfg|
-          if parsed_uri.username.present?
+          if parsed_uri.user.present?
             # hosted-data://<JWT>:<hosted_data_domain>
-            cfg[:token] = parsed_uri.username
+            cfg[:token] = parsed_uri.user
             cfg[:host] = parsed_uri.host
           else
             # hosted-data://<JWT>
             cfg[:token] = parsed_uri.host
           end
+
+          cfg[:table_prefix] = parsed_uri.params[:table_prefix]
         end
       end
 

data/lib/inst_data_shipper/destinations/s3.rb

@@ -4,7 +4,7 @@ module InstDataShipper
       include Concerns::Chunking
 
       def chunk_data(generator, table:, extra: nil)
-        warehouse_name = table_def[:warehouse_name]
+        warehouse_name = table[:warehouse_name]
 
         super(generator) do |batch, idx|
           bits = [warehouse_name, extra, idx].compact

data/lib/inst_data_shipper/dumper.rb

@@ -5,7 +5,7 @@ module InstDataShipper
     define_hook :initialize_dump_batch
     define_hook :finalize_dump_batch
 
-    def self.perform_dump(destinations:)
+    def self.perform_dump(destinations)
       raise "Must subclass Dumper to use perform_dump" if self == Dumper
 
       dumper = new(destinations)
@@ -14,48 +14,134 @@ module InstDataShipper
       dumper.tracker
     end
 
-    protected
-
-    attr_reader :executor
+    def self.define(include: [], schema: , &blk)
+      Class.new(self) do
+        include(*include)
 
-    def initialize(destinations = nil, executor: nil)
-      @raw_destinations = destinations
-      @executor = executor
+        define_method(:enqueue_tasks, &blk)
+        define_method(:table_schemas) { schema }
+      end
     end
 
-    def enqueue_tasks
-      raise NotImplementedError
-    end
+    public
 
     def begin_dump
       raise "Dump already begun" unless @raw_destinations.present?
 
-      @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, status: 'in_progress')
+      @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, genre: export_genre, status: 'in_progress')
 
-      destinations.each do |dest|
-        dest.initialize_dump()
+      @batch_context = context = {
+        # TODO Allow to be hooked by Destination, likely via initialize_dump_batch and batch_context, so that if an earlier destination fails we can resend data
+        # TODO Consider behavior if last is still running
+        incremental_since: DumpBatch.where(genre: export_genre, status: 'completed').order(created_at: :desc).first&.created_at,
+      }
+
+      begin
+        begin
+          destinations.each do |dest|
+            dest.initialize_dump()
+          end
+
+          run_hook(:initialize_dump_batch, context)
+        ensure
+          @batch_context = nil
+          context[:tracker_id] = tracker.id
+          context[:origin_class] = batch_context[:origin_class] || self.class.to_s
+          context[:destinations] = @raw_destinations
+        end
+
+        Sidekiq::Batch.new.tap do |batch|
+          context[:root_bid] = batch.bid
+
+          batch.description = "HD #{export_genre} Export #{tracker.id} Root"
+          batch.context = context
+          batch.on(:success, "#{self.class}#finalize_dump")
+          batch.on(:death, "#{self.class}#cleanup_fatal_error!")
+          batch.jobs do
+            enqueue_tasks
+          rescue => ex
+            delayed :cleanup_fatal_error!
+            InstDataShipper.handle_suppressed_error(ex)
+          end
+        end
+      rescue => ex
+        if context
+          batch ||= Sidekiq::Batch.new.tap do |batch|
+            batch.description = "HD #{export_genre} Export #{tracker.id} Early Failure Cleanup"
+            batch.context = context
+            batch.jobs do
+              delayed :cleanup_fatal_error!
+            end
+          end
+        end
+        raise ex
       end
+    end
 
-      context = {}
-      run_hook(:initialize_dump_batch, context)
-
-      Sidekiq::Batch.new.tap do |batch|
-        batch.description = "HD #{export_genre} Export #{tracker.id} Root"
-        batch.context = {
-          **context,
-          root_bid: batch.bid,
-          tracker_id: tracker.id,
-          origin_class: batch_context[:origin_class] || self.class.to_s,
-          destinations: @raw_destinations,
-        }
-        batch.on(:success, "#{self.class}#finalize_dump")
-        batch.on(:death, "#{self.class}#cleanup_fatal_error!")
-        batch.jobs do
-          enqueue_tasks
+    def tracker
+      @tracker ||= batch_context[:tracker_id].present? ? DumpBatch.find(batch_context[:tracker_id]) : nil
+    end
+
+    def export_genre
+      self.class.to_s.gsub(/HD|ExportJob/, '')
+    end
+
+    def origin_class
+      batch_context[:origin_class]&.constantize || self.class
+    end
+
+    def table_is_incremental?(table_def)
+      if (inc = table_def[:incremental]).present?
+        differ = inc[:if]
+        return !!incremental_since if differ.nil?
+
+        differ = :"#{differ}".to_proc if differ.is_a?(Symbol)
+        differ = instance_exec(&differ) if differ.is_a?(Proc)
+        return !!differ
+      end
+
+      false
+    end
+
+    def incremental_since
+      batch_context[:incremental_since]
+    end
+
+    def lookup_table_schema(*identifiers)
+      identifiers.compact.each do |ident|
+        if ident.is_a?(Hash)
+          key = ident.keys.first
+          value = ident.values.first
+        else
+          key = :warehouse_name
+          value = ident
+        end
+
+        value = Array(value).compact
+
+        table_schemas.each do |ts|
+          return ts if value.include?(ts[key])
         end
       end
 
-      # TODO Catch errors in here and cleanup as needed
+      nil
+    end
+
+    def lookup_table_schema!(*identifiers)
+      lookup_table_schema(*identifiers) || raise("No table schema found for #{identifiers.inspect}")
+    end
+
+    protected
+
+    attr_reader :executor
+
+    def initialize(destinations = nil, executor: nil)
+      @raw_destinations = Array(destinations)
+      @executor = executor
+    end
+
+    def enqueue_tasks
+      raise NotImplementedError
     end
 
     def upload_data(table_def, extra: nil, &datagen)
@@ -96,7 +182,7 @@ module InstDataShipper
     def finalize_dump(_status, _opts)
       run_hook(:finalize_dump_batch)
 
-      destination.each do |dest|
+      destinations.each do |dest|
         dest.finalize_dump
       end
 
@@ -108,14 +194,15 @@ module InstDataShipper
 
       run_hook(:finalize_dump_batch)
 
-      destination.each do |dest|
+      destinations.each do |dest|
         dest.cleanup_fatal_error
-      rescue StandardError # rubocop:disable Lint/SuppressedException
+      rescue => ex
+        InstDataShipper.handle_suppressed_error(ex)
       end
 
       DumpBatch.find(batch_context[:tracker_id]).update(status: 'failed')
 
-      CanvasSync::JobBatches::Batch.delete_prematurely!(batch_context[:root_bid])
+      CanvasSync::JobBatches::Batch.delete_prematurely!(batch_context[:root_bid]) if batch_context[:root_bid].present?
     end
 
     # Helper Methods
@@ -126,23 +213,17 @@ module InstDataShipper
     end
 
     def delayed(mthd, *args, **kwargs)
-      AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
-    end
-
-    def tracker
-      @tracker ||= batch_context[:tracker_id].present? ? DumpBatch.find(batch_context[:tracker_id]) : nil
+      Jobs::AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
     end
 
-    def export_genre
-      self.class.to_s.gsub(/HD|ExportJob/, '')
-    end
+    delegate :working_dir, to: :executor
 
-    def origin_class
-      batch_context[:origin_class]&.constantize || self.class
+    def batch
+      Thread.current[CanvasSync::JobBatches::CURRENT_BATCH_THREAD_KEY]
     end
 
-    def working_dir
-      executor.working_dir
+    def batch_context
+      @batch_context || batch&.context || {}
     end
 
     def destinations_for_table(table_def)
@@ -150,7 +231,7 @@ module InstDataShipper
     end
 
     def destinations
-      @destinations ||= (@raw_destinations || batch_context[:destinations]).map.with_index do |dest, i|
+      @destinations ||= (@raw_destinations.presence || batch_context[:destinations]).map.with_index do |dest, i|
         dcls = InstDataShipper.resolve_destination(dest)
         dcls.new("#{InstDataShipper.redis_prefix}:dump#{tracker.id}:dest#{i}", dest, self)
       end

data/lib/inst_data_shipper/engine.rb

@@ -4,5 +4,11 @@ module InstDataShipper
   class Engine < ::Rails::Engine
     isolate_namespace InstDataShipper
 
+    initializer :append_migrations do |app|
+      config.paths["db/migrate"].expanded.each do |expanded_path|
+        app.config.paths["db/migrate"] << expanded_path
+      end
+      ActiveRecord::Migrator.migrations_paths = Rails.application.paths['db/migrate'].to_a
+    end
   end
 end

data/lib/inst_data_shipper/jobs/async_caller.rb

@@ -1,7 +1,14 @@
+
+require "sidekiq"
+
 module InstDataShipper
   module Jobs
     class AsyncCaller < InstDataShipper::Jobs::Base
-      sidekiq_options retry: 6 if defined?(sidekiq_options)
+      sidekiq_options(retry: 0) if defined?(sidekiq_options)
+
+      def self.get_sidekiq_options
+        { retry: 0 }
+      end
 
       def self.call_from_pool(pool, clazz, method, *args, **kwargs)
         pool.add_job(
@@ -12,7 +19,8 @@ module InstDataShipper
       end
 
       def perform(clazz, method, *args, **kwargs)
-        clazz.constantize.new(executor: self).send(method.to_sym, *args, **kwargs)
+        clazz = clazz.constantize if clazz.is_a?(String)
+        clazz.new(executor: self).send(method.to_sym, *args, **kwargs)
       end
     end
   end

data/lib/inst_data_shipper/jobs/basic_dump_job.rb

@@ -3,9 +3,13 @@ module InstDataShipper
     class BasicDumpJob < InstDataShipper::Jobs::Base
       sidekiq_options retry: 3 if defined?(sidekiq_options)
 
-      def perform(endpoints, dump_class = nil)
-        dumper.perform_dump(endpoints: endpoints)
+      def perform(endpoints)
+
       end
+
+      protected
+
+
     end
   end
 end

data/lib/inst_data_shipper/schema_builder.rb

@@ -1,4 +1,6 @@
 module InstDataShipper
+  # This class ends up fill two roles - Schema and Mapping.
+  # It makes for a clean API, but it's a little less canonical since, (eg) the S3 destination doesn't need column type annotations.
   class SchemaBuilder
     attr_reader :tables
 
@@ -12,24 +14,40 @@ module InstDataShipper
       builder.tables
     end
 
-    def table(model_or_name, description = nil, as: nil, includes: nil, incremental: false, &block)
-      as ||= model_or_name
-      as = as.table_name if as.respond_to?(:table_name)
+    def extend_table_builder(&block)
+      @table_builder_class ||= Class.new(TableSchemaBuilder)
+      @table_builder_class.class_eval(&block)
+    end
 
+    def table(model_or_name, description = nil, model: nil, query: nil, **extra, &block)
       tdef = {
+        warehouse_name: nil,
         description: description,
-        model: model_or_name.is_a?(String) ? nil : model_or_name,
-        warehouse_name: as.to_s,
-        incremental: incremental,
         columns: [],
-        includes: includes,
+
+        model: model,
+        query: query,
+        **extra,
       }
 
-      TableSchemaBuilder.build(tdef, &block)
+      if model_or_name.is_a?(ActiveRecord::Relation)
+        raise "model specified twice" if model.present?
+        raise "query specified twice" if query.present?
+
+        tdef[:query] = model_or_name
+        tdef[:model] = model_or_name.model
+      elsif model_or_name.is_a?(Class) && model_or_name < ActiveRecord::Base
+        tdef[:warehouse_name] = model_or_name.table_name
+        tdef[:model] = model_or_name
+      else
+        tdef[:warehouse_name] = model_or_name
+      end
+
+      @table_builder_class.build(tdef, &block)
 
       @tables << tdef
 
-      self
+      tdef
     end
 
     class TableSchemaBuilder
@@ -46,48 +64,82 @@ module InstDataShipper
         builder.columns
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      def column(name, *args, **kwargs, &block)
-        name = { name => name } unless name.is_a?(Hash)
-        raise ArgumentError, 'Exactly one name must be provided' unless name.count == 1
+      # def annotate(key, value)
+      #   options[key] = value
+      # end
+
+      def incremental(scope="updated_at", **kwargs)
+        if (extras = kwargs.keys - %i[on if]).present?
+          raise ArgumentError, "Unsuppored options: #{extras.inspect}"
+        end
+
+        options[:incremental] = {
+          on: Array(kwargs[:on]),
+          scope: scope,
+          if: kwargs[:if],
+        }
+      end
+
+      def column(name, *args, refs: [], from: nil, **extra, &block)
+        from ||= name.to_s
 
         cdef = {
-          local_name: name.keys[0].to_s,
-          warehouse_name: name.values[0].to_s,
-          transformer: block,
+          warehouse_name: name.to_s,
+          from: from,
+          **extra,
         }
 
-        [:description, :type, :refs => :references].each do |k|
-          if k.is_a? Hash
-            k.each do |hk, hv|
-              cdef[hv] = kwargs.delete(hk) if kwargs.key?(hk)
-            end
-          elsif kwargs.key?(k)
-            cdef[k] = kwargs.delete(k)
-          end
+        if args[0].is_a?(Symbol)
+          cdef[:type] = args.shift()
         end
 
-        cdef[:references] = Array(cdef[:references])
-
-        args[0..1].each do |a|
-          k = (a.is_a?(String) && :description) || (a.is_a?(Symbol) && :type) || nil
-          raise ArgumentError, 'Unsupported Argument' if k.nil?
-          raise ArgumentError, "Duplicate Argument for #{k}" if cdef.key?(k)
+        if args[0].is_a?(String)
+          cdef[:description] = args.shift()
+        end
 
-          cdef[k] = a
+        if args.present?
+          raise ArgumentError, "Received unexpected arguments: #{args.inspect}"
         end
 
+        cdef[:references] = Array(refs)
+
         if options[:model].is_a?(Class) && cdef[:local_name].to_s.ends_with?('_id')
           rel_name = cdef[:local_name].to_s[0...-3]
           refl = options[:model].reflections[rel_name]
           cdef[:references] << "#{refl.klass}##{refl.options[:primary_key] || 'id'}" if refl.present? && !refl.polymorphic?
         end
 
+        compiled_from = compile_transformer(from)
+
+        cdef[:block] = ->(row) {
+          value = instance_exec(row, &compiled_from)
+          value = instance_exec(value, row, &block) if block.present?
+          value
+        }
+
         @columns << cdef
 
-        self
+        cdef
       end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+      protected
+
+      def compile_transformer(from)
+        if from.present?
+          if from.is_a?(Symbol)
+            ->(row) { row.send(from) }
+          elsif from.is_a?(Proc)
+            from
+          elsif from.is_a?(String)
+            ->(row) { row[from] }
+          else
+            raise ArgumentError, "Invalid transformer: #{from.inspect}"
+          end
+        else
+          ->(row) { row }
+        end
+      end
+
     end
   end
 end

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.1.0.beta1".freeze
+  VERSION = "0.1.0.beta2".freeze
 end

data/lib/inst_data_shipper.rb

@@ -23,13 +23,20 @@ module InstDataShipper
         destination = @destination_aliases[type]
       end
 
-      safe_constantize(destination)
+      destination.constantize
     end
 
     def start_basic_dump(*args, **kwargs, &block)
       BasicDumper.perform_dump(*args, **kwargs, &block)
     end
 
+    def handle_suppressed_error(ex)
+      logger.error "Suppressed Error: #{ex.message}"
+      logger.error ex.backtrace.join("\n")
+      Raven.capture_exception(ex) if defined?(Raven)
+      Sentry.capture_exception(ex) if defined?(Sentry)
+    end
+
     def logger
       return @logger if defined? @logger
       @logger = Logger.new(STDOUT)
@@ -66,6 +73,8 @@ Dir[File.dirname(__FILE__) + "/inst_data_shipper/destinations/*.rb"].each do |fi
   basename = File.basename(file, ".rb")
   next if basename == "base"
 
-  InstDataShipper.alias_destination(basename.dasherize, "InstDataShipper::Destinations::#{basename.classify}")
+  InstDataShipper.alias_destination(basename.dasherize, "InstDataShipper::Destinations::#{basename.camelize}")
 end
 
+require "inst_data_shipper/dumper"
+require "inst_data_shipper/basic_dumper"

data/spec/spec_helper.rb

@@ -7,7 +7,7 @@ require File.expand_path("../dummy/config/environment.rb", __FILE__)
 require "bundler/setup"
 require 'rspec/rails'
 require 'spec_helper'
-require 'factory_girl_rails'
+require 'factory_bot_rails'
 require 'timecop'
 require 'webmock/rspec'
 require 'support/fake_canvas'
@@ -29,7 +29,7 @@ ActiveRecord::Migration.maintain_test_schema!
 RSpec.configure do |config|
   config.extend WithModel
 
-  config.include FactoryGirl::Syntax::Methods
+  config.include FactoryBot::Syntax::Methods
   config.use_transactional_fixtures = true
   config.infer_spec_type_from_file_location!
   config.filter_rails_from_backtrace!

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta1
+  version: 0.1.0.beta2
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-06 00:00:00.000000000 Z
+date: 2024-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
@@ -360,6 +360,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - pseng@instructure.com
@@ -369,8 +383,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
-- app/models/hosted_data_dumper/dump_batch.rb
-- db/migrate/20240301090836_create_canvas_sync_sync_batches.rb
+- app/models/inst_data_shipper/dump_batch.rb
+- db/migrate/20240301090836_create_inst_data_shipper_dump_batches.rb
 - lib/inst_data_shipper.rb
 - lib/inst_data_shipper/basic_dumper.rb
 - lib/inst_data_shipper/concerns/hooks.rb