inst_data_shipper 0.1.0.beta1 → 0.2.0

data/lib/inst_data_shipper/destinations/s3.rb

@@ -4,7 +4,7 @@ module InstDataShipper
       include Concerns::Chunking
 
       def chunk_data(generator, table:, extra: nil)
-        warehouse_name = table_def[:warehouse_name]
+        warehouse_name = table[:warehouse_name]
 
         super(generator) do |batch, idx|
           bits = [warehouse_name, extra, idx].compact

data/lib/inst_data_shipper/dumper.rb

@@ -5,7 +5,7 @@ module InstDataShipper
     define_hook :initialize_dump_batch
     define_hook :finalize_dump_batch
 
-    def self.perform_dump(destinations:)
+    def self.perform_dump(destinations)
       raise "Must subclass Dumper to use perform_dump" if self == Dumper
 
       dumper = new(destinations)
@@ -14,48 +14,166 @@ module InstDataShipper
       dumper.tracker
     end
 
-    protected
-
-    attr_reader :executor
+    def self.define(include: [], schema: , &blk)
+      Class.new(self) do
+        include(*include)
 
-    def initialize(destinations = nil, executor: nil)
-      @raw_destinations = destinations
-      @executor = executor
+        define_method(:enqueue_tasks, &blk)
+        define_method(:schema) { schema }
+      end
     end
 
-    def enqueue_tasks
-      raise NotImplementedError
+    def self.current(executor: nil)
+      cur_batch = Thread.current[CanvasSync::JobBatches::CURRENT_BATCH_THREAD_KEY]
+      ctx = cur_batch&.context || {}
+      return nil unless ctx[:origin_class].present? && ctx[:tracker_id].present?
+
+      clazz = ctx[:origin_class]
+      clazz = clazz.constantize if clazz.is_a?(String)
+      clazz.new(executor: executor)
     end
 
+    public
+
     def begin_dump
       raise "Dump already begun" unless @raw_destinations.present?
 
-      @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, status: 'in_progress')
+      @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, genre: export_genre, status: 'in_progress')
+
+      @batch_context = context = {
+        # TODO Consider behavior if last is still running
+        incremental_since: last_successful_tracker&.created_at,
+      }
 
       destinations.each do |dest|
-        dest.initialize_dump()
+        dest.preinitialize_dump(context)
+      end
+
+      begin
+        begin
+          destinations.each do |dest|
+            dest.initialize_dump(context)
+          end
+
+          run_hook(:initialize_dump_batch, context)
+        ensure
+          @batch_context = nil
+          context[:tracker_id] = tracker.id
+          context[:origin_class] = batch_context[:origin_class] || self.class.to_s
+          context[:destinations] = @raw_destinations
+        end
+
+        Sidekiq::Batch.new.tap do |batch|
+          context[:root_bid] = batch.bid
+          tracker.update(batch_id: batch.bid)
+
+          batch.description = "HD #{export_genre} Export #{tracker.id} Root"
+          batch.context = context
+          batch.on(:success, "#{self.class}#finalize_dump")
+          batch.on(:death, "#{self.class}#cleanup_fatal_error!")
+          batch.jobs do
+            enqueue_tasks
+          rescue => ex
+            delayed :cleanup_fatal_error!
+            InstDataShipper.handle_suppressed_error(ex)
+            tracker.update(status: 'failed', exception: ex.message, backtrace: ex.backtrace.join("\n"))
+          end
+        end
+      rescue => ex
+        if context
+          batch ||= Sidekiq::Batch.new.tap do |batch|
+            batch.description = "HD #{export_genre} Export #{tracker.id} Early Failure Cleanup"
+            batch.context = context
+            batch.jobs do
+              delayed :cleanup_fatal_error!
+            end
+          end
+        end
+        tracker.update(status: 'failed', exception: ex.message, backtrace: ex.backtrace.join("\n"))
+        raise ex
+      end
+    end
+
+    def tracker
+      @tracker ||= batch_context[:tracker_id].present? ? DumpBatch.find(batch_context[:tracker_id]) : nil
+    end
+
+    def last_successful_tracker
+      @last_successful_tracker ||= DumpBatch.where(job_class: self.class.to_s, genre: export_genre, status: 'completed').order(created_at: :desc).first
+    end
+
+    def export_genre
+      self.class.to_s
+    end
+
+    def origin_class
+      batch_context[:origin_class]&.constantize || self.class
+    end
+
+    def schema
+      return origin_class::SCHEMA if defined?(origin_class::SCHEMA)
+      raise NotImplementedError
+    end
+
+    def schema_digest
+      Digest::MD5.hexdigest(schema.to_json)[0...8]
+    end
+
+    def table_is_incremental?(table_def)
+      return false unless incremental_since.present?
+
+      # TODO Return false if table's schema changes
+      if (inc = table_def[:incremental]).present?
+        differ = inc[:if]
+        return !!incremental_since if differ.nil?
+
+        differ = :"#{differ}".to_proc if differ.is_a?(Symbol)
+        differ = instance_exec(&differ) if differ.is_a?(Proc)
+        return !!differ
       end
 
-      context = {}
-      run_hook(:initialize_dump_batch, context)
-
-      Sidekiq::Batch.new.tap do |batch|
-        batch.description = "HD #{export_genre} Export #{tracker.id} Root"
-        batch.context = {
-          **context,
-          root_bid: batch.bid,
-          tracker_id: tracker.id,
-          origin_class: batch_context[:origin_class] || self.class.to_s,
-          destinations: @raw_destinations,
-        }
-        batch.on(:success, "#{self.class}#finalize_dump")
-        batch.on(:death, "#{self.class}#cleanup_fatal_error!")
-        batch.jobs do
-          enqueue_tasks
+      false
+    end
+
+    def incremental_since
+      batch_context[:incremental_since]
+    end
+
+    def lookup_table_schema(*identifiers)
+      identifiers.compact.each do |ident|
+        if ident.is_a?(Hash)
+          key = ident.keys.first
+          value = ident.values.first
+        else
+          key = :warehouse_name
+          value = ident
+        end
+
+        value = Array(value).compact
+
+        schema[:tables].each do |ts|
+          return ts if value.include?(ts[key])
         end
       end
 
-      # TODO Catch errors in here and cleanup as needed
+      nil
+    end
+
+    def lookup_table_schema!(*identifiers)
+      lookup_table_schema(*identifiers) || raise("No table schema found for #{identifiers.inspect}")
+    end
+
+    protected
+
+    attr_reader :executor
+
+    def initialize(destinations = nil, executor: nil)
+      @raw_destinations = Array(destinations)
+      @executor = executor
+    end
+
+    def enqueue_tasks
+      raise NotImplementedError
     end
 
     def upload_data(table_def, extra: nil, &datagen)
@@ -96,7 +214,7 @@ module InstDataShipper
     def finalize_dump(_status, _opts)
       run_hook(:finalize_dump_batch)
 
-      destination.each do |dest|
+      destinations.each do |dest|
         dest.finalize_dump
       end
 
@@ -108,41 +226,31 @@ module InstDataShipper
 
       run_hook(:finalize_dump_batch)
 
-      destination.each do |dest|
+      destinations.each do |dest|
         dest.cleanup_fatal_error
-      rescue StandardError # rubocop:disable Lint/SuppressedException
+      rescue => ex
+        InstDataShipper.handle_suppressed_error(ex)
       end
 
       DumpBatch.find(batch_context[:tracker_id]).update(status: 'failed')
 
-      CanvasSync::JobBatches::Batch.delete_prematurely!(batch_context[:root_bid])
+      CanvasSync::JobBatches::Batch.delete_prematurely!(batch_context[:root_bid]) if batch_context[:root_bid].present?
     end
 
     # Helper Methods
 
-    def table_schemas
-      return origin_class::TABLE_SCHEMAS if defined?(origin_class::TABLE_SCHEMAS)
-      raise NotImplementedError
-    end
-
     def delayed(mthd, *args, **kwargs)
-      AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
+      Jobs::AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
     end
 
-    def tracker
-      @tracker ||= batch_context[:tracker_id].present? ? DumpBatch.find(batch_context[:tracker_id]) : nil
-    end
+    delegate :working_dir, to: :executor
 
-    def export_genre
-      self.class.to_s.gsub(/HD|ExportJob/, '')
-    end
-
-    def origin_class
-      batch_context[:origin_class]&.constantize || self.class
+    def batch
+      Thread.current[CanvasSync::JobBatches::CURRENT_BATCH_THREAD_KEY]
     end
 
-    def working_dir
-      executor.working_dir
+    def batch_context
+      @batch_context || batch&.context || {}
     end
 
     def destinations_for_table(table_def)
@@ -150,7 +258,7 @@ module InstDataShipper
     end
 
     def destinations
-      @destinations ||= (@raw_destinations || batch_context[:destinations]).map.with_index do |dest, i|
+      @destinations ||= (@raw_destinations.presence || batch_context[:destinations]).map.with_index do |dest, i|
         dcls = InstDataShipper.resolve_destination(dest)
         dcls.new("#{InstDataShipper.redis_prefix}:dump#{tracker.id}:dest#{i}", dest, self)
       end

data/lib/inst_data_shipper/engine.rb

@@ -4,5 +4,11 @@ module InstDataShipper
   class Engine < ::Rails::Engine
     isolate_namespace InstDataShipper
 
+    initializer :append_migrations do |app|
+      config.paths["db/migrate"].expanded.each do |expanded_path|
+        app.config.paths["db/migrate"] << expanded_path
+      end
+      ActiveRecord::Migrator.migrations_paths = Rails.application.paths['db/migrate'].to_a
+    end
   end
 end

data/lib/inst_data_shipper/jobs/async_caller.rb

@@ -1,7 +1,14 @@
+
+require "sidekiq"
+
 module InstDataShipper
   module Jobs
     class AsyncCaller < InstDataShipper::Jobs::Base
-      sidekiq_options retry: 6 if defined?(sidekiq_options)
+      sidekiq_options(retry: 0) if defined?(sidekiq_options)
+
+      def self.get_sidekiq_options
+        { retry: 0 }
+      end
 
       def self.call_from_pool(pool, clazz, method, *args, **kwargs)
         pool.add_job(
@@ -12,7 +19,8 @@ module InstDataShipper
       end
 
       def perform(clazz, method, *args, **kwargs)
-        clazz.constantize.new(executor: self).send(method.to_sym, *args, **kwargs)
+        clazz = clazz.constantize if clazz.is_a?(String)
+        clazz.new(executor: self).send(method.to_sym, *args, **kwargs)
       end
     end
   end

data/lib/inst_data_shipper/schema_builder.rb

@@ -1,35 +1,59 @@
 module InstDataShipper
+  # This class ends up fill two roles - Schema and Mapping.
+  # It makes for a clean API, but it's a little less canonical since, (eg) the S3 destination doesn't need column type annotations.
   class SchemaBuilder
-    attr_reader :tables
+    attr_reader :schema
 
     def initialize
-      @tables = []
+      @schema = {
+        tables: [],
+      }
     end
 
     def self.build(&block)
       builder = new
       builder.instance_exec(&block)
-      builder.tables
+      builder.schema
+    end
+
+    def version(version)
+      @schema[:version] = version
     end
 
-    def table(model_or_name, description = nil, as: nil, includes: nil, incremental: false, &block)
-      as ||= model_or_name
-      as = as.table_name if as.respond_to?(:table_name)
+    def extend_table_builder(&block)
+      @table_builder_class ||= Class.new(TableSchemaBuilder)
+      @table_builder_class.class_eval(&block)
+    end
 
+    def table(model_or_name, description = nil, model: nil, query: nil, **extra, &block)
       tdef = {
+        warehouse_name: nil,
         description: description,
-        model: model_or_name.is_a?(String) ? nil : model_or_name,
-        warehouse_name: as.to_s,
-        incremental: incremental,
         columns: [],
-        includes: includes,
+
+        model: model,
+        query: query,
+        **extra,
       }
 
-      TableSchemaBuilder.build(tdef, &block)
+      if model_or_name.is_a?(ActiveRecord::Relation)
+        raise "model specified twice" if model.present?
+        raise "query specified twice" if query.present?
+
+        tdef[:query] = model_or_name
+        tdef[:model] = model_or_name.model
+      elsif model_or_name.is_a?(Class) && model_or_name < ActiveRecord::Base
+        tdef[:warehouse_name] = model_or_name.table_name
+        tdef[:model] = model_or_name
+      else
+        tdef[:warehouse_name] = model_or_name
+      end
+
+      @table_builder_class.build(tdef, &block)
 
-      @tables << tdef
+      @schema[:tables] << tdef
 
-      self
+      tdef
     end
 
     class TableSchemaBuilder
@@ -46,48 +70,86 @@ module InstDataShipper
         builder.columns
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      def column(name, *args, **kwargs, &block)
-        name = { name => name } unless name.is_a?(Hash)
-        raise ArgumentError, 'Exactly one name must be provided' unless name.count == 1
+      # def annotate(key, value)
+      #   options[key] = value
+      # end
+
+      def version(version)
+        options[:version] = version
+      end
+
+      def incremental(scope=nil, **kwargs)
+        if (extras = kwargs.keys - %i[on if]).present?
+          raise ArgumentError, "Unsuppored options: #{extras.inspect}"
+        end
+
+        options[:incremental] = {
+          on: Array(kwargs[:on]),
+          scope: scope,
+          if: kwargs[:if],
+        }
+      end
+
+      def column(name, *args, refs: [], from: nil, **extra, &block)
+        from ||= name.to_s
 
         cdef = {
-          local_name: name.keys[0].to_s,
-          warehouse_name: name.values[0].to_s,
-          transformer: block,
+          warehouse_name: name.to_s,
+          from: from,
+          **extra,
         }
 
-        [:description, :type, :refs => :references].each do |k|
-          if k.is_a? Hash
-            k.each do |hk, hv|
-              cdef[hv] = kwargs.delete(hk) if kwargs.key?(hk)
-            end
-          elsif kwargs.key?(k)
-            cdef[k] = kwargs.delete(k)
-          end
+        if args[0].is_a?(Symbol)
+          cdef[:type] = args.shift()
         end
 
-        cdef[:references] = Array(cdef[:references])
-
-        args[0..1].each do |a|
-          k = (a.is_a?(String) && :description) || (a.is_a?(Symbol) && :type) || nil
-          raise ArgumentError, 'Unsupported Argument' if k.nil?
-          raise ArgumentError, "Duplicate Argument for #{k}" if cdef.key?(k)
+        if args[0].is_a?(String)
+          cdef[:description] = args.shift()
+        end
 
-          cdef[k] = a
+        if args.present?
+          raise ArgumentError, "Received unexpected arguments: #{args.inspect}"
         end
 
+        cdef[:references] = Array(refs)
+
         if options[:model].is_a?(Class) && cdef[:local_name].to_s.ends_with?('_id')
           rel_name = cdef[:local_name].to_s[0...-3]
           refl = options[:model].reflections[rel_name]
           cdef[:references] << "#{refl.klass}##{refl.options[:primary_key] || 'id'}" if refl.present? && !refl.polymorphic?
         end
 
+        compiled_from = compile_transformer(from)
+
+        cdef[:block] = ->(row) {
+          value = instance_exec(row, &compiled_from)
+          value = instance_exec(value, row, &block) if block.present?
+          value
+        }
+
         @columns << cdef
 
-        self
+        cdef
+      end
+
+      protected
+
+      def compile_transformer(from)
+        if from.present?
+          if from.is_a?(Symbol)
+            ->(row) { row.send(from) }
+          elsif from.is_a?(Proc)
+            from
+          elsif from.is_a?(String)
+            ->(row) { row[from] }
+          else
+            raise ArgumentError, "Invalid transformer: #{from.inspect}"
+          end
+        else
+          ->(row) { row }
+        end
       end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
     end
   end
 end

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.1.0.beta1".freeze
+  VERSION = "0.2.0".freeze
 end

data/lib/inst_data_shipper.rb

@@ -23,15 +23,23 @@ module InstDataShipper
         destination = @destination_aliases[type]
       end
 
-      safe_constantize(destination)
+      destination.constantize
     end
 
     def start_basic_dump(*args, **kwargs, &block)
       BasicDumper.perform_dump(*args, **kwargs, &block)
     end
 
+    def handle_suppressed_error(ex)
+      logger.error "Suppressed Error: #{ex.message}"
+      logger.error ex.backtrace.join("\n")
+      Raven.capture_exception(ex) if defined?(Raven)
+      Sentry.capture_exception(ex) if defined?(Sentry)
+    end
+
     def logger
       return @logger if defined? @logger
+      # TODO Annotate logs with DumpBatch ID
       @logger = Logger.new(STDOUT)
       @logger.level = Logger::DEBUG
       @logger
@@ -42,7 +50,7 @@ module InstDataShipper
     end
 
     def redis_prefix
-      pfx = "hdd"
+      pfx = "ids"
       pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
       pfx
     end
@@ -66,6 +74,8 @@ Dir[File.dirname(__FILE__) + "/inst_data_shipper/destinations/*.rb"].each do |fi
   basename = File.basename(file, ".rb")
   next if basename == "base"
 
-  InstDataShipper.alias_destination(basename.dasherize, "InstDataShipper::Destinations::#{basename.classify}")
+  InstDataShipper.alias_destination(basename.dasherize, "InstDataShipper::Destinations::#{basename.camelize}")
 end
 
+require "inst_data_shipper/dumper"
+require "inst_data_shipper/basic_dumper"

data/spec/spec_helper.rb

@@ -7,7 +7,7 @@ require File.expand_path("../dummy/config/environment.rb", __FILE__)
 require "bundler/setup"
 require 'rspec/rails'
 require 'spec_helper'
-require 'factory_girl_rails'
+require 'factory_bot_rails'
 require 'timecop'
 require 'webmock/rspec'
 require 'support/fake_canvas'
@@ -29,7 +29,7 @@ ActiveRecord::Migration.maintain_test_schema!
 RSpec.configure do |config|
   config.extend WithModel
 
-  config.include FactoryGirl::Syntax::Methods
+  config.include FactoryBot::Syntax::Methods
   config.use_transactional_fixtures = true
   config.infer_spec_type_from_file_location!
   config.filter_rails_from_backtrace!

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta1
+  version: 0.2.0
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-06 00:00:00.000000000 Z
+date: 2024-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
@@ -360,6 +360,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - pseng@instructure.com
@@ -369,8 +383,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
-- app/models/hosted_data_dumper/dump_batch.rb
-- db/migrate/20240301090836_create_canvas_sync_sync_batches.rb
+- app/models/inst_data_shipper/dump_batch.rb
+- db/migrate/20240301090836_create_inst_data_shipper_dump_batches.rb
 - lib/inst_data_shipper.rb
 - lib/inst_data_shipper/basic_dumper.rb
 - lib/inst_data_shipper/concerns/hooks.rb
@@ -385,7 +399,6 @@ files:
 - lib/inst_data_shipper/engine.rb
 - lib/inst_data_shipper/jobs/async_caller.rb
 - lib/inst_data_shipper/jobs/base.rb
-- lib/inst_data_shipper/jobs/basic_dump_job.rb
 - lib/inst_data_shipper/record.rb
 - lib/inst_data_shipper/schema_builder.rb
 - lib/inst_data_shipper/version.rb
@@ -422,9 +435,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.6
 signing_key: 

data/lib/inst_data_shipper/jobs/basic_dump_job.rb

@@ -1,11 +0,0 @@
-module InstDataShipper
-  module Jobs
-    class BasicDumpJob < InstDataShipper::Jobs::Base
-      sidekiq_options retry: 3 if defined?(sidekiq_options)
-
-      def perform(endpoints, dump_class = nil)
-        dumper.perform_dump(endpoints: endpoints)
-      end
-    end
-  end
-end