inst_data_shipper 0.1.0.beta2 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 146f5b93819d7950f9bd256a99eb690a63d453b86be4ac6ac7cf4c5901724cdd
-  data.tar.gz: 2410298ebb3b1ddc565ca70d49a274a129e83087d461dcfae4d4981979795ea5
+  metadata.gz: 4c84d0f1543bfcb42bce30e69ff12cc683b6f6bf51e9275f7529dd0b070bbb79
+  data.tar.gz: 22b5128af3f2d03b14565c656abca685c6e7196083576d1fd20a42a834690cba
 SHA512:
-  metadata.gz: c6dc93902e0ef7a114d2434d3901c677021d57603b07c1122d72bd2f953184b207d0e57f5441fafb7035a00bf28f2e5035d34cd0edfb73da6d2f93d93874f344
-  data.tar.gz: 2e7babf6a2ed86f9a2e5769bfb393549fb07b771b700513efd640ba2b98ebc3eeadec99578e4c828738903ef121f8e3bebe8490f6f75de0ce9afac43ac28b8fa
+  metadata.gz: 14e5eb97d9c5dbe5049eb94cd3690a2ab59df7bf6e5e0c62b21e8893f4cb75ca294cce6d28a4b0f7dccd1c53d34d5c7a34be7b2f4699f1b320f74df14d50f672
+  data.tar.gz: e7d22728140a7fc331abb5f608da98d44013d49027bf932ef183a4f75babe797632aca83955cb77a205d6004d29a7377dde87aaba0c245a3564541b7e1f55cbe

data/README.md

@@ -1,6 +1,6 @@
 # InstDataShipper
 
-This gem is intended to facilitate fast and easy syncing of Canvas data.
+This gem is intended to facilitate easy upload of LTI datasets to Instructure Hosted Data.
 
 ## Installation
 
@@ -16,6 +16,157 @@ Then run the migrations:
 bundle exec rake db:migrate
 ```
 
+## Usage
+
+### Dumper
+
+The main tool provided by this Gem is the `InstDataDumper::Dumper` class. It is used to define a "Dump" which is a combination of tasks and schema.
+
+Here is an example `Dumper` implementation, wrapped in an ActiveJob job:
+```ruby
+class HostedDataPushJob < ApplicationJob
+  # The schema serves two purposes: defining the schema and mapping data
+  SCHEMA = InstDataShipper::SchemaBuilder.build do
+    # You can augment the Table-builder DSL with custom methods like so:
+    extend_table_builder do
+      # It may be useful to define a custom column definition helpers:
+      def custom_column(*args, from: nil, **kwargs, &blk)
+        # In this example, the helper reads the value from a `data` jsonb column - without it, you'd need
+        #   to define `from: ->(row) { row.data["<KEY>"] }` on each column that needs to read from the jsonb
+        from ||= args[0].to_s
+        from = ->(row) { row.data[from] } if from.is_a?(String)
+        column(*args, **kwargs, from: from, &blk)
+      end
+
+      # `extend_table_builder` uses `class_eval`, so you could alternatively write your helpers in a Concern or Module and include them like normal:
+      include SomeConcern
+    end
+
+    table(ALocalModel, "<TABLE DESCRIPTION>") do
+      # If you define a table as incremental, it'll only export changes made since the start of the last successful Dumper run
+      #  The first argument "scope" can be interpreted in different ways:
+      #    If exporting a local model it may be a: (default: `updated_at`)
+      #      Proc that will receive a Relation and return a Relation (use `incremental_since`)
+      #      String of a column to compare with `incremental_since`
+      #    If exporting a Canvas report it may be a: (default: `updated_after`)
+      #      Proc that will receive report params and return modified report params (use `incremental_since`)
+      #      String of a report param to set to `incremental_since`
+      #  `on:` is passed to Hosted Data and is used as the unique key. It may be an array to form a composite-key
+      #  `if:` may be a Proc or a Symbol (of a method on the Dumper)
+      incremental "updated_at", on: [:id], if: ->() {}
+
+      # Schema's may declaratively define the data source.
+      # This can be used for basic schemas where there's a 1:1 mapping between source table and destination table, and there is no conditional logic that needs to be performed.
+      # In order to apply these statements, your Dumper must call `auto_enqueue_from_schema`.
+      source :local_table
+      # A Proc can also be passed. The below is equivalent to the above
+      source ->(table_def) { import_local_table(table_def[:model] || table_def[:warehouse_name]) }
+
+      column :name_in_destinations, :maybe_optional_sql_type, "Optional description of column"
+
+      # The type may usually be omitted if the `table()` is passed a Model class, but strings are an exception to this
+      custom_column :name, :"varchar(128)"
+
+      # `from:` May be...
+      # A Symbol of a method to be called on the record
+      custom_column :sis_type, :"varchar(32)", from: :some_model_method
+      # A String of a column to read from the record
+      custom_column :sis_type, :"varchar(32)", from: "sis_source_type"
+      # A Proc to be called with each record
+      custom_column :sis_type, :"varchar(32)", from: ->(rec) { ... }
+      # Not specified. Will default to using the Schema Column Name as a String ("sis_type" in this case)
+      custom_column :sis_type, :"varchar(32)"
+    end
+
+    table("my_table", model: ALocalModel) do
+      # ...
+    end
+
+    table("proserv_student_submissions_csv") do
+      column :canvas_id, :bigint, from: "canvas user id"
+      column :sis_id, :"varchar(64)", from: "sis user id"
+      column :name, :"varchar(64)", from: "user name"
+      column :submission_id, :bigint, from: "submission id"
+    end
+  end
+
+  Dumper = InstDataShipper::Dumper.define(schema: SCHEMA, include: [
+    InstDataShipper::DataSources::LocalTables,
+    InstDataShipper::DataSources::CanvasReports,
+  ]) do
+    import_local_table(ALocalModel)
+    import_canvas_report_by_terms("proserv_student_submissions_csv", terms: Term.all.pluck(:canvas_id))
+
+    # If the report_name/Model don't directly match the Schema, a schema_name: parameter may be passed:
+    import_local_table(SomeModel, schema_name: "my_table")
+    import_canvas_report_by_terms("some_report", terms: Term.all.pluck(:canvas_id), schema_name: "my_table")
+
+    # Iterate through the Tables defined in the Schema and apply any defined `source` statements.
+    # This is the default behavior if `define()` is called w/o a block.
+    auto_enqueue_from_schema
+  end
+
+  def perform
+    Dumper.perform_dump([
+      "hosted-data://<JWT>@<HOSTED DATA SERVER>?table_prefix=example",
+      "s3://<access_key_id>:<access_key_secret>@<region>/<bucket>/<path>",
+    ])
+  end
+end
+```
+
+`Dumper`s may also be formed as a normal Ruby subclass:
+```ruby
+class HostedDataPushJob < ApplicationJob
+  SCHEMA = InstDataShipper::SchemaBuilder.build do
+    # ...
+  end
+
+  class Dumper < InstDataShipper::Dumper
+    include InstDataShipper::DataSources::LocalTables
+    include InstDataShipper::DataSources::CanvasReports
+
+    def enqueue_tasks
+      import_local_table(ALocalModel)
+      import_canvas_report_by_terms("proserv_student_submissions_csv", terms: Term.all.pluck(:canvas_id))
+
+      # auto_enqueue_from_schema
+    end
+
+    def table_schemas
+      SCHEMA
+    end
+  end
+
+  def perform
+    Dumper.perform_dump([
+      "hosted-data://<JWT>@<HOSTED DATA SERVER>?table_prefix=example",
+      "s3://<access_key_id>:<access_key_secret>@<region>/<bucket>/<path>",
+    ])
+  end
+end
+```
+
+### Destinations
+
+This Gem is mainly designed for use with Hosted Data, but it tries to abstract that a little to allow for other destinations/backends. Out of the box, support for Hosted Data and S3 are included.
+
+Destinations are passed as URI-formatted strings. Passing Hashes is also supported, but the format/keys are destination specific.
+
+Destinations blindly accept URI Fragments (the `#` chunk at the end of the URI). These options are not used internally but will be made available as `dest.user_config`. Ideally these are in the same format as query parameters (`x=1&y=2`, which it will try to parse into a Hash), but it can be any string.
+
+#### Hosted Data
+`hosted-data://<JWT>@<HOSTED DATA SERVER>`
+
+##### Optional Parameters:
+- `table_prefix`: An optional string to prefix onto each table name in the schema when declaring the schema in Hosted Data
+
+#### S3
+`s3://<access_key_id>:<access_key_secret>@<region>/<bucket>/<optional path>`
+
+##### Optional Parameters:
+_None_
+
 ## Development
 
 When adding to or updating this gem, make sure you do the following:

data/db/migrate/20240301090836_create_inst_data_shipper_dump_batches.rb

@@ -7,11 +7,12 @@ class CreateInstDataShipperDumpBatches < CanvasSync::MiscHelper::MigrationClass
 
       t.string :job_class
       t.string :genre
+      t.string :batch_id
 
       t.string :exception
       t.text :backtrace
-      t.text :metadata
-      t.text :job_arguments
+      # t.text :metadata
+      # t.text :job_arguments
 
       t.timestamps
     end

data/lib/inst_data_shipper/basic_dumper.rb

@@ -19,7 +19,7 @@ module InstDataShipper
       instance_exec(&@body_block)
     end
 
-    def table_schemas
+    def schema
       pointer = @schema_pointer || batch_context[:schema_pointer]
       pointer.constantize
     end

data/lib/inst_data_shipper/data_sources/base.rb

@@ -1,7 +1,31 @@
 module InstDataShipper
   module DataSources
     module Base
-      extend ActiveSupport::Concern
+      # This could be a Concern, but we don't want Concern inheritance logic kicking in if it is included into another Concern.
+
+      def self.included(base)
+        base.extend ActiveSupport::Concern unless self.is_a?(Class) || base < ActiveSupport::Concern
+        base.extend ModuleHelperMethods
+        base.send(:include, Concern)
+      end
+
+      module Concern
+        extend ActiveSupport::Concern
+      end
+
+      module ModuleHelperMethods
+        def delayed(mthd_sym)
+          mthd = instance_method(mthd_sym)
+          pmthd_sym = :"_delayed_#{mthd_sym}"
+
+          alias_method pmthd_sym, mthd_sym
+          private pmthd_sym
+
+          define_method(mthd_sym) do |*args, **kwargs|
+            delayed(pmthd_sym, *args, **kwargs)
+          end
+        end
+      end
     end
   end
 end

data/lib/inst_data_shipper/data_sources/canvas_reports.rb

@@ -2,7 +2,7 @@ module InstDataShipper
   module DataSources
     # This module contains the logic for processing Canvas reports
     module CanvasReports
-      extend ActiveSupport::Concern
+      include Base
 
       included do
         hook :initialize_dump_batch do |context|

data/lib/inst_data_shipper/data_sources/local_tables.rb

@@ -2,17 +2,11 @@ module InstDataShipper
   module DataSources
     # This module contains the logic for processing local AR tables
     module LocalTables
-      extend ActiveSupport::Concern
+      include Base
 
       public
 
-      def import_local_table(*args, **kwargs)
-        delayed(:_import_local_table, *args, **kwargs)
-      end
-
-      private
-
-      def _import_local_table(model, schema_name: nil)
+      delayed def import_local_table(model, schema_name: nil)
         model = model.safe_constantize if model.is_a?(String)
 
         table_def = lookup_table_schema!(schema_name, { model: model })
@@ -22,7 +16,12 @@ module InstDataShipper
           query = _resolve_model_query(query, table_def[:query])
 
           if table_is_incremental?(table_def)
-            query = _resolve_model_query(query, table_def.dig(:incremental, :scope), string: ->(r, c) { r.where("? > ?", c, incremental_since) })
+            query = _resolve_model_query(
+              query,
+              table_def.dig(:incremental, :scope),
+              string: ->(query, column) { query.where("#{column} > ?", incremental_since) },
+              default: "updated_at",
+            )
           end
 
           query.find_each do |m|
@@ -35,7 +34,11 @@ module InstDataShipper
         upload_data(table_def, &inner_block)
       end
 
-      def _resolve_model_query(relation, query, string: nil)
+      private
+
+      def _resolve_model_query(relation, query, string: nil, default: nil)
+        return relation if query == false
+        query = default if query.nil?
         return relation if query.nil?
 
         if query.is_a?(Symbol)

data/lib/inst_data_shipper/destinations/base.rb

@@ -3,7 +3,7 @@ module InstDataShipper
     class Base
       attr_reader :dumper
 
-      delegate :tracker, :table_schemas, :working_dir, to: :dumper
+      delegate :tracker, :schema, :working_dir, to: :dumper
 
       def initialize(cache_key, config, dumper)
         @cache_key = cache_key
@@ -11,9 +11,13 @@ module InstDataShipper
         @dumper = dumper
       end
 
+      # This method is called before taking any actions.
+      # It should be used to make any necessarry state assumptions (eg, the HostedData destination checks for a previous dump to determine if it can use incremental_since)
+      def preinitialize_dump(context); end
+
       # This method is called before processing any data.
       # It should be used to initialize any external resources needed for the dump.
-      def initialize_dump; end
+      def initialize_dump(context); end
 
       # Yields an object (can be anything) that will be passed to `upload_data_chunk` as `chunk`.
       #

data/lib/inst_data_shipper/destinations/concerns/chunking.rb

@@ -7,7 +7,7 @@ module InstDataShipper
         DEFAULT_CHUNK_SIZE = 100_000
 
         def chunk_data(generator, chunk_size: nil, **kwargs)
-          chunk_size ||= config.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
+          chunk_size ||= config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
           slice = 1
 
           btchr = CanvasSync::BatchProcessor.new(of: chunk_size) do |batch|
@@ -24,7 +24,7 @@ module InstDataShipper
 
         def group_key
           super.tap do |k|
-            k[:chunk_size] = config.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
+            k[:chunk_size] = config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
           end
         end
 

data/lib/inst_data_shipper/destinations/hosted_data.rb

@@ -5,11 +5,43 @@ module InstDataShipper
     class HostedData < Base
       include Concerns::Chunking
 
-      def initialize_dump
+      def preinitialize_dump(context)
+        if context[:incremental_since].present?
+          begin
+            last_dump = hosted_data_client.get("api/v1/custom_dumps/last", {
+              status: 'imported',
+              # schema_version: convert_schema[:version],
+              tags: [
+                "ids-schema=#{dumper.schema_digest}",
+                "ids-genre=#{dumper.export_genre}",
+              ],
+            }).body.with_indifferent_access
+
+            if last_dump[:created_at] < context[:incremental_since]
+              InstDataShipper.logger.info("Last successful HostedData dump is older than incremental_since - bumping back incremental_since")
+              context[:incremental_since] = last_dump[:created_at]
+            end
+          rescue Faraday::ResourceNotFound
+            # TODO It'd be nice to make this per-table
+            InstDataShipper.logger.info("No Last successful HostedData dump of the same schema - not using incremental_since")
+            context[:incremental_since] = nil
+          end
+        end
+      end
+
+      def initialize_dump(context)
+        tags = [
+          "ids-schema=#{dumper.schema_digest}",
+          "ids-genre=#{dumper.export_genre}",
+        ]
+        tags << "ids-app=#{Rails.application.class.name.gsub(/::Application$/, '')}" if defined?(Rails) && Rails.application
+        tags << "ids-schema-version=#{schema[:version]}" if schema[:version].present?
+
         dump = hosted_data_client.post(
           'api/v1/custom_dumps/',
           reference_id: tracker.id,
           schema: convert_schema,
+          tags: tags,
         ).body.with_indifferent_access
 
         redis.hset(rk(:state), :dump_id, dump[:id])
@@ -62,6 +94,7 @@ module InstDataShipper
 
       def convert_schema
         definititions = {}
+        table_schemas = schema[:tables]
         table_schemas.each do |ts|
           ts = ts.dup
           tname = table_name(ts)
@@ -86,7 +119,7 @@ module InstDataShipper
         end
 
         {
-          version: "#{dumper.export_genre.downcase}-#{Digest::MD5.hexdigest(definititions.to_json)[0...6]}",
+          version: "#{dumper.schema_digest}-#{Digest::MD5.hexdigest(definititions.to_json)[0...6]}",
           definition: definititions,
         }
       end

data/lib/inst_data_shipper/destinations/speccable.rb

@@ -0,0 +1,17 @@
+module InstDataShipper
+  module Destinations
+    class Speccable < Base
+      include Concerns::Chunking
+
+      def chunk_data(generator, table:, extra: nil)
+        super(generator) do |batch, idx|
+          yield batch
+        end
+      end
+
+      def upload_data_chunk(table_def, chunk); end
+
+      def parse_configuration(uri); end
+    end
+  end
+end

data/lib/inst_data_shipper/dumper.rb

@@ -18,8 +18,34 @@ module InstDataShipper
       Class.new(self) do
         include(*include)
 
+        if blk.nil? && schema[:tables].any? { |t| t[:sourcer].present? }
+          blk = -> { auto_enqueue_from_schema }
+        elsif blk.nil?
+          raise ArgumentError, "Must provide a block or a schema with source definitions"
+        end
+
         define_method(:enqueue_tasks, &blk)
-        define_method(:table_schemas) { schema }
+        define_method(:schema) { schema }
+      end
+    end
+
+    def self.current(executor: nil)
+      cur_batch = Thread.current[CanvasSync::JobBatches::CURRENT_BATCH_THREAD_KEY]
+      ctx = cur_batch&.context || {}
+      return nil unless ctx[:origin_class].present? && ctx[:tracker_id].present?
+
+      clazz = ctx[:origin_class]
+      clazz = clazz.constantize if clazz.is_a?(String)
+      clazz.new(executor: executor)
+    end
+
+    if defined?(Rails) && Rails.env.test?
+      def for_specs!
+        @raw_destinations = ["speccable://nil"]
+        @executor = InstDataShipper::Jobs::AsyncCaller.new
+        @tracker = DumpBatch.new(job_class: self.class.to_s, genre: export_genre, status: 'in_progress')
+        define_singleton_method(:spec_destination) { destinations.first }
+        self
       end
     end
 
@@ -31,15 +57,18 @@ module InstDataShipper
       @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, genre: export_genre, status: 'in_progress')
 
       @batch_context = context = {
-        # TODO Allow to be hooked by Destination, likely via initialize_dump_batch and batch_context, so that if an earlier destination fails we can resend data
         # TODO Consider behavior if last is still running
-        incremental_since: DumpBatch.where(genre: export_genre, status: 'completed').order(created_at: :desc).first&.created_at,
+        incremental_since: last_successful_tracker&.created_at,
       }
 
+      destinations.each do |dest|
+        dest.preinitialize_dump(context)
+      end
+
       begin
         begin
           destinations.each do |dest|
-            dest.initialize_dump()
+            dest.initialize_dump(context)
           end
 
           run_hook(:initialize_dump_batch, context)
@@ -52,6 +81,7 @@ module InstDataShipper
 
         Sidekiq::Batch.new.tap do |batch|
           context[:root_bid] = batch.bid
+          tracker.update(batch_id: batch.bid)
 
           batch.description = "HD #{export_genre} Export #{tracker.id} Root"
           batch.context = context
@@ -62,6 +92,7 @@ module InstDataShipper
           rescue => ex
             delayed :cleanup_fatal_error!
             InstDataShipper.handle_suppressed_error(ex)
+            tracker.update(status: 'failed', exception: ex.message, backtrace: ex.backtrace.join("\n"))
           end
         end
       rescue => ex
@@ -74,6 +105,7 @@ module InstDataShipper
             end
           end
         end
+        tracker.update(status: 'failed', exception: ex.message, backtrace: ex.backtrace.join("\n"))
         raise ex
       end
     end
@@ -82,15 +114,31 @@ module InstDataShipper
       @tracker ||= batch_context[:tracker_id].present? ? DumpBatch.find(batch_context[:tracker_id]) : nil
     end
 
+    def last_successful_tracker
+      @last_successful_tracker ||= DumpBatch.where(job_class: self.class.to_s, genre: export_genre, status: 'completed').order(created_at: :desc).first
+    end
+
     def export_genre
-      self.class.to_s.gsub(/HD|ExportJob/, '')
+      self.class.to_s
     end
 
     def origin_class
       batch_context[:origin_class]&.constantize || self.class
     end
 
+    def schema
+      return origin_class::SCHEMA if defined?(origin_class::SCHEMA)
+      raise NotImplementedError
+    end
+
+    def schema_digest
+      Digest::MD5.hexdigest(schema.to_json)[0...8]
+    end
+
     def table_is_incremental?(table_def)
+      return false unless incremental_since.present?
+
+      # TODO Return false if table's schema changes
       if (inc = table_def[:incremental]).present?
         differ = inc[:if]
         return !!incremental_since if differ.nil?
@@ -119,7 +167,7 @@ module InstDataShipper
 
         value = Array(value).compact
 
-        table_schemas.each do |ts|
+        schema[:tables].each do |ts|
           return ts if value.include?(ts[key])
         end
       end
@@ -144,6 +192,14 @@ module InstDataShipper
       raise NotImplementedError
     end
 
+    def auto_enqueue_from_schema
+      schema[:tables].each do |table_def|
+        src = table_def[:sourcer]
+        next unless src.present?
+        instance_exec(table_def, &src)
+      end
+    end
+
     def upload_data(table_def, extra: nil, &datagen)
       # Allow muxing, allowing a hook to prevent some files going to certain destinations
       dests = destinations_for_table(table_def)
@@ -207,11 +263,6 @@ module InstDataShipper
 
     # Helper Methods
 
-    def table_schemas
-      return origin_class::TABLE_SCHEMAS if defined?(origin_class::TABLE_SCHEMAS)
-      raise NotImplementedError
-    end
-
     def delayed(mthd, *args, **kwargs)
       Jobs::AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
     end
@@ -231,7 +282,7 @@ module InstDataShipper
     end
 
     def destinations
-      @destinations ||= (@raw_destinations.presence || batch_context[:destinations]).map.with_index do |dest, i|
+      @destinations ||= (@raw_destinations.presence || batch_context[:destinations] || []).map.with_index do |dest, i|
         dcls = InstDataShipper.resolve_destination(dest)
         dcls.new("#{InstDataShipper.redis_prefix}:dump#{tracker.id}:dest#{i}", dest, self)
       end

data/lib/inst_data_shipper/schema_builder.rb

@@ -2,21 +2,28 @@ module InstDataShipper
   # This class ends up fill two roles - Schema and Mapping.
   # It makes for a clean API, but it's a little less canonical since, (eg) the S3 destination doesn't need column type annotations.
   class SchemaBuilder
-    attr_reader :tables
+    attr_reader :schema
 
     def initialize
-      @tables = []
+      @schema = {
+        tables: [],
+      }
     end
 
     def self.build(&block)
       builder = new
       builder.instance_exec(&block)
-      builder.tables
+      builder.schema
+    end
+
+    def version(version)
+      @schema[:version] = version
     end
 
-    def extend_table_builder(&block)
+    def extend_table_builder(modul = nil, &block)
       @table_builder_class ||= Class.new(TableSchemaBuilder)
-      @table_builder_class.class_eval(&block)
+      @table_builder_class.class_eval(&block) if block.present?
+      @table_builder_class.extend(modul) if modul.present?
     end
 
     def table(model_or_name, description = nil, model: nil, query: nil, **extra, &block)
@@ -36,6 +43,7 @@ module InstDataShipper
 
         tdef[:query] = model_or_name
         tdef[:model] = model_or_name.model
+        tdef[:warehouse_name] = model_or_name.model.table_name
       elsif model_or_name.is_a?(Class) && model_or_name < ActiveRecord::Base
         tdef[:warehouse_name] = model_or_name.table_name
         tdef[:model] = model_or_name
@@ -43,9 +51,9 @@ module InstDataShipper
         tdef[:warehouse_name] = model_or_name
       end
 
-      @table_builder_class.build(tdef, &block)
+      (@table_builder_class || TableSchemaBuilder).build(tdef, &block)
 
-      @tables << tdef
+      @schema[:tables] << tdef
 
       tdef
     end
@@ -61,14 +69,18 @@ module InstDataShipper
       def self.build(tdef, &block)
         builder = new(tdef)
         builder.instance_exec(&block)
-        builder.columns
+        builder.options
+      end
+
+      def annotate(key, value)
+        options[key] = value
       end
 
-      # def annotate(key, value)
-      #   options[key] = value
-      # end
+      def version(version)
+        options[:version] = version
+      end
 
-      def incremental(scope="updated_at", **kwargs)
+      def incremental(scope=nil, **kwargs)
         if (extras = kwargs.keys - %i[on if]).present?
           raise ArgumentError, "Unsuppored options: #{extras.inspect}"
         end
@@ -80,6 +92,17 @@ module InstDataShipper
         }
       end
 
+      def source(source, override_model=nil, **kwargs)
+        raise "Source already set" if options[:sourcer].present?
+
+        if source.is_a?(Symbol)
+          mthd = :"import_#{source}"
+          options = self.options
+          source = ->(table_def) { send(mthd, override_model || options[:model] || options[:warehouse_name], **kwargs) }
+        end
+        options[:sourcer] = source
+      end
+
       def column(name, *args, refs: [], from: nil, **extra, &block)
         from ||= name.to_s
 

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.1.0.beta2".freeze
+  VERSION = "0.2.1".freeze
 end

data/lib/inst_data_shipper.rb

@@ -39,6 +39,7 @@ module InstDataShipper
 
     def logger
       return @logger if defined? @logger
+      # TODO Annotate logs with DumpBatch ID
       @logger = Logger.new(STDOUT)
       @logger.level = Logger::DEBUG
       @logger
@@ -49,7 +50,7 @@ module InstDataShipper
     end
 
     def redis_prefix
-      pfx = "hdd"
+      pfx = "ids"
       pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
       pfx
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta2
+  version: 0.2.1
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-08 00:00:00.000000000 Z
+date: 2024-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -395,11 +395,11 @@ files:
 - lib/inst_data_shipper/destinations/concerns/chunking.rb
 - lib/inst_data_shipper/destinations/hosted_data.rb
 - lib/inst_data_shipper/destinations/s3.rb
+- lib/inst_data_shipper/destinations/speccable.rb
 - lib/inst_data_shipper/dumper.rb
 - lib/inst_data_shipper/engine.rb
 - lib/inst_data_shipper/jobs/async_caller.rb
 - lib/inst_data_shipper/jobs/base.rb
-- lib/inst_data_shipper/jobs/basic_dump_job.rb
 - lib/inst_data_shipper/record.rb
 - lib/inst_data_shipper/schema_builder.rb
 - lib/inst_data_shipper/version.rb
@@ -436,9 +436,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.6
 signing_key: 

data/lib/inst_data_shipper/jobs/basic_dump_job.rb

@@ -1,15 +0,0 @@
-module InstDataShipper
-  module Jobs
-    class BasicDumpJob < InstDataShipper::Jobs::Base
-      sidekiq_options retry: 3 if defined?(sidekiq_options)
-
-      def perform(endpoints)
-
-      end
-
-      protected
-
-
-    end
-  end
-end