inst_data_shipper 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7909aa44e9dabd1d43d58a5a3c2c081891104d64336294dce287c06804804df
-  data.tar.gz: 5da874689ac1de3e016a7feefce5866b211e6f7595021b565564f796685ed104
+  metadata.gz: 4c84d0f1543bfcb42bce30e69ff12cc683b6f6bf51e9275f7529dd0b070bbb79
+  data.tar.gz: 22b5128af3f2d03b14565c656abca685c6e7196083576d1fd20a42a834690cba
 SHA512:
-  metadata.gz: cd81e6c26e2416ce1a32de588e04f560496cfb7cfdac3f4c837828a1c65798bec405d98197032b0d8935a1ba2b24a291aa25f1b73a469ac7a9c6ef8d2286103f
-  data.tar.gz: 66c5ccfd82128e8c5dc39c7c937ee7f4f9412743b7202e221e53c575d4b0e572f0b014b4f41ae5924b1d2d119a05cd5de2acbae4eb81022df844a1fea181faec
+  metadata.gz: 14e5eb97d9c5dbe5049eb94cd3690a2ab59df7bf6e5e0c62b21e8893f4cb75ca294cce6d28a4b0f7dccd1c53d34d5c7a34be7b2f4699f1b320f74df14d50f672
+  data.tar.gz: e7d22728140a7fc331abb5f608da98d44013d49027bf932ef183a4f75babe797632aca83955cb77a205d6004d29a7377dde87aaba0c245a3564541b7e1f55cbe

data/README.md

@@ -55,6 +55,13 @@ class HostedDataPushJob < ApplicationJob
       #  `if:` may be a Proc or a Symbol (of a method on the Dumper)
       incremental "updated_at", on: [:id], if: ->() {}
 
+      # Schema's may declaratively define the data source.
+      # This can be used for basic schemas where there's a 1:1 mapping between source table and destination table, and there is no conditional logic that needs to be performed.
+      # In order to apply these statements, your Dumper must call `auto_enqueue_from_schema`.
+      source :local_table
+      # A Proc can also be passed. The below is equivalent to the above
+      source ->(table_def) { import_local_table(table_def[:model] || table_def[:warehouse_name]) }
+
       column :name_in_destinations, :maybe_optional_sql_type, "Optional description of column"
 
       # The type may usually be omitted if the `table()` is passed a Model class, but strings are an exception to this
@@ -93,6 +100,10 @@ class HostedDataPushJob < ApplicationJob
     # If the report_name/Model don't directly match the Schema, a schema_name: parameter may be passed:
     import_local_table(SomeModel, schema_name: "my_table")
     import_canvas_report_by_terms("some_report", terms: Term.all.pluck(:canvas_id), schema_name: "my_table")
+
+    # Iterate through the Tables defined in the Schema and apply any defined `source` statements.
+    # This is the default behavior if `define()` is called w/o a block.
+    auto_enqueue_from_schema
   end
 
   def perform
@@ -118,6 +129,8 @@ class HostedDataPushJob < ApplicationJob
     def enqueue_tasks
       import_local_table(ALocalModel)
       import_canvas_report_by_terms("proserv_student_submissions_csv", terms: Term.all.pluck(:canvas_id))
+
+      # auto_enqueue_from_schema
     end
 
     def table_schemas

data/lib/inst_data_shipper/data_sources/base.rb

@@ -1,7 +1,31 @@
 module InstDataShipper
   module DataSources
     module Base
-      extend ActiveSupport::Concern
+      # This could be a Concern, but we don't want Concern inheritance logic kicking in if it is included into another Concern.
+
+      def self.included(base)
+        base.extend ActiveSupport::Concern unless self.is_a?(Class) || base < ActiveSupport::Concern
+        base.extend ModuleHelperMethods
+        base.send(:include, Concern)
+      end
+
+      module Concern
+        extend ActiveSupport::Concern
+      end
+
+      module ModuleHelperMethods
+        def delayed(mthd_sym)
+          mthd = instance_method(mthd_sym)
+          pmthd_sym = :"_delayed_#{mthd_sym}"
+
+          alias_method pmthd_sym, mthd_sym
+          private pmthd_sym
+
+          define_method(mthd_sym) do |*args, **kwargs|
+            delayed(pmthd_sym, *args, **kwargs)
+          end
+        end
+      end
     end
   end
 end

data/lib/inst_data_shipper/data_sources/canvas_reports.rb

@@ -2,7 +2,7 @@ module InstDataShipper
   module DataSources
     # This module contains the logic for processing Canvas reports
     module CanvasReports
-      extend ActiveSupport::Concern
+      include Base
 
       included do
         hook :initialize_dump_batch do |context|

data/lib/inst_data_shipper/data_sources/local_tables.rb

@@ -2,17 +2,11 @@ module InstDataShipper
   module DataSources
     # This module contains the logic for processing local AR tables
     module LocalTables
-      extend ActiveSupport::Concern
+      include Base
 
       public
 
-      def import_local_table(*args, **kwargs)
-        delayed(:_import_local_table, *args, **kwargs)
-      end
-
-      private
-
-      def _import_local_table(model, schema_name: nil)
+      delayed def import_local_table(model, schema_name: nil)
         model = model.safe_constantize if model.is_a?(String)
 
         table_def = lookup_table_schema!(schema_name, { model: model })
@@ -40,6 +34,8 @@ module InstDataShipper
         upload_data(table_def, &inner_block)
       end
 
+      private
+
       def _resolve_model_query(relation, query, string: nil, default: nil)
         return relation if query == false
         query = default if query.nil?

data/lib/inst_data_shipper/destinations/concerns/chunking.rb

@@ -7,7 +7,7 @@ module InstDataShipper
         DEFAULT_CHUNK_SIZE = 100_000
 
         def chunk_data(generator, chunk_size: nil, **kwargs)
-          chunk_size ||= config.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
+          chunk_size ||= config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
           slice = 1
 
           btchr = CanvasSync::BatchProcessor.new(of: chunk_size) do |batch|
@@ -24,7 +24,7 @@ module InstDataShipper
 
         def group_key
           super.tap do |k|
-            k[:chunk_size] = config.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
+            k[:chunk_size] = config&.dig(:params, :chunk_size) || DEFAULT_CHUNK_SIZE
           end
         end
 

data/lib/inst_data_shipper/destinations/speccable.rb

@@ -0,0 +1,17 @@
+module InstDataShipper
+  module Destinations
+    class Speccable < Base
+      include Concerns::Chunking
+
+      def chunk_data(generator, table:, extra: nil)
+        super(generator) do |batch, idx|
+          yield batch
+        end
+      end
+
+      def upload_data_chunk(table_def, chunk); end
+
+      def parse_configuration(uri); end
+    end
+  end
+end

data/lib/inst_data_shipper/dumper.rb

@@ -18,6 +18,12 @@ module InstDataShipper
       Class.new(self) do
         include(*include)
 
+        if blk.nil? && schema[:tables].any? { |t| t[:sourcer].present? }
+          blk = -> { auto_enqueue_from_schema }
+        elsif blk.nil?
+          raise ArgumentError, "Must provide a block or a schema with source definitions"
+        end
+
         define_method(:enqueue_tasks, &blk)
         define_method(:schema) { schema }
       end
@@ -33,6 +39,16 @@ module InstDataShipper
       clazz.new(executor: executor)
     end
 
+    if defined?(Rails) && Rails.env.test?
+      def for_specs!
+        @raw_destinations = ["speccable://nil"]
+        @executor = InstDataShipper::Jobs::AsyncCaller.new
+        @tracker = DumpBatch.new(job_class: self.class.to_s, genre: export_genre, status: 'in_progress')
+        define_singleton_method(:spec_destination) { destinations.first }
+        self
+      end
+    end
+
     public
 
     def begin_dump
@@ -176,6 +192,14 @@ module InstDataShipper
       raise NotImplementedError
     end
 
+    def auto_enqueue_from_schema
+      schema[:tables].each do |table_def|
+        src = table_def[:sourcer]
+        next unless src.present?
+        instance_exec(table_def, &src)
+      end
+    end
+
     def upload_data(table_def, extra: nil, &datagen)
       # Allow muxing, allowing a hook to prevent some files going to certain destinations
       dests = destinations_for_table(table_def)
@@ -258,7 +282,7 @@ module InstDataShipper
     end
 
     def destinations
-      @destinations ||= (@raw_destinations.presence || batch_context[:destinations]).map.with_index do |dest, i|
+      @destinations ||= (@raw_destinations.presence || batch_context[:destinations] || []).map.with_index do |dest, i|
         dcls = InstDataShipper.resolve_destination(dest)
         dcls.new("#{InstDataShipper.redis_prefix}:dump#{tracker.id}:dest#{i}", dest, self)
       end

data/lib/inst_data_shipper/schema_builder.rb

@@ -20,9 +20,10 @@ module InstDataShipper
       @schema[:version] = version
     end
 
-    def extend_table_builder(&block)
+    def extend_table_builder(modul = nil, &block)
       @table_builder_class ||= Class.new(TableSchemaBuilder)
-      @table_builder_class.class_eval(&block)
+      @table_builder_class.class_eval(&block) if block.present?
+      @table_builder_class.extend(modul) if modul.present?
     end
 
     def table(model_or_name, description = nil, model: nil, query: nil, **extra, &block)
@@ -42,6 +43,7 @@ module InstDataShipper
 
         tdef[:query] = model_or_name
         tdef[:model] = model_or_name.model
+        tdef[:warehouse_name] = model_or_name.model.table_name
       elsif model_or_name.is_a?(Class) && model_or_name < ActiveRecord::Base
         tdef[:warehouse_name] = model_or_name.table_name
         tdef[:model] = model_or_name
@@ -49,7 +51,7 @@ module InstDataShipper
         tdef[:warehouse_name] = model_or_name
       end
 
-      @table_builder_class.build(tdef, &block)
+      (@table_builder_class || TableSchemaBuilder).build(tdef, &block)
 
       @schema[:tables] << tdef
 
@@ -67,12 +69,12 @@ module InstDataShipper
       def self.build(tdef, &block)
         builder = new(tdef)
         builder.instance_exec(&block)
-        builder.columns
+        builder.options
       end
 
-      # def annotate(key, value)
-      #   options[key] = value
-      # end
+      def annotate(key, value)
+        options[key] = value
+      end
 
       def version(version)
         options[:version] = version
@@ -90,6 +92,17 @@ module InstDataShipper
         }
       end
 
+      def source(source, override_model=nil, **kwargs)
+        raise "Source already set" if options[:sourcer].present?
+
+        if source.is_a?(Symbol)
+          mthd = :"import_#{source}"
+          options = self.options
+          source = ->(table_def) { send(mthd, override_model || options[:model] || options[:warehouse_name], **kwargs) }
+        end
+        options[:sourcer] = source
+      end
+
       def column(name, *args, refs: [], from: nil, **extra, &block)
         from ||= name.to_s
 

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.2.0".freeze
+  VERSION = "0.2.1".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-11 00:00:00.000000000 Z
+date: 2024-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -395,6 +395,7 @@ files:
 - lib/inst_data_shipper/destinations/concerns/chunking.rb
 - lib/inst_data_shipper/destinations/hosted_data.rb
 - lib/inst_data_shipper/destinations/s3.rb
+- lib/inst_data_shipper/destinations/speccable.rb
 - lib/inst_data_shipper/dumper.rb
 - lib/inst_data_shipper/engine.rb
 - lib/inst_data_shipper/jobs/async_caller.rb