inst_data_shipper 0.2.6 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9374e4353fc38c211cd8d7565b271f9a1543c8e6165d1716cffa93f0704d974b
-  data.tar.gz: de10a934800be0a8560fd241341da120e7b453cff3d7806664e7e8260c0ef33d
+  metadata.gz: 75cf640fef52a5846408607d92b4cbeaedefec03e733542508c17ea245b7d3cf
+  data.tar.gz: 07bae82a7237af5d1720503a45f35208711c299c05485adc02f045de65462e49
 SHA512:
-  metadata.gz: 4b0599be4475101ea1ad5e5c5db90134628cdbf1e60e3b440a1f9f4fa90023da5da31c39d868f3fb431a9037f4a25159f0e283e253888da9432153db1b838edf
-  data.tar.gz: a8be277a20e03c4363f3a527dd0aa382252e330d73bdcb02f58a9c032f69b5cf19279c316d5fd5712e0570f71da9306b38d103af431a04b6aa38d7915155819d
+  metadata.gz: 81cb6a9801d399f0952f74083506938f9e3fd68b966a1afa4c24cbd9df393659ef01c1c4ae5cbd3a526809c2c1214966b798a9e30727d059fdfc00730a6931a1
+  data.tar.gz: 39930b880f8acd83230fda54bf559c9deb7fc95a31039212b7ab92eb81cc69b270cc0ed6e28237f0449a6e314146082d4c1f3203cc176eacf0b8c75dc6d3407f

data/app/models/inst_data_shipper/dump_batch.rb

@@ -1,5 +1,5 @@
 module InstDataShipper
-  class DumpBatch < ApplicationRecord
+  class DumpBatch < ActiveRecord::Base
     ERROR_STATUS = "error".freeze
     SUCCESS_STATUS = "success".freeze
     ENQUEUED_STATUS = "enqueued".freeze

data/lib/inst_data_shipper/data_sources/base.rb

@@ -9,6 +9,25 @@ module InstDataShipper
         base.send(:include, Concern)
       end
 
+      def process_raw_data_enumerator(table_def, enumerator, dest)
+        enumerator.each_slice(1000) do |batch|
+          Array(table_def[:batch_preprocess]).each do |preprocess|
+            batch = instance_exec(batch, &preprocess).compact
+          end
+
+          batch.each do |m|
+            Array(table_def[:row_preprocess]).each do |preprocess|
+              m = instance_exec(m, &preprocess)
+              next if m.nil?
+            end
+
+            dest << table_def[:columns].map do |c|
+              instance_exec(m, &c[:block])
+            end
+          end
+        end
+      end
+
       module Concern
         extend ActiveSupport::Concern
       end

data/lib/inst_data_shipper/data_sources/canvas_reports.rb

@@ -101,7 +101,7 @@ module InstDataShipper
 
       def _process_canvas_report(report:, schema_name: nil)
         file_path = "#{working_dir}/temp_report"
-        IO.copy_stream(URI.parse(report['attachment']['url']).open, file_path)
+        canvas_sync_client.download_report(report['attachment']['url'], file_path)
 
         if report['attachment']['content-type'] == 'application/zip'
           unzipped_file_path = "#{file_path}_unzipped"
@@ -123,11 +123,8 @@ module InstDataShipper
             table_def = lookup_table_schema!(schema_name&.[](file_name), report[:report])
 
             inner_block = ->(file) {
-              CSV.foreach(file_path, headers: true) do |m|
-                file << table_def[:columns].map do |c|
-                  instance_exec(m, &c[:block])
-                end
-              end
+              enumerator = CSV.foreach(file_path, headers: true)
+              process_raw_data_enumerator(table_def, enumerator, file)
             }
 
             upload_data(table_def, extra: report['id'], &inner_block)

data/lib/inst_data_shipper/data_sources/local_tables.rb

@@ -24,11 +24,7 @@ module InstDataShipper
             )
           end
 
-          query.find_each do |m|
-            file << table_def[:columns].map do |c|
-              instance_exec(m, &c[:block])
-            end
-          end
+          process_raw_data_enumerator(table_def, query.find_each, file)
         }
 
         upload_data(table_def, &inner_block)

data/lib/inst_data_shipper/dumper.rb

@@ -172,9 +172,12 @@ module InstDataShipper
 
       table_def = lookup_table_schema!(table_def) if table_def.is_a?(String)
 
+      # Return false if the table was added to the force list
+      #   (Captures: return false if table's schema changes)
       return false if batch_context[:force_full_tables]&.include?(table_def[:warehouse_name])
 
-      # TODO Return false if table's schema changes
+      return false unless incremental_since
+
       if (inc = table_def[:incremental]).present?
         differ = inc[:if]
         return !!incremental_since if differ.nil?

data/lib/inst_data_shipper/schema_builder.rb

@@ -82,7 +82,7 @@ module InstDataShipper
 
       def incremental(scope=nil, **kwargs)
         if (extras = kwargs.keys - %i[on if]).present?
-          raise ArgumentError, "Unsuppored options: #{extras.inspect}"
+          raise ArgumentError, "Unsupported options: #{extras.inspect}"
         end
 
         options[:incremental] = {
@@ -105,6 +105,16 @@ module InstDataShipper
         self.options[:sourcer] = source
       end
 
+      def batch_preprocess(&block)
+        options[:batch_preprocess] ||= []
+        options[:batch_preprocess] << block
+      end
+
+      def row_preprocess(&block)
+        options[:row_preprocess] ||= []
+        options[:row_preprocess] << block
+      end
+
       def column(name, *args, refs: [], from: nil, **extra, &block)
         from ||= name.to_s
 

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.2.6".freeze
+  VERSION = "0.2.8".freeze
 end

data/spec/internal/config/database.yml

@@ -0,0 +1,5 @@
+test:
+  database: ids-test
+  adapter: postgresql
+  encoding: unicode
+  pool: 15

data/spec/internal/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.routes.draw do
+  # Add your own routes here, or remove this file if you don't have need for it.
+end

data/spec/internal/config/storage.yml

@@ -0,0 +1,3 @@
+test:
+  service: Disk
+  root: /Users/eknapp/code/inst_data_shipper/tmp/storage

data/spec/internal/db/schema.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+ActiveRecord::Schema.define do
+  # Set up any tables you need to exist for your test suite that don't belong
+  # in migrations.
+end

data/spec/spec_helper.rb

@@ -1,9 +1,14 @@
 ENV["RAILS_ENV"] ||= 'test'
 ENV['REDIS_URL'] ||= 'redis://localhost:6379/9'
 
-require "inst_data_shipper"
-require File.expand_path("../dummy/config/environment.rb", __FILE__)
-# Rails::Generators.invoke("canvas_sync:install", ["--models", "all"], destination_root: Rails.root)
+require "bundler/setup"
+Bundler.require :default, :development
+
+Combustion::Application.configure do
+  config.enable_reloading = true # Required for with_model
+end
+Combustion.initialize! :active_record, :active_job
+
 require "bundler/setup"
 require 'rspec/rails'
 require 'spec_helper'
@@ -16,6 +21,8 @@ require 'pry'
 require 'pry-nav'
 require 'with_model'
 
+require "inst_data_shipper"
+
 require 'sidekiq/testing'
 Sidekiq::Testing.fake!
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.8
 platform: ruby
 authors:
 - Instructure CustomDev
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-04 00:00:00.000000000 Z
+date: 2025-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -296,14 +296,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.8
+        version: 1.5.39
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.8
+        version: 1.5.39
 - !ruby/object:Gem::Dependency
   name: canvas_sync
   requirement: !ruby/object:Gem::Requirement
@@ -332,7 +332,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - pseng@instructure.com
 executables: []
@@ -361,26 +361,14 @@ files:
 - lib/inst_data_shipper/record.rb
 - lib/inst_data_shipper/schema_builder.rb
 - lib/inst_data_shipper/version.rb
-- spec/dummy/README.rdoc
-- spec/dummy/Rakefile
-- spec/dummy/app/models/application_record.rb
-- spec/dummy/bin/rails
-- spec/dummy/config.ru
-- spec/dummy/config/application.rb
-- spec/dummy/config/boot.rb
-- spec/dummy/config/database.yml
-- spec/dummy/config/environment.rb
-- spec/dummy/config/environments/development.rb
-- spec/dummy/config/environments/test.rb
-- spec/dummy/config/initializers/assets.rb
-- spec/dummy/config/initializers/session_store.rb
-- spec/dummy/config/initializers/wrap_parameters.rb
-- spec/dummy/config/routes.rb
-- spec/dummy/config/secrets.yml
-- spec/dummy/db/schema.rb
-- spec/dummy/tmp/local_secret.txt
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
+- spec/internal/app/assets/config/manifest.js
+- spec/internal/config/database.yml
+- spec/internal/config/routes.rb
+- spec/internal/config/storage.yml
+- spec/internal/db/schema.rb
+- spec/internal/public/favicon.ico
 - spec/spec_helper.rb
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
@@ -388,7 +376,7 @@ files:
 homepage: https://instructure.com
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -404,30 +392,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.5.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Gem for uploading app data to Instructure CustomDev Hosted Data tooling
 test_files:
-- spec/dummy/README.rdoc
-- spec/dummy/Rakefile
-- spec/dummy/app/models/application_record.rb
-- spec/dummy/bin/rails
-- spec/dummy/config/application.rb
-- spec/dummy/config/boot.rb
-- spec/dummy/config/database.yml
-- spec/dummy/config/environment.rb
-- spec/dummy/config/environments/development.rb
-- spec/dummy/config/environments/test.rb
-- spec/dummy/config/initializers/assets.rb
-- spec/dummy/config/initializers/session_store.rb
-- spec/dummy/config/initializers/wrap_parameters.rb
-- spec/dummy/config/routes.rb
-- spec/dummy/config/secrets.yml
-- spec/dummy/config.ru
-- spec/dummy/db/schema.rb
-- spec/dummy/tmp/local_secret.txt
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
+- spec/internal/app/assets/config/manifest.js
+- spec/internal/config/database.yml
+- spec/internal/config/routes.rb
+- spec/internal/config/storage.yml
+- spec/internal/db/schema.rb
+- spec/internal/public/favicon.ico
 - spec/spec_helper.rb
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv

data/spec/dummy/README.rdoc

@@ -1 +0,0 @@
-Dummy Rails App for gem testing

data/spec/dummy/Rakefile

@@ -1,6 +0,0 @@
-# Add your own tasks in files placed in lib/tasks ending in .rake,
-# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
-
-require File.expand_path('../config/application', __FILE__)
-
-Rails.application.load_tasks

data/spec/dummy/app/models/application_record.rb

@@ -1,3 +0,0 @@
-class ApplicationRecord < ActiveRecord::Base
-  self.abstract_class = true
-end

data/spec/dummy/bin/rails

@@ -1,4 +0,0 @@
-#!/usr/bin/env ruby
-APP_PATH = File.expand_path('../../config/application', __FILE__)
-require_relative '../config/boot'
-require 'rails/commands'

data/spec/dummy/config/application.rb

@@ -1,37 +0,0 @@
-require File.expand_path('../boot', __FILE__)
-
-require 'rails'
-require 'active_record/railtie'
-# require 'active_storage/engine'
-require 'action_controller/railtie'
-require 'action_view/railtie'
-require 'action_mailer/railtie'
-require 'active_job/railtie'
-# require 'action_cable/engine'
-# require 'action_mailbox/engine'
-# require 'action_text/engine'
-require 'rails/test_unit/railtie'
-# require 'sprockets/railtie'
-
-Bundler.require(*Rails.groups)
-require "inst_data_shipper"
-
-module Dummy
-  class Application < Rails::Application
-    # Settings in config/environments/* take precedence over those specified here.
-    # Application configuration should go into files in config/initializers
-    # -- all .rb files in that directory are automatically loaded.
-
-    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
-    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
-    # config.time_zone = 'Central Time (US & Canada)'
-
-    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
-    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
-    # config.i18n.default_locale = :de
-
-    # Do not swallow errors in after_commit/after_rollback callbacks.
-    # config.active_record.raise_in_transactional_callbacks = true
-  end
-end
-

data/spec/dummy/config/boot.rb

@@ -1,5 +0,0 @@
-# Set up gems listed in the Gemfile.
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
-
-require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
-$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/spec/dummy/config/database.yml

@@ -1,25 +0,0 @@
-
-default: &default
-  adapter: postgresql
-  encoding: unicode
-  # For details on connection pooling, see Rails configuration guide
-  # http://guides.rubyonrails.org/configuring.html#database-pooling
-  pool: <%= ENV.fetch("DB_POOL_SIZE", nil) || (ENV.fetch("RAILS_MAX_THREADS") { 5 }.to_i + 10) %>
-  username: <%= ENV.fetch("DB_USERNAME", "") %>
-  password: <%= ENV.fetch("DB_PASSWORD", "") %>
-  host: <%= ENV.fetch("DB_ADDRESS", "localhost") %>
-
-development:
-  <<: *default
-  database: inst_data_shipper_development
-
-test:
-  <<: *default
-  database: inst_data_shipper_test
-
-production:
-  <<: *default
-  host: <%= ENV.fetch('DB_ADDRESS', 'localhost') %>
-  database: inst_data_shipper_production
-  username: <%= ENV.fetch("DB_USERNAME", "inst_data_shipper_specs_postgres_user") %>
-  password: <%= ENV.fetch("DB_PASSWORD", 'inst_data_shipper_specs_postgres_password') %>

data/spec/dummy/config/environment.rb

@@ -1,5 +0,0 @@
-# Load the Rails application.
-require File.expand_path('../application', __FILE__)
-
-# Initialize the Rails application.
-Rails.application.initialize!

data/spec/dummy/config/environments/development.rb

@@ -1,41 +0,0 @@
-Rails.application.configure do
-  # Settings specified here will take precedence over those in config/application.rb.
-
-  # In the development environment your application's code is reloaded on
-  # every request. This slows down response time but is perfect for development
-  # since you don't have to restart the web server when you make code changes.
-  config.cache_classes = false
-
-  # Do not eager load code on boot.
-  config.eager_load = false
-
-  # Show full error reports and disable caching.
-  config.consider_all_requests_local       = true
-  config.action_controller.perform_caching = false
-
-  # Don't care if the mailer can't send.
-  config.action_mailer.raise_delivery_errors = false
-
-  # Print deprecation notices to the Rails logger.
-  config.active_support.deprecation = :log
-
-  # Raise an error on page load if there are pending migrations.
-  config.active_record.migration_error = :page_load
-
-  # Debug mode disables concatenation and preprocessing of assets.
-  # This option may cause significant delays in view rendering with a large
-  # number of complex assets.
-  # config.assets.debug = true
-
-  # Asset digests allow you to set far-future HTTP expiration dates on all assets,
-  # yet still be able to expire them through the digest params.
-  # config.assets.digest = true
-
-  # Adds additional error checking when serving assets at runtime.
-  # Checks for improperly declared sprockets dependencies.
-  # Raises helpful error messages.
-  # config.assets.raise_runtime_errors = true
-
-  # Raises error for missing translations
-  # config.action_view.raise_on_missing_translations = true
-end

data/spec/dummy/config/environments/test.rb

@@ -1,44 +0,0 @@
-Rails.application.configure do
-  # Settings specified here will take precedence over those in config/application.rb.
-
-  # The test environment is used exclusively to run your application's
-  # test suite. You never need to work with it otherwise. Remember that
-  # your test database is "scratch space" for the test suite and is wiped
-  # and recreated between test runs. Don't rely on the data there!
-  config.cache_classes = true
-
-  # Do not eager load code on boot. This avoids loading your whole application
-  # just for the purpose of running a single test. If you are using a tool that
-  # preloads Rails for running tests, you may have to set it to true.
-  config.eager_load = false
-
-  # Configure static file server for tests with Cache-Control for performance.
-  config.serve_static_files   = true
-  config.static_cache_control = 'public, max-age=3600'
-
-  # Show full error reports and disable caching.
-  config.consider_all_requests_local       = true
-  config.action_controller.perform_caching = false
-
-  # Raise exceptions instead of rendering exception templates.
-  config.action_dispatch.show_exceptions = false
-
-  # Disable request forgery protection in test environment.
-  config.action_controller.allow_forgery_protection = false
-
-  # Tell Action Mailer not to deliver emails to the real world.
-  # The :test delivery method accumulates sent emails in the
-  # ActionMailer::Base.deliveries array.
-  config.action_mailer.delivery_method = :test
-
-  # Randomize the order test cases are executed.
-  config.active_support.test_order = :random
-
-  # Print deprecation notices to the stderr.
-  config.active_support.deprecation = :stderr
-
-  config.active_job.queue_adapter = :test
-
-  # Raises error for missing translations
-  # config.action_view.raise_on_missing_translations = true
-end

data/spec/dummy/config/initializers/assets.rb

@@ -1,11 +0,0 @@
-# Be sure to restart your server when you modify this file.
-
-# Version of your assets, change this if you want to expire all your assets.
-# Rails.application.config.assets.version = '1.0'
-
-# Add additional assets to the asset load path
-# Rails.application.config.assets.paths << Emoji.images_path
-
-# Precompile additional assets.
-# application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
-# Rails.application.config.assets.precompile += %w( search.js )

data/spec/dummy/config/initializers/session_store.rb

@@ -1,3 +0,0 @@
-# Be sure to restart your server when you modify this file.
-
-Rails.application.config.session_store :cookie_store, key: '_dummy_session'

data/spec/dummy/config/initializers/wrap_parameters.rb

@@ -1,14 +0,0 @@
-# Be sure to restart your server when you modify this file.
-
-# This file contains settings for ActionController::ParamsWrapper which
-# is enabled by default.
-
-# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
-ActiveSupport.on_load(:action_controller) do
-  wrap_parameters format: [:json] if respond_to?(:wrap_parameters)
-end
-
-# To enable root element in JSON for ActiveRecord objects.
-# ActiveSupport.on_load(:active_record) do
-#  self.include_root_in_json = true
-# end

data/spec/dummy/config/routes.rb

@@ -1,2 +0,0 @@
-Rails.application.routes.draw do
-end

data/spec/dummy/config/secrets.yml

@@ -1,22 +0,0 @@
-# Be sure to restart your server when you modify this file.
-
-# Your secret key is used for verifying the integrity of signed cookies.
-# If you change this key, all old signed cookies will become invalid!
-
-# Make sure the secret is at least 30 characters and all random,
-# no regular words or you'll be exposed to dictionary attacks.
-# You can use `rake secret` to generate a secure secret key.
-
-# Make sure the secrets in this file are kept private
-# if you're sharing your code publicly.
-
-development:
-  secret_key_base: 4fc03e27b5bfc390d7beaf99e739c1a9ad4264d58cd2a1db40fbb83b6ed320e1fa77fb4e0dd7c24816811170a59d65b623bc887b4ca1e0bd0c57348e9e304f73
-
-test:
-  secret_key_base: 7700238de042256ef63c58a57c6bf431b6ebca79f95f66ccd04b5ad8b89761db15a9b89a85b08ace8a69ae3867462cb9514eb21d0e591d8f3881de79d7b72675
-
-# Do not keep production secrets in the repository,
-# instead read values from the environment.
-production:
-  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>

data/spec/dummy/config.ru

@@ -1,4 +0,0 @@
-# This file is used by Rack-based servers to start the application.
-
-require ::File.expand_path('../config/environment', __FILE__)
-run Rails.application

data/spec/dummy/db/schema.rb

@@ -1,58 +0,0 @@
-# This file is auto-generated from the current state of the database. Instead
-# of editing this file, please use the migrations feature of Active Record to
-# incrementally modify your database, and then regenerate this schema definition.
-#
-# Note that this schema.rb definition is the authoritative source for your
-# database schema. If you need to create the application database on another
-# system, you should be using db:schema:load, not running all the migrations
-# from scratch. The latter is a flawed and unsustainable approach (the more migrations
-# you'll amass, the slower it'll run and the greater likelihood for issues).
-#
-# It's strongly recommended that you check this file into your version control system.
-
-ActiveRecord::Schema.define(version: 2024_03_01_090836) do
-
-  # These are extensions that must be enabled in order to support this database
-  enable_extension "plpgsql"
-
-  create_table "canvas_sync_job_logs", id: :serial, force: :cascade do |t|
-    t.datetime "started_at"
-    t.datetime "completed_at"
-    t.string "exception"
-    t.text "backtrace"
-    t.string "job_class"
-    t.string "status"
-    t.text "metadata"
-    t.text "job_arguments"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string "job_id"
-    t.integer "fork_count"
-    t.index ["job_id"], name: "index_canvas_sync_job_logs_on_job_id"
-  end
-
-  create_table "canvas_sync_sync_batches", id: :serial, force: :cascade do |t|
-    t.datetime "started_at"
-    t.datetime "completed_at"
-    t.string "status"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.boolean "full_sync", default: false
-    t.string "batch_genre"
-    t.string "batch_bid"
-  end
-
-  create_table "inst_data_shipper_dump_batches", id: :serial, force: :cascade do |t|
-    t.datetime "started_at"
-    t.datetime "completed_at"
-    t.string "status"
-    t.string "job_class"
-    t.string "genre"
-    t.string "batch_id"
-    t.string "exception"
-    t.text "backtrace"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-
-end

data/spec/dummy/tmp/local_secret.txt

@@ -1 +0,0 @@
-6ddf19e3f9f936190c3c79bc546382849bcf6efb4205cbe13797b683ec9a3dbc02cb87e348121c3502809d9475683c26b12c92d973d571953ebb50b0c4cc3641