inst_data_shipper 0.1.0.beta1

data/lib/inst_data_shipper/dumper.rb

@@ -0,0 +1,159 @@
+module InstDataShipper
+  class Dumper
+    include Hooks
+
+    define_hook :initialize_dump_batch
+    define_hook :finalize_dump_batch
+
+    def self.perform_dump(destinations:)
+      raise "Must subclass Dumper to use perform_dump" if self == Dumper
+
+      dumper = new(destinations)
+      dumper.begin_dump
+
+      dumper.tracker
+    end
+
+    protected
+
+    attr_reader :executor
+
+    def initialize(destinations = nil, executor: nil)
+      @raw_destinations = destinations
+      @executor = executor
+    end
+
+    def enqueue_tasks
+      raise NotImplementedError
+    end
+
+    def begin_dump
+      raise "Dump already begun" unless @raw_destinations.present?
+
+      @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, status: 'in_progress')
+
+      destinations.each do |dest|
+        dest.initialize_dump()
+      end
+
+      context = {}
+      run_hook(:initialize_dump_batch, context)
+
+      Sidekiq::Batch.new.tap do |batch|
+        batch.description = "HD #{export_genre} Export #{tracker.id} Root"
+        batch.context = {
+          **context,
+          root_bid: batch.bid,
+          tracker_id: tracker.id,
+          origin_class: batch_context[:origin_class] || self.class.to_s,
+          destinations: @raw_destinations,
+        }
+        batch.on(:success, "#{self.class}#finalize_dump")
+        batch.on(:death, "#{self.class}#cleanup_fatal_error!")
+        batch.jobs do
+          enqueue_tasks
+        end
+      end
+
+      # TODO Catch errors in here and cleanup as needed
+    end
+
+    def upload_data(table_def, extra: nil, &datagen)
+      # Allow muxing, allowing a hook to prevent some files going to certain destinations
+      dests = destinations_for_table(table_def)
+
+      dest_groups = dests.group_by do |dest|
+        catch(:not_groupable) do
+          next dest.group_key
+        end
+        :not_groupable
+      end
+
+      not_groupable = dest_groups.delete(:not_groupable)
+
+      # If multiple destinations will produce the same data, only generate the chunk once...
+      dest_groups.each do |group_key, dests|
+        dests[0].chunk_data(datagen, table: table_def, extra: extra) do |chunk|
+          # ...and upload it to each
+          dests.each do |dest|
+            dest.upload_data_chunk(table_def, chunk)
+          end
+        end
+      end
+
+      not_groupable&.each do |dest|
+        dest.chunk_data(datagen, table: table_def, extra: extra) do |chunk|
+          dest.upload_data_chunk(table_def, chunk)
+        end
+      end
+
+      # TODO Consider how to handle errors in this method.
+      #   Retriable errors must not be allowed to bubble - if dest 1 succeeds and dest 2 fails, dest 1 must not be retried
+      #   Each destination should handle its own retries
+      #   If a destination errors-out, we should continue with the other destinations
+    end
+
+    def finalize_dump(_status, _opts)
+      run_hook(:finalize_dump_batch)
+
+      destination.each do |dest|
+        dest.finalize_dump
+      end
+
+      DumpBatch.find(batch_context[:tracker_id]).update(status: 'completed')
+    end
+
+    def cleanup_fatal_error!(*_args)
+      return unless batch.present?
+
+      run_hook(:finalize_dump_batch)
+
+      destination.each do |dest|
+        dest.cleanup_fatal_error
+      rescue StandardError # rubocop:disable Lint/SuppressedException
+      end
+
+      DumpBatch.find(batch_context[:tracker_id]).update(status: 'failed')
+
+      CanvasSync::JobBatches::Batch.delete_prematurely!(batch_context[:root_bid])
+    end
+
+    # Helper Methods
+
+    def table_schemas
+      return origin_class::TABLE_SCHEMAS if defined?(origin_class::TABLE_SCHEMAS)
+      raise NotImplementedError
+    end
+
+    def delayed(mthd, *args, **kwargs)
+      AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
+    end
+
+    def tracker
+      @tracker ||= batch_context[:tracker_id].present? ? DumpBatch.find(batch_context[:tracker_id]) : nil
+    end
+
+    def export_genre
+      self.class.to_s.gsub(/HD|ExportJob/, '')
+    end
+
+    def origin_class
+      batch_context[:origin_class]&.constantize || self.class
+    end
+
+    def working_dir
+      executor.working_dir
+    end
+
+    def destinations_for_table(table_def)
+      destinations
+    end
+
+    def destinations
+      @destinations ||= (@raw_destinations || batch_context[:destinations]).map.with_index do |dest, i|
+        dcls = InstDataShipper.resolve_destination(dest)
+        dcls.new("#{InstDataShipper.redis_prefix}:dump#{tracker.id}:dest#{i}", dest, self)
+      end
+    end
+  end
+end

data/lib/inst_data_shipper/engine.rb

@@ -0,0 +1,8 @@
+require "rails"
+
+module InstDataShipper
+  class Engine < ::Rails::Engine
+    isolate_namespace InstDataShipper
+
+  end
+end

data/lib/inst_data_shipper/jobs/async_caller.rb

@@ -0,0 +1,19 @@
+module InstDataShipper
+  module Jobs
+    class AsyncCaller < InstDataShipper::Jobs::Base
+      sidekiq_options retry: 6 if defined?(sidekiq_options)
+
+      def self.call_from_pool(pool, clazz, method, *args, **kwargs)
+        pool.add_job(
+          job: self,
+          args: [clazz.to_s, method.to_s, *args],
+          kwargs: kwargs,
+        )
+      end
+
+      def perform(clazz, method, *args, **kwargs)
+        clazz.constantize.new(executor: self).send(method.to_sym, *args, **kwargs)
+      end
+    end
+  end
+end

data/lib/inst_data_shipper/jobs/base.rb

@@ -0,0 +1,27 @@
+require "active_job"
+
+module InstDataShipper
+  module Jobs
+    class Base < ActiveJob::Base
+      attr_reader :start_time
+
+      before_perform do
+        @start_time = DateTime.now
+      end
+
+      around_perform :cleanup_working_dir
+
+      def working_dir
+        @working_dir ||= Dir.mktmpdir
+      end
+
+      private
+
+      def cleanup_working_dir
+        yield if block_given?
+      ensure
+        FileUtils.remove_entry @working_dir if @working_dir
+      end
+    end
+  end
+end

data/lib/inst_data_shipper/jobs/basic_dump_job.rb

@@ -0,0 +1,11 @@
+module InstDataShipper
+  module Jobs
+    class BasicDumpJob < InstDataShipper::Jobs::Base
+      sidekiq_options retry: 3 if defined?(sidekiq_options)
+
+      def perform(endpoints, dump_class = nil)
+        dumper.perform_dump(endpoints: endpoints)
+      end
+    end
+  end
+end

data/lib/inst_data_shipper/record.rb

@@ -0,0 +1,6 @@
+
+module InstDataShipper
+  module Record
+    extend ActiveSupport::Concern
+  end
+end

data/lib/inst_data_shipper/schema_builder.rb

@@ -0,0 +1,93 @@
+module InstDataShipper
+  class SchemaBuilder
+    attr_reader :tables
+
+    def initialize
+      @tables = []
+    end
+
+    def self.build(&block)
+      builder = new
+      builder.instance_exec(&block)
+      builder.tables
+    end
+
+    def table(model_or_name, description = nil, as: nil, includes: nil, incremental: false, &block)
+      as ||= model_or_name
+      as = as.table_name if as.respond_to?(:table_name)
+
+      tdef = {
+        description: description,
+        model: model_or_name.is_a?(String) ? nil : model_or_name,
+        warehouse_name: as.to_s,
+        incremental: incremental,
+        columns: [],
+        includes: includes,
+      }
+
+      TableSchemaBuilder.build(tdef, &block)
+
+      @tables << tdef
+
+      self
+    end
+
+    class TableSchemaBuilder
+      attr_reader :options, :columns
+
+      def initialize(table)
+        @options = table
+        @columns = table[:columns]
+      end
+
+      def self.build(tdef, &block)
+        builder = new(tdef)
+        builder.instance_exec(&block)
+        builder.columns
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def column(name, *args, **kwargs, &block)
+        name = { name => name } unless name.is_a?(Hash)
+        raise ArgumentError, 'Exactly one name must be provided' unless name.count == 1
+
+        cdef = {
+          local_name: name.keys[0].to_s,
+          warehouse_name: name.values[0].to_s,
+          transformer: block,
+        }
+
+        [:description, :type, :refs => :references].each do |k|
+          if k.is_a? Hash
+            k.each do |hk, hv|
+              cdef[hv] = kwargs.delete(hk) if kwargs.key?(hk)
+            end
+          elsif kwargs.key?(k)
+            cdef[k] = kwargs.delete(k)
+          end
+        end
+
+        cdef[:references] = Array(cdef[:references])
+
+        args[0..1].each do |a|
+          k = (a.is_a?(String) && :description) || (a.is_a?(Symbol) && :type) || nil
+          raise ArgumentError, 'Unsupported Argument' if k.nil?
+          raise ArgumentError, "Duplicate Argument for #{k}" if cdef.key?(k)
+
+          cdef[k] = a
+        end
+
+        if options[:model].is_a?(Class) && cdef[:local_name].to_s.ends_with?('_id')
+          rel_name = cdef[:local_name].to_s[0...-3]
+          refl = options[:model].reflections[rel_name]
+          cdef[:references] << "#{refl.klass}##{refl.options[:primary_key] || 'id'}" if refl.present? && !refl.polymorphic?
+        end
+
+        @columns << cdef
+
+        self
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    end
+  end
+end

data/lib/inst_data_shipper/version.rb

@@ -0,0 +1,3 @@
+module InstDataShipper
+  VERSION = "0.1.0.beta1".freeze
+end

data/lib/inst_data_shipper.rb

@@ -0,0 +1,71 @@
+require "canvas_sync"
+
+require "inst_data_shipper/version"
+require "inst_data_shipper/engine"
+require "inst_data_shipper/record"
+
+module InstDataShipper
+  class << self
+
+    def alias_destination(from, to)
+      @destination_aliases ||= {}
+      @destination_aliases[from.to_s] = to.to_s
+    end
+
+    def resolve_destination(destination)
+      return destination if destination.is_a?(Class)
+
+      if destination.is_a?(String) && destination.include?("://")
+        type, rest = destination.split("://", 2)
+        destination = @destination_aliases[type]
+      elsif destination.is_a?(Hash)
+        type = destination[:type] || destination["type"]
+        destination = @destination_aliases[type]
+      end
+
+      safe_constantize(destination)
+    end
+
+    def start_basic_dump(*args, **kwargs, &block)
+      BasicDumper.perform_dump(*args, **kwargs, &block)
+    end
+
+    def logger
+      return @logger if defined? @logger
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::DEBUG
+      @logger
+    end
+
+    def redis(*args, &blk)
+      CanvasSync::JobBatches::Batch.redis(*args, &blk)
+    end
+
+    def redis_prefix
+      pfx = "hdd"
+      pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
+      pfx
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/inst_data_shipper/concerns/**/*.rb"].each { |file| require file }
+
+require "inst_data_shipper/schema_builder"
+
+require "inst_data_shipper/jobs/base"
+Dir[File.dirname(__FILE__) + "/inst_data_shipper/jobs/**/*.rb"].each { |file| require file }
+
+require "inst_data_shipper/data_sources/base"
+Dir[File.dirname(__FILE__) + "/inst_data_shipper/data_sources/**/*.rb"].each { |file| require file }
+
+require "inst_data_shipper/destinations/base"
+Dir[File.dirname(__FILE__) + "/inst_data_shipper/destinations/concerns/*.rb"].each { |file| require file }
+Dir[File.dirname(__FILE__) + "/inst_data_shipper/destinations/*.rb"].each do |file|
+  require file
+  basename = File.basename(file, ".rb")
+  next if basename == "base"
+
+  InstDataShipper.alias_destination(basename.dasherize, "InstDataShipper::Destinations::#{basename.classify}")
+end
+

data/spec/dummy/README.rdoc

@@ -0,0 +1 @@
+Dummy Rails App for gem testing

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application', __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/spec/dummy/config/application.rb

@@ -0,0 +1,37 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'rails'
+require 'active_record/railtie'
+# require 'active_storage/engine'
+require 'action_controller/railtie'
+require 'action_view/railtie'
+require 'action_mailer/railtie'
+require 'active_job/railtie'
+# require 'action_cable/engine'
+# require 'action_mailbox/engine'
+# require 'action_text/engine'
+require 'rails/test_unit/railtie'
+# require 'sprockets/railtie'
+
+Bundler.require(*Rails.groups)
+require "inst_data_shipper"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+
+    # Do not swallow errors in after_commit/after_rollback callbacks.
+    # config.active_record.raise_in_transactional_callbacks = true
+  end
+end
+

data/spec/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/spec/dummy/config/database.yml

@@ -0,0 +1,25 @@
+
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  # For details on connection pooling, see Rails configuration guide
+  # http://guides.rubyonrails.org/configuring.html#database-pooling
+  pool: <%= ENV.fetch("DB_POOL_SIZE", nil) || (ENV.fetch("RAILS_MAX_THREADS") { 5 }.to_i + 10) %>
+  username: <%= ENV.fetch("DB_USERNAME", "") %>
+  password: <%= ENV.fetch("DB_PASSWORD", "") %>
+  host: <%= ENV.fetch("DB_ADDRESS", "localhost") %>
+
+development:
+  <<: *default
+  database: inst_data_shipper_development
+
+test:
+  <<: *default
+  database: inst_data_shipper_test
+
+production:
+  <<: *default
+  host: <%= ENV.fetch('DB_ADDRESS', 'localhost') %>
+  database: inst_data_shipper_production
+  username: <%= ENV.fetch("DB_USERNAME", "inst_data_shipper_specs_postgres_user") %>
+  password: <%= ENV.fetch("DB_PASSWORD", 'inst_data_shipper_specs_postgres_password') %>

data/spec/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require File.expand_path('../application', __FILE__)
+
+# Initialize the Rails application.
+Rails.application.initialize!

data/spec/dummy/config/environments/development.rb

@@ -0,0 +1,41 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  # config.assets.debug = true
+
+  # Asset digests allow you to set far-future HTTP expiration dates on all assets,
+  # yet still be able to expire them through the digest params.
+  # config.assets.digest = true
+
+  # Adds additional error checking when serving assets at runtime.
+  # Checks for improperly declared sprockets dependencies.
+  # Raises helpful error messages.
+  # config.assets.raise_runtime_errors = true
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+end

data/spec/dummy/config/environments/test.rb

@@ -0,0 +1,44 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # The test environment is used exclusively to run your application's
+  # test suite. You never need to work with it otherwise. Remember that
+  # your test database is "scratch space" for the test suite and is wiped
+  # and recreated between test runs. Don't rely on the data there!
+  config.cache_classes = true
+
+  # Do not eager load code on boot. This avoids loading your whole application
+  # just for the purpose of running a single test. If you are using a tool that
+  # preloads Rails for running tests, you may have to set it to true.
+  config.eager_load = false
+
+  # Configure static file server for tests with Cache-Control for performance.
+  config.serve_static_files   = true
+  config.static_cache_control = 'public, max-age=3600'
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Raise exceptions instead of rendering exception templates.
+  config.action_dispatch.show_exceptions = false
+
+  # Disable request forgery protection in test environment.
+  config.action_controller.allow_forgery_protection = false
+
+  # Tell Action Mailer not to deliver emails to the real world.
+  # The :test delivery method accumulates sent emails in the
+  # ActionMailer::Base.deliveries array.
+  config.action_mailer.delivery_method = :test
+
+  # Randomize the order test cases are executed.
+  config.active_support.test_order = :random
+
+  # Print deprecation notices to the stderr.
+  config.active_support.deprecation = :stderr
+
+  config.active_job.queue_adapter = :test
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+end

data/spec/dummy/config/initializers/assets.rb

@@ -0,0 +1,11 @@
+# Be sure to restart your server when you modify this file.
+
+# Version of your assets, change this if you want to expire all your assets.
+# Rails.application.config.assets.version = '1.0'
+
+# Add additional assets to the asset load path
+# Rails.application.config.assets.paths << Emoji.images_path
+
+# Precompile additional assets.
+# application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
+# Rails.application.config.assets.precompile += %w( search.js )

data/spec/dummy/config/initializers/session_store.rb

@@ -0,0 +1,3 @@
+# Be sure to restart your server when you modify this file.
+
+Rails.application.config.session_store :cookie_store, key: '_dummy_session'

data/spec/dummy/config/initializers/wrap_parameters.rb

@@ -0,0 +1,14 @@
+# Be sure to restart your server when you modify this file.
+
+# This file contains settings for ActionController::ParamsWrapper which
+# is enabled by default.
+
+# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
+ActiveSupport.on_load(:action_controller) do
+  wrap_parameters format: [:json] if respond_to?(:wrap_parameters)
+end
+
+# To enable root element in JSON for ActiveRecord objects.
+# ActiveSupport.on_load(:active_record) do
+#  self.include_root_in_json = true
+# end

data/spec/dummy/config/routes.rb

@@ -0,0 +1,2 @@
+Rails.application.routes.draw do
+end

data/spec/dummy/config/secrets.yml

@@ -0,0 +1,22 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key is used for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+# You can use `rake secret` to generate a secure secret key.
+
+# Make sure the secrets in this file are kept private
+# if you're sharing your code publicly.
+
+development:
+  secret_key_base: 4fc03e27b5bfc390d7beaf99e739c1a9ad4264d58cd2a1db40fbb83b6ed320e1fa77fb4e0dd7c24816811170a59d65b623bc887b4ca1e0bd0c57348e9e304f73
+
+test:
+  secret_key_base: 7700238de042256ef63c58a57c6bf431b6ebca79f95f66ccd04b5ad8b89761db15a9b89a85b08ace8a69ae3867462cb9514eb21d0e591d8f3881de79d7b72675
+
+# Do not keep production secrets in the repository,
+# instead read values from the environment.
+production:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>

data/spec/dummy/config.ru

@@ -0,0 +1,4 @@
+# This file is used by Rack-based servers to start the application.
+
+require ::File.expand_path('../config/environment', __FILE__)
+run Rails.application