micdrop 0.1.0

data/examples/json_files_to_sql.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "micdrop"
+require "sequel"
+require "micdrop/ext/sequel"
+
+DB = Sequel.sqlite "test.db"
+
+# Create the destination data structure.
+# Obviously in a real import script, these would probably already exist.
+
+DB.create_table :people do
+  primary_key :id
+  String :f_name
+  String :l_name
+  String :addr1
+  String :addr2
+  String :city
+  String :state
+  String :zip
+  String :_tmp_id # Add a temporary column for storing the old system IDs
+end
+
+# Now start the migration
+
+# Our source will iterate over all the JSON files in the given directory
+source = Micdrop::FilesSource.new(__dir__, glob: "data/json/*.json")
+sink = Micdrop::Ext::Sequel::InsertSink.new DB[:people]
+
+Micdrop.migrate source, sink do
+  # The files source exposes the basename and content as takeable items
+  take :basename, put: :_tmp_id
+  take(:content).parse_json do
+    # parse_json accepts a block that will enter the sub-record
+    take "residency" do
+      regex(/(?<street>.+?) ?(?<unit>(?:Apt\.?|Suite|Unit|#) [#0-9a-zA-Z-]+)?\n(?<city>.+?), (?<state>[A-Z]{2}) (?<zip>\d{5}(?:-\d{4})?)/) do
+        # regex also enters a sub-record
+        take :street, put: :addr1
+        take :unit, put: :addr2
+        take :city, put: :city
+        take :state, put: :state
+        take :zip, put: :zip
+      end
+    end
+    take "name" do
+      split " " do
+        # split can enter a sub-record as well
+        take 0, put: :f_name
+        take 1, put: :l_name
+      end
+    end
+  end
+end

data/lib/micdrop/errors.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Micdrop
+  ##
+  # An error on the sink side of the migration
+  class SinkError < StandardError
+  end
+
+  ##
+  # An error on the source side of the migration
+  class SourceError < StandardError
+  end
+
+  ##
+  # An error with the current data value that prevents conversion operations from working
+  class ValueError < StandardError
+  end
+
+  ##
+  # An error with the data pipeline iteself
+  class PipelineError < StandardError
+  end
+end

data/lib/micdrop/ext/sequel.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "sequel"
+
+module Micdrop
+  module Ext
+    module Sequel
+      ##
+      # A sink which will exclusively insert new items into the database
+      class InsertSink
+        def initialize(dataset)
+          @dataset = dataset
+        end
+
+        def <<(collector)
+          @dataset.insert(**collector)
+        end
+      end
+
+      ##
+      # A sink which will always issue an update statement
+      class UpdateSink
+        def initialize(dataset, key_columns)
+          @dataset = dataset
+          @key_columns = if key_columns.is_a? Symbol
+                           [key_columns]
+                         elsif key_columns.respond_to? :each
+                           key_columns
+                           # TODO: else throw error
+                         end
+        end
+
+        def <<(collector)
+          dataset = @dataset
+          @key_columns.each do |col|
+            dataset = dataset.where(**{ col => collector[col] })
+          end
+          dataset.update(**collector)
+        end
+      end
+
+      ##
+      # A sink which will update an item if it exists, or insert it otherwise
+      class InsertUpdateSink
+        def initialize(dataset, key_columns, update_actions: {}, default_update_action: :coalesce,
+                       match_empty_key: false)
+          @dataset = dataset
+          @key_columns = if key_columns.is_a? Symbol
+                           [key_columns]
+                         elsif key_columns.respond_to? :each
+                           key_columns
+                           # TODO: else throw error
+                         end
+          @update_actions = update_actions
+          @default_update_action = default_update_action
+          @match_empty_key = match_empty_key
+        end
+
+        def <<(collector)
+          dataset = @dataset
+          @key_columns.each do |col|
+            dataset = dataset.where(**{ col => collector[col] })
+          end
+          existing = dataset.limit(2).all
+          if existing.count > 1
+            raise Micdrop::SinkError, "Key column(s) of this InsertUpdateSink are not unique"
+          elsif existing.empty?
+            dataset.insert(**collector)
+          else
+            dataset.update(**update_merge(existing.first, collector))
+          end
+        end
+
+        private
+
+        def update_merge(existing, collector)
+          if @update_actions.empty?
+            # If we don't have per-column actions, we can take shortcuts for some actions types
+            return collector if @default_update_action == :always_overwrite
+            return collector.compact if @default_update_action == :coalesce
+          end
+          # Otherwise merge according to the rules specified
+          existing.merge(collector) do |key, oldval, newval|
+            case @update_actions.fetch(key, @default_update_action)
+            when :coalesce then newval.nil? ? oldval : newval
+            when :overwrite_nulls then oldval.nil? ? newval : oldval
+            when :always_overwrite then newval
+            when :keep_existing then oldval
+            when :append then format("%s %s", oldval, newval)
+            when :append_line then format("%s\n%s", oldval, newval)
+            when :prepend then format("%s %s", newval, oldval)
+            when :prepend_line then format("%s\n%s", newval, oldval)
+            when :add then oldval + newval
+            end
+          end
+        end
+      end
+    end
+  end
+
+  ##
+  # Sequel-specific extensions for ItemContext
+  class ItemContext
+    def db_lookup(dataset, key_col, val_col, pass_if_not_found: false, warn_if_not_found: nil, apply_if_not_found: nil)
+      # TODO: allow registering db_lookups like we do normal lookups
+      warn_if_not_found = true if warn_if_not_found.nil? && apply_if_not_found.nil?
+      found = dataset.where(key_col => @value).get(val_col)
+      if found.nil?
+        warn format "Value %s not found in db_lookup", @value if warn_if_not_found
+        if !apply_if_not_found.nil?
+          apply apply_if_not_found
+        elsif !pass_if_not_found
+          @value = nil
+        end
+      else
+        @value = found
+      end
+      self
+    end
+  end
+end

data/lib/micdrop/files_source.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Micdrop
+  ##
+  # Takes a list of files, directory, or glob pattern as a source
+  #
+  # Records in a file source will have the following items available to `take`:
+  # * :content (The full content of the file, lazy-loaded only if you `take` it)
+  # * :stream (A file IO stream)
+  # * :filename (The filename that was used to load the file, e.g. files/s.json)
+  # * :basename (The basename of the name, e.g. x.json)
+  # * :path (The full path to the file, e.g. /data/migration/files/x.json)
+  # * Anything returned by File.stat (:ctime, :mtime, :size, etc...)
+  class FilesSource
+    def initialize(dir, files: nil, glob: nil, binary_mode: false, **file_opts)
+      @dir = dir
+      @files = if files.nil?
+                 if glob.nil?
+                   Dir.children(dir)
+                 else
+                   Dir.glob(glob, flags: File::FNM_EXTGLOB, base: dir)
+                 end
+               else
+                 files
+               end
+      @binary_mode = binary_mode
+      @file_opts = file_opts
+    end
+
+    def each_pair
+      return enum_for :each_pair unless block_given?
+
+      @files.each do |filename|
+        path = File.join(@dir, filename)
+        unless File.file? path
+          warn format("%s is not a file and will be skipped", path)
+          next
+        end
+
+        yield filename, FilesSourceRecord.new(path, @binary_mode, @file_opts)
+      end
+    end
+  end
+
+  ##
+  # Wrapper object to expose files as a source item
+  class FilesSourceRecord
+    def initialize(filename, binary_mode, file_opts)
+      @filename = filename
+      @binary_mode = binary_mode
+      @file_opts = file_opts
+      @stat = nil
+    end
+
+    def [](key)
+      case key
+      when :contents, :content
+        File.open @filename, @binary_mode ? "rb" : "r", **@file_opts, &:read
+      when :stream
+        File.open @filename, @binary_mode ? "rb" : "r", **@file_opts
+      when :path
+        File.absolute_path @filename
+      when :basename
+        File.basename @filename
+      when :filename
+        @filename
+      else
+        @stat = File.stat(@filename) if @stat.nil?
+        @stat.method(key).call
+      end
+    end
+  end
+end