micdrop 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e2a6458190e5d2bc6537824586b135a8d6a9a0c24ab55d9b4de5db9197b5c88
-  data.tar.gz: 869d3c01befdb76e074c339caf9288a3be3734365fe5d3b4b9d6cf0e772cdc91
+  metadata.gz: 8e567ed6bfc0a336d28ec7f7cee0d3bd987109b83d1ac634071a79d875002824
+  data.tar.gz: 0bd90f322d2201b782b3cca29213ffd8ba4078f671ce3a04302e9404ec7a14f7
 SHA512:
-  metadata.gz: 182e84bd143229048a783333aabaa4fdbf7ad26020b75c06ae6692e142ce223aad0262c60e3260916df96b1dc5432623b22e9e5ab40c2115fe4cb0b8aee1792d
-  data.tar.gz: 6b81a5ef76726c2738e09a8fdb6f117f8e1df2d7c0cb3c699eb0fc65efd7ba3584ebdb16791dcad4d8e9612d94eacd4a7a5860e59972b1bd7112ba858dcdb00b
+  metadata.gz: ee163fb3d7be2fd634e465196fd5053622d9b46246cf5177a75c00bbbdf7cba99c60d7fae661648c652e8a8f500580f3342b7f4fb99764330960c3fecbd62c86
+  data.tar.gz: d081c87bcb894e3718290794892be8e48283507615cdeee2b6b47206fb3497f1e45f26d32894b58f7b0cc312031f75c9cd48aca540e3fd35d4d912117cfa75c9

data/README.md

@@ -446,3 +446,15 @@ Micdrop.migrate source, sink do
     end
 ```
 
+If needed, you can also use the `before_flush` or `after_flush` hooks to add actions before or after the flush. Both take the same form:
+
+```ruby
+Micdrop.migrate source, sink do
+    after_flush do |record, collected|
+        # `record` is the RootRecordContext, `collected` is the hash of `put` values.
+        # For example, you could do something like this if the sink was a Sequel InsertSource
+        puts "Inserted ID #{record.sink.insert_id} with data #{collected.inspect}"
+    end
+    # Then do your normal migration operations here
+end
+```

data/examples/data/catalog.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<catalog>
+  <product id="P001">
+    <name>Laptop Pro 15</name>
+    <category>Electronics</category>
+    <price currency="USD">1299.99</price>
+    <inStock>true</inStock>
+    <quantity>45</quantity>
+    <specifications>
+      <processor>Intel Core i7-13700H</processor>
+      <ram>16GB DDR5</ram>
+      <storage>512GB NVMe SSD</storage>
+      <display>15.6" FHD IPS</display>
+      <graphics>NVIDIA RTX 4060</graphics>
+    </specifications>
+    <images>
+      <image type="thumbnail">laptop-thumb.jpg</image>
+      <image type="main">laptop-main.jpg</image>
+    </images>
+    <ratings>
+      <average>4.5</average>
+      <count>127</count>
+    </ratings>
+  </product>
+  <product id="P002">
+    <name>Wireless Mouse</name>
+    <category>Accessories</category>
+    <price currency="USD">29.99</price>
+    <inStock>true</inStock>
+    <quantity>230</quantity>
+    <specifications>
+      <connectivity>Bluetooth 5.0</connectivity>
+      <battery>2x AA</battery>
+      <dpi>1600</dpi>
+      <buttons>6</buttons>
+    </specifications>
+    <images>
+      <image type="thumbnail">mouse-thumb.jpg</image>
+      <image type="main">mouse-main.jpg</image>
+    </images>
+    <ratings>
+      <average>4.7</average>
+      <count>89</count>
+    </ratings>
+  </product>
+</catalog>

data/examples/data/readme.md

@@ -2,4 +2,5 @@
 
 Sources:
 * <https://github.com/datablist/sample-csv-files>
-* JSON made with [Faker](https://faker.readthedocs.io/en/master/)
+* JSON made with [Faker](https://faker.readthedocs.io/en/master/)
+* <https://jsontotable.org/blog/xml/sample-xml-files>

data/examples/xml_to_sql.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "micdrop"
+require "sequel"
+require "micdrop/ext/sequel"
+require "micdrop/ext/nokogiri"
+
+DB = Sequel.sqlite "test.db"
+
+# Create the destination data structure.
+# Obviously in a real import script, these would probably already exist.
+
+DB.create_table :products do
+  String :code, primary_key: true
+  String :name
+  String :category
+  BigDecimal :price, size: [6, 2]
+  FixNum :stock
+end
+
+DB.create_table :product_specs do
+  String :code
+  String :key
+  String :value
+  primary_key %i[code key]
+end
+
+# Now start the migration
+document = Nokogiri::XML.parse File.open File.join(__dir__, "data/catalog.xml")
+
+# Our source will iterate over the <product> elements in the XML document
+source = document.css("product")
+sink = Micdrop::Ext::Sequel::InsertSink.new DB[:products]
+
+Micdrop.migrate source, sink do
+  # The files source exposes the basename and content as takeable items
+  take "id", put: :code
+  at_css("name").take_content put: :name
+  at_css("category").take_content put: :category
+  at_css("price").take_content do
+    parse_float
+    put :price
+  end
+  at_css("quantity").take_content put: :stock
+end
+
+# Then over the individual specs
+source = document.css("product")
+sink = Micdrop::Ext::Sequel::InsertSink.new DB[:product_specs]
+
+Micdrop.migrate source, sink do
+  # The files source exposes the basename and content as takeable items
+  code = take "id"
+  css("specifications > *").each_subrecord(flush: true, reset: true) do
+    code.put :code
+    take_node_name do
+      lookup({
+               "battery" => "Battery",
+               "buttons" => "Button Count",
+               "connectivity" => "Connectivity",
+               "display" => "Screen",
+               "dpi" => "Screen DPI",
+               "graphics" => "GPU",
+               "processor" => "CPU",
+               "ram" => "Memory",
+               "storage" => "Storage"
+             })
+      put :key
+    end
+    take_content.put :value
+  end
+end

data/lib/micdrop/ext/microfocus.rb

@@ -0,0 +1,240 @@
+require "date"
+require "forwardable"
+
+module Micdrop
+  module Ext
+    ##
+    # A simple parser to extract data from a "Micro Focus File with Header (DAT)" file.
+    #
+    # Based on this spec: https://www.microfocus.com/documentation/server-express/sx20books/fhfile.htm
+    #
+    # This format comes from old COBOL programs, and each file is conceptually similar to an SQL
+    # database table. Unlike SQL though, these DAT files lack type information; each row is raw
+    # binary and must be unpacked.
+    #
+    # This does not implement the full spec, and is not well tested, but "works on my machine".
+    module Microfocus
+      ##
+      # A header value that appears at the beginning of each record to determine the record type
+      module RecordType
+        DUPLICATE_SYSTEM    = 0b0001
+        DELETED             = 0b0010
+        SYSTEM              = 0b0011
+        NORMAL              = 0b0100
+        REDUCED             = 0b0101
+        POINTER             = 0b0110
+        POINTER_REF         = 0b0111
+        REDUCED_POINTER_REF = 0b1000
+      end
+
+      ##
+      # Flag indicating how records are organized in the file
+      module RecordOrganization
+        SEQUENTIAL = 1
+        INDEXED = 2
+        RELATIVE = 3
+      end
+
+      ##
+      # Representation of a single record within a file
+      class Record
+        extend Forwardable
+
+        def initialize(type, body, unpack_spec: nil, unpack_mapping: nil)
+          @type = type
+          @body = body
+          @fields = nil
+          unpack unpack_spec, unpack_mapping unless unpack_spec.nil?
+        end
+
+        attr_reader :type, :body, :fields
+
+        def_delegators :@fields, :[], :each
+
+        private
+
+        def unpack(spec, mapping = nil)
+          fields = @body.unpack spec
+          fields = if mapping.nil?
+                     fields
+                   else
+                     mapping.transform_values { |value| fields[value] }
+                   end
+          @fields = fields.freeze
+        end
+      end
+
+      ##
+      # Read a MicroFocus data file
+      class MicroFocusReader
+        def initialize(data_file, unpack_spec: nil, unpack_mapping: nil)
+          @data_file = data_file
+          @unpack_spec = unpack_spec
+          @unpack_mapping = unpack_mapping
+          read_data_header
+        end
+
+        attr_reader :creation_time, :compression, :index_type, :variable_length, :min_legth, :max_length, :index_version
+
+        def long_records?
+          @long_records
+        end
+
+        def sequential?
+          @organization == RecordOrganization::SEQUENTIAL
+        end
+
+        def indexed?
+          @organization == RecordOrganization::INDEXED
+        end
+
+        def relative?
+          @organization == RecordOrganization::RELATIVE
+        end
+
+        def each
+          return enum_for :each unless block_given?
+
+          yield read_record until @data_file.eof?
+        end
+
+        private
+
+        def read_data_header
+          parse_data_file_header @data_file.read(128)
+        end
+
+        def read_record
+          header = @data_file.read(@long_records ? 4 : 2)
+          type = header.unpack1("C") >> 4
+          length = header.unpack1(@long_records ? "N" : "n") & (@long_records ? 0xFFFFFFF : 0xFFF)
+          body = @data_file.read length
+          scan_padding
+          Record.new type, body, unpack_spec: @unpack_spec, unpack_mapping: @unpack_mapping
+        end
+
+        ##
+        # Parse the first four bytes of the header, which are used to determine the record size
+        def parse_data_file_header(data)
+          # The first 4 bits are the record type, which must be SYSTEM
+          type = data.unpack1("C") >> 4
+          raise StandardError, "This file does not have a valid header" unless type == RecordType::SYSTEM
+
+          # The next 12 bits (or 28 bits, depending on the max record size) are the header record size
+          length = data.unpack1("n") & 0xFFF
+          if length == 126
+            # Header data is 126 bytes, max record length is less than 4095 bytes
+            @long_records = false
+          elsif length == 0
+            # Header data is 124 bytes, max record length is 4095 bytes or greater
+            length = data.unpack1("N") & 0xFFF
+            raise StandardError, "Invalid header record length" unless length == 124
+
+            @long_records = true
+          else
+            raise StandardError, "Invalid header record length"
+          end
+
+          # Regardless of the listed header length, actual header data always at the same byte offsets
+          (
+              @db_seq,
+              integrity, # The specs say this integrity flag is 3 bytes, not 2, but I think the spec must be wrong
+              creation_time,
+              special62,
+              @organization,
+              @compression,
+              @index_type,
+              variable_length,
+              @min_legth,
+              @max_length,
+              @index_version
+          ) = data.unpack "x4 n n A14 x14 n x C x C x C x C x5 N N x46 N"
+
+          # Check integrity
+          raise StandardError, "Integrity flag non-zero; file is corrupt" if integrity != 0
+          raise StandardError, "Bytes 36-37 not equal to 64; file is corrupt" if special62 != 62
+
+          # Type-cast some of the header values
+          @creation_time = DateTime.strptime creation_time[0..11], "%y%m%d%H%M%S"
+          @variable_length = !!variable_length.nil?
+        end
+
+        ##
+        # Scan forward to the next non-null byte
+        def scan_padding
+          # TODO: This is a work-around because it seems I don't have align_cursor working correctly yet
+          return if @data_file.eof?
+
+          return if @data_file.eof? until @data_file.readbyte.positive?
+          @data_file.seek(-1, :CUR)
+        end
+
+        ##
+        # Aligns the file cursor to the next address which is a multiple of the data alignment value
+        #
+        # Automatically detect the the alignment from the index if not provided
+        #
+        # Index formats 1 and 2 have no alignment, 3 and 4 are aligned to 4 bytes, and 8 is aligned to 8 bytes
+        def align_cursor
+          alignment = if @index_type < 3
+                        return # offset of 1, so we don't need to do anything
+                      elsif @index_type < 5
+                        4
+                      else
+                        8
+                      end
+
+          offset = @data_file.tell % alignment
+          @file.seek offset, :CUR if offset
+        end
+      end
+
+      ##
+      # This is the main entrypoint to read a file, and its output is usable as a source.
+      #
+      # `unpack_spec` is an optional spec, as would be passed to `String#unpack`, to extract the
+      # individual columns from the record. You may also provice an `unpack_mapping` which maps more
+      # human-readable columns names to column indexes.
+      def self.read_microfocus_file(filename, unpack_spec: nil, unpack_mapping: nil)
+        File.open filename, "rb" do |file|
+          reader = MicroFocusReader.new file, unpack_spec: unpack_spec, unpack_mapping: unpack_mapping
+          reader.each.entries
+        end
+      end
+    end
+  end
+
+  ##
+  # Extend ItemContext with parse_microfocus
+  class ItemContext
+    ##
+    # Parse a string as JSON
+    #
+    # If a block is provided, it will act as a record context where object properties can be taken.
+    #
+    # If include_header is true, the value will be a hash containing both the header information
+    # and the actual records.
+    def parse_microfocus(include_header: false, unpack_spec: nil, unpack_mapping: nil, &block)
+      return self if @value.nil?
+
+      reader = Micdrop::Ext::Microfocus::MicroFocusReader.new @value, unpack_spec: unpack_spec,
+                                                                      unpack_mapping: unpack_mapping
+      @value = if include_header
+                 {
+                   creation_time: reader.creation_time,
+                   compression: reader.compression,
+                   index_type: reader.index_type,
+                   variable_length: reader.variable_length,
+                   min_legth: reader.min_legth,
+                   max_length: reader.max_length,
+                   index_version: reader.index_version,
+                   records: reader.each.entries
+                 }
+               else
+                 reader.each.entries
+               end
+      enter(&block) unless block.nil?
+      self
+    end
+  end
+end

data/lib/micdrop/ext/nokogiri.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Micdrop
+  ##
+  # Extend ItemContext with HTML/XML functions
+  class ItemContext
+    ##
+    # Alias for scope.enter.take_content
+    def take_content(put: nil, convert: nil, apply: nil, &block)
+      scope.enter.take_content(put: put, convert: convert, apply: apply, &block)
+    end
+
+    ##
+    # Alias for scope.enter.take_node_name
+    def take_node_name(put: nil, convert: nil, apply: nil, &block)
+      scope.enter.take_node_name(put: put, convert: convert, apply: apply, &block)
+    end
+
+    ##
+    # Parse HTML and enter a sub-record context for the root node
+    def parse_html(&block)
+      doc = @value.nil? ? nil : ::Nokogiri::HTML.parse(@value)
+      nokogiri_node_subrecord_helper(doc, block)
+    end
+
+    ##
+    # Parse HTML5 and enter a sub-record context for the root node
+    def parse_html5(&block)
+      doc = @value.nil? ? nil : ::Nokogiri::HTML5.parse(@value)
+      nokogiri_node_subrecord_helper(doc, block)
+    end
+
+    ##
+    # Parse XML and enter a sub-record context for the root node
+    def parse_xml(&block)
+      doc = @value.nil? ? nil : ::Nokogiri::XML.parse(@value)
+      nokogiri_node_subrecord_helper(doc, block)
+    end
+
+    ##
+    # Parse an HTML fragment and enter a sub-record context for the root node
+    def parse_html_fragment(&block)
+      doc = @value.nil? ? nil : ::Nokogiri::HTML.fragment(@value)
+      nokogiri_node_subrecord_helper(doc, block)
+    end
+
+    ##
+    # Parse an HTML5 fragment and enter a sub-record context for the root node
+    def parse_html5_fragment(&block)
+      doc = @value.nil? ? nil : ::Nokogiri::HTML5.fragment(@value)
+      nokogiri_node_subrecord_helper(doc, block)
+    end
+
+    ##
+    # Parse an XML fragment and enter a sub-record context for the root node
+    def parse_xml_fragment(&block)
+      doc = @value.nil? ? nil : Nokogiri::XML.fragment(@value)
+      nokogiri_node_subrecord_helper(doc, block)
+    end
+
+    ##
+    # Decode an HTML entity-encoded string to plain text
+    def decode_html
+      return self if @value.nil?
+
+      frag = ::Nokogiri::HTML.fragment @value
+      @value = frag.content
+      self
+    end
+
+    ##
+    # Encode a string using HTML entities
+    def encode_html(nl2br: false)
+      return self if @value.nil?
+
+      frag = ::Nokogiri::HTML.fragment ""
+      frag.content = @value
+      @value = frag.to_s
+      @value = @value.sub "\n", "<br/>" if nl2br
+      self
+    end
+
+    ##
+    # Decode an HTML5 entity-encoded string to plain text
+    def decode_html5
+      return self if @value.nil?
+
+      frag = ::Nokogiri::HTML5.fragment @value
+      @value = frag.content
+      self
+    end
+
+    ##
+    # Encode a string using HTML5 entities
+    def encode_html5(nl2br: false)
+      return self if @value.nil?
+
+      frag = ::Nokogiri::HTML5.fragment ""
+      frag.content = @value
+      @value = frag.to_s
+      @value = @value.sub "\n", "<br/>" if nl2br
+      self
+    end
+
+    ##
+    # Decode an XML entity-encoded string to plain text
+    def decode_xml
+      return self if @value.nil?
+
+      frag = ::Nokogiri::XML.fragment @value
+      @value = frag.content
+      self
+    end
+
+    ##
+    # Encode a string using XML entities
+    def encode_xml
+      return self if @value.nil?
+
+      frag = ::Nokogiri::XML.fragment ""
+      frag.content = @value
+      @value = frag.to_s
+      self
+    end
+
+    private
+
+    def nokogiri_node_subrecord_helper(node, block)
+      item_ctx = ItemContext.new @record_context, node
+      subrec_ctx = SubRecordContext.new item_ctx, @record_context
+      subrec_ctx.instance_eval(&block) unless block.nil?
+      subrec_ctx
+    end
+  end
+
+  ##
+  # Extend RecordContext with HTML/XML functions
+  class RecordContext
+    ##
+    # Take the text content of the XML or HTML node
+    def take_content(put: nil, convert: nil, apply: nil, &block)
+      value = @record&.content
+      process_item_helper(value, put, convert, apply, block)
+    end
+
+    ##
+    # Take the node name of the XML or HTML node
+    def take_node_name(put: nil, convert: nil, apply: nil, &block)
+      value = @record&.node_name
+      process_item_helper(value, put, convert, apply, block)
+    end
+
+    def xpath(*args, &block)
+      nokogiri_node_subrecord_helper(@record.xpath(*args), block)
+    end
+
+    def at_xpath(*args, &block)
+      nokogiri_node_subrecord_helper(@record.at_xpath(*args), block)
+    end
+
+    def css(*args, &block)
+      nokogiri_node_subrecord_helper(@record.css(*args), block)
+    end
+
+    def at_css(*args, &block)
+      nokogiri_node_subrecord_helper(@record.at_css(*args), block)
+    end
+
+    private
+
+    def nokogiri_node_subrecord_helper(node, block)
+      item_ctx = ItemContext.new self, node
+      subrec_ctx = SubRecordContext.new item_ctx, self
+      subrec_ctx.instance_eval(&block) unless block.nil?
+      subrec_ctx
+    end
+  end
+end

data/lib/micdrop/ext/sequel.rb

@@ -12,8 +12,10 @@ module Micdrop
           @dataset = dataset
         end
 
+        attr_reader :insert_id
+
         def <<(collector)
-          @dataset.insert(**collector)
+          @insert_id = @dataset.insert(**collector)
         end
       end
 
@@ -56,6 +58,8 @@ module Micdrop
           @match_empty_key = match_empty_key
         end
 
+        attr_reader :insert_id, :was_insert
+
         def <<(collector)
           dataset = @dataset
           @key_columns.each do |col|
@@ -65,9 +69,12 @@ module Micdrop
           if existing.count > 1
             raise Micdrop::SinkError, "Key column(s) of this InsertUpdateSink are not unique"
           elsif existing.empty?
-            dataset.insert(**collector)
+            @insert_id = dataset.insert(**collector)
+            @was_insert = true
           else
             dataset.update(**update_merge(existing.first, collector))
+            @insert_id = nil
+            @was_insert = false
           end
         end
 
@@ -101,10 +108,15 @@ module Micdrop
   ##
   # Sequel-specific extensions for ItemContext
   class ItemContext
-    def db_lookup(dataset, key_col, val_col, pass_if_not_found: false, warn_if_not_found: nil, apply_if_not_found: nil)
+    def db_lookup(dataset, key_col, val_col = nil, pass_if_not_found: false, warn_if_not_found: nil,
+                  apply_if_not_found: nil)
       # TODO: allow registering db_lookups like we do normal lookups
       warn_if_not_found = true if warn_if_not_found.nil? && apply_if_not_found.nil?
-      found = dataset.where(key_col => @value).get(val_col)
+      found = if val_col.nil?
+                dataset.where(key_col => @value).first
+              else
+                dataset.where(key_col => @value).get(val_col)
+              end
       if found.nil?
         warn format "Value %s not found in db_lookup", @value if warn_if_not_found
         if !apply_if_not_found.nil?

data/lib/micdrop/item_context.rb

@@ -251,6 +251,13 @@ module Micdrop
 
     ### Common operations ###
 
+    def send(*args)
+      return self if @value.nil?
+
+      @value = @value.send(*args)
+      self
+    end
+
     ##
     # Lookup the value in a hash
     #

data/lib/micdrop/record_context.rb

@@ -38,6 +38,18 @@ module Micdrop
       process_item_helper(value, put, convert, apply, block)
     end
 
+    ##
+    # Take the entire record as a single item
+    def take_whole(put: nil, convert: nil, apply: nil, &block)
+      process_item_helper(record, put, convert, apply, block)
+    end
+
+    ##
+    # alias for take_whole.each_subrecord
+    def each_subrecord(flush: false, reset: false, &block)
+      take_whole.each_subrecord(flush: flush, reset: reset, &block)
+    end
+
     ##
     # A combined take/put shorthand, for migrations where many of the column names are the same
     def passthru(*names)
@@ -89,8 +101,6 @@ module Micdrop
       process_item_helper(value, put, convert, apply, block)
     end
 
-    # TODO: collect_hash (not sure what the signature of it should be?)
-
     ##
     # Skip the current record. This is similar to a plain-ruby `next` statement.
     def skip
@@ -124,6 +134,8 @@ module Micdrop
       @loop_item = loop_item
       @record = loop_item
       @loop_index = loop_index
+      @before_flush = nil
+      @after_flush = nil
       reset
     end
 
@@ -153,7 +165,9 @@ module Micdrop
     def flush(reset: true)
       return unless @dirty
 
+      @before_flush&.call self, @collector
       @sink << @collector
+      @after_flush&.call self, @collector
       self.reset if reset
     end
 
@@ -176,6 +190,22 @@ module Micdrop
                      {}
                    end
     end
+
+    ##
+    # Allows specifying a hook which will run before flush. The block will receive the record and the collector.
+    #
+    # Note that this must be called *before* any manual flush occurs to have any effect.
+    def before_flush(&block)
+      @before_flush = block
+    end
+
+    ##
+    # Allows specifying a hook which will run after flush. The block will receive the record and the collector.
+    #
+    # Note that this must be called *before* any manual flush occurs to have any effect.
+    def after_flush(&block)
+      @after_flush = block
+    end
   end
 
   ##

data/lib/micdrop/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Micdrop
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: micdrop
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Dominick Johnson
@@ -27,6 +27,7 @@ files:
 - Rakefile
 - TODO.md
 - examples/csvs_to_sql.rb
+- examples/data/catalog.xml
 - examples/data/customers-100.csv
 - examples/data/json/1.json
 - examples/data/json/2.json
@@ -42,8 +43,11 @@ files:
 - examples/data/people-100.csv
 - examples/data/readme.md
 - examples/json_files_to_sql.rb
+- examples/xml_to_sql.rb
 - lib/micdrop.rb
 - lib/micdrop/errors.rb
+- lib/micdrop/ext/microfocus.rb
+- lib/micdrop/ext/nokogiri.rb
 - lib/micdrop/ext/sequel.rb
 - lib/micdrop/files_source.rb
 - lib/micdrop/item_context.rb