flat_kit 0.1.0

data/lib/flat_kit/output.rb

@@ -0,0 +1,32 @@
+module FlatKit
+  class Output
+    extend DescendantTracker
+
+    def self.from(out)
+      return out if out.kind_of?(::FlatKit::Output)
+
+      out_klass = find_child(:handles?, out)
+      if out_klass then
+        return out_klass.new(out)
+      end
+
+      raise FlatKit::Error, "Unable to create output from #{out.class} : #{out.inspect}"
+    end
+
+    def name
+      raise NotImplementedError, "#{self.class} must implement #name"
+    end
+
+    #
+    def io
+      raise NotImplementedError, "#{self.class} must implement #io"
+    end
+
+    def close
+      raise NotImplementedError, "#{self.class} must implement #close"
+    end
+  end
+end
+
+require 'flat_kit/output/io'
+require 'flat_kit/output/file'

data/lib/flat_kit/output/file.rb

@@ -0,0 +1,55 @@
+require 'zlib'
+
+module FlatKit
+  class Output
+    class File < Output
+      attr_reader :path
+
+      def self.handles?(obj)
+        return true if obj.instance_of?(Pathname)
+        return false unless obj.instance_of?(String)
+
+        # incase these get loaded in different orders
+        return false if ::FlatKit::Output::IO.is_stdout?(obj)
+        return false if ::FlatKit::Output::IO.is_stderr?(obj)
+
+        return true
+      end
+
+      def initialize(obj)
+        @path = Pathname.new(obj)
+        path.dirname.mkpath
+        @io = open_output(path)
+      end
+
+      def name
+        path.to_s
+      end
+
+      def close
+        @io.close
+      end
+
+      # internal api method for testing purposes
+      def io
+        @io
+      end
+
+      private
+
+      # open the opropriate otuput type depending on the destination file name
+      #
+      # TODO: add in bzip
+      def open_output(path)
+        case path.extname
+        when ".gz"
+          Zlib::GzipWriter.open(path.to_s)
+        # when ".gz"
+        #   ::IO.popen("gzip -c > #{path}", "w")
+        else
+          path.open("wb")
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/output/io.rb

@@ -0,0 +1,73 @@
+module FlatKit
+  class Output
+    class IO < Output
+      attr_reader :count
+
+      STDOUTS = %w[ stdout STDOUT - <stdout> ]
+      STDERRS = %w[ stderr STDERR <stderr> ]
+
+      def self.handles?(obj)
+        return true if is_stderr?(obj)
+        return true if is_stdout?(obj)
+        return true if [ ::File, ::StringIO, ::IO ].any? { |klass| obj.kind_of?(klass) }
+        return false
+      end
+
+      def self.is_stderr?(obj)
+        case obj
+        when String
+          return true if STDERRS.include?(obj)
+        when ::IO
+          return true if obj == ::STDERR
+        end
+        return false
+      end
+
+      def self.is_stdout?(obj)
+        case obj
+        when String
+          return true if STDOUTS.include?(obj)
+        when ::IO
+          return true if obj == ::STDOUT
+        end
+        return false
+      end
+
+      def initialize(obj)
+        @count = 0
+        if self.class.is_stdout?(obj) then
+          @name = "<STDOUT>"
+          @io = $stdout
+        elsif self.class.is_stderr?(obj) then
+          @name = "<STDERR>"
+          @io = $stderr
+        elsif obj.kind_of?(::File) then
+          @name = obj.path
+          @io = obj
+        elsif obj.kind_of?(::StringIO) then
+          @name = obj.inspect
+          @io = obj
+        elsif obj.kind_of?(::IO) then
+          @name = obj.inspect
+          @io = obj
+        else
+          raise ::FlatKit::Error, "Unable to create #{self.class} from #{obj.class} : #{obj.inspect}"
+        end
+      end
+
+      def name
+        @name
+      end
+
+      # this goes to an io stream and we are not in charge of opening it
+      def close
+        @io.close
+      end
+
+      # internal api method for testing
+      def io
+        @io
+      end
+    end
+  end
+end

data/lib/flat_kit/reader.rb

@@ -0,0 +1,61 @@
+module FlatKit
+  # Public: the base class for all format readers.
+  #
+  # A format reader needs to be able to open the appropriate file format and
+  # implement Enumerable to iterate over all the records in the file format.
+  #
+  # If it is appropriate for the reader to be able to read from a IO object
+  # directly, that needs to be supported also.
+  #
+  # The ::FlatKit::Reader class should never be used directly, only the reader
+  # from the appropriate format should be used.
+  #
+  #
+  # API:
+  #
+  #   initialize(source:, compare_fields:)
+  #   each -> Yields / returns 
+  #
+  class Reader
+    include Enumerable
+
+    attr_reader :source
+    attr_reader :compare_fields
+
+    def self.create_reader_from_path(path: "-", fallback: "auto", compare_fields: :none)
+      format = ::FlatKit::Format.for_with_fallback!(path: path, fallback: fallback)
+      return format.reader.new(source: path, compare_fields: compare_fields)
+    end
+
+    def self.create_readers_from_paths(paths:, fallback: "auto", compare_fields: :none)
+      # default to stdin if there are no paths
+      if paths.empty? then
+        paths << "-"
+      end
+
+      paths.map do |path|
+        create_reader_from_path(path: path, fallback: fallback, compare_fields: compare_fields)
+      end
+    end
+
+    def initialize(source:, compare_fields: :none)
+      @source = source
+      @compare_fields = resolve_compare_fields(compare_fields)
+    end
+
+    def format_name
+      self.class.format_name
+    end
+
+    def each
+      raise NotImplementedError, "#{self.class} needs to implement #each"
+    end
+
+    private
+
+    def resolve_compare_fields(value)
+      return [] if value == :none
+      return value
+    end
+  end
+end

data/lib/flat_kit/record.rb

@@ -0,0 +1,83 @@
+module FlatKit
+  # Public: The base class that all record classes should inherit from.
+  #
+  # Its goal is to be an efficient comparator of data that can be inflated from
+  # a source structure to a fully realized hash.
+  #
+  # All records need to be able to be initialized from a data structure that it
+  # is handed to it by the Reader intance within the same Format.
+  #
+  # Records are generally not going to be created outside of this library, they
+  # are tied to a specific format and provide a common interface that can be
+  # used for:
+  #
+  #   * comparison between records from different source / destinations formats
+  #   * conversion to a different format
+  #
+  # Given that - the way to create a record is either from another Record
+  # instance:
+  #
+  #   Record.from_record(other)  # create a record from another record
+  #
+  # or the way a Reader will do it
+  #
+  #   Record.new(...)            # generally only used by a Reader instance to
+  #                              # yield new reocrds
+  #
+  #
+  # When Implementing a new Format, the corresponding Record class for that
+  # Format must:
+  #
+  #   * implement `#[](key)` which will be used to lookup the values of the
+  #     comparable fields.
+  #   * implement `#to_hash` which is used when conversions
+  #   * implement `.from_record` which is used in conversion
+  #   # the initialize method must call super(data:, compare_fields:) to
+  #     initializa the root data structures
+  class Record
+
+    include Comparable
+
+    attr_reader :data
+    attr_reader :compare_fields
+
+    def initialize(data:, compare_fields:)
+      @data = data
+      @compare_fields = compare_fields
+    end
+
+    def format_name
+      self.class.format_name
+    end
+
+    def <=>(other)
+      compare_result = nil
+
+      compare_fields.each do |field|
+        my_val         = self[field]
+        other_val      = other[field]
+
+        return  0 if my_val.nil? && other_val.nil?
+        return -1 if my_val.nil?
+        return  1 if other_val.nil?
+
+        compare_result = my_val.<=>(other_val)
+
+        return compare_result unless compare_result.zero?
+      end
+      compare_result
+    end
+
+    def [](key)
+      raise NotImplementedError, "#{self.class} must implement #[](key)"
+    end
+
+    def to_hash
+      raise NotImplementedError, "#{self.class} must implement #to_hash"
+    end
+
+    def self.from_record
+      raise NotImplementedError, "#{self.class} must implement #{self.class}.from_record"
+    end
+  end
+end

data/lib/flat_kit/sentinel_internal_node.rb

@@ -0,0 +1,37 @@
+module FlatKit
+  # Private: The Sentinel Internal Node is a private class used by the MergeTree
+  # class.
+  #
+  # This class represents an empty / completed node in the merge tree where all
+  # the data from the descendant leaf node is full used up.
+  #
+  class SentinelInternalNode
+    include Comparable
+
+    attr_reader :left
+    attr_reader :right
+    attr_reader :winner
+    attr_accessor :next_level
+
+    def initialize(left: nil, right: nil)
+      @left = nil
+      @right = nil
+      @winner = nil
+      @next_level = nil
+    end
+
+    def sentinel?
+      true
+    end
+
+    def leaf?
+      true
+    end
+
+    # A sentinal node is always greater than any other node
+    def <=>(other)
+      return 0 if other.sentinel?
+      return 1
+    end
+  end
+end

data/lib/flat_kit/sentinel_leaf_node.rb

@@ -0,0 +1,37 @@
+module FlatKit
+  # Private: The Sentinel Leaf Node is used internally by the MergeTree
+  #
+  # This class represents a LeafNode that has no more data
+  #
+  class SentinelLeafNode
+    include Comparable
+
+    attr_accessor :next_level
+
+    def sentinel?
+      true
+    end
+
+    def leaf?
+      true
+    end
+
+    def next
+      nil
+    end
+
+    def finished?
+      true
+    end
+
+    def value
+      nil
+    end
+
+    # A sentinal node is always greater than any other node
+    def <=>(other)
+      return 0 if other.sentinel?
+      return 1
+    end
+  end
+end

data/lib/flat_kit/sort.rb

@@ -0,0 +1,35 @@
+module FlatKit
+  class Sort
+    attr_reader :reader
+    attr_reader :writer
+    attr_reader :compare_fields
+
+    def initialize(input:, input_fallback: "auto",
+                   output:, output_fallback: "auto",
+                   compare_fields:)
+
+      @compare_fields = compare_fields
+      @reader = ::FlatKit::Reader.create_reader_from_path(path: input, compare_fields: @compare_fields,
+                                                          fallback: input_fallback)
+      @writer = ::FlatKit::Writer.create_writer_from_path(path: output, fallback: output_fallback,
+                                                          reader_format: @reader.format_name)
+    end
+
+    def call
+      ::FlatKit.logger.info "Sorting #{reader.source} into #{writer.destination} using key #{compare_fields.join(", ")}"
+      records = Array.new.tap do |a|
+        reader.each do |r|
+          a << r
+        end
+      end
+      ::FlatKit.logger.info "Read #{reader.count} records into #{records.size} element array"
+      records.sort!
+      ::FlatKit.logger.info "Sorted #{records.size} records"
+      records.each do |r|
+        writer.write(r)
+      end
+      writer.close
+      ::FlatKit.logger.info "Wrote #{writer.count} records to #{writer.destination}"
+    end
+  end
+end

data/lib/flat_kit/writer.rb

@@ -0,0 +1,38 @@
+module FlatKit
+  # Public: The base class for all format writers.
+  #
+  # A format writer will only write those Records, and on that, only those of
+  # its own format.
+  #
+  # It must implement a #write methods takes a Record. It can convert the record
+  # to one matching its own format if it whishes. But it should in any case
+  # check the Record format to make sure it matches
+  #
+  # See the Xsv::Writer and Jsonl::Writer for examples.
+  #
+  class Writer
+    attr_reader :destination
+
+    def self.create_writer_from_path(path:, fallback:, reader_format:)
+      fallback = reader_format if fallback == "auto"
+      format   = ::FlatKit::Format.for_with_fallback!(path: path, fallback: fallback)
+      format.writer.new(destination: path)
+    end
+
+    def initialize(destination:)
+      @destination = destination
+    end
+
+    def format_name
+      self.class.format_name
+    end
+
+    def write(record)
+      raise NotImplementedError, "#{self.class} needs to implement #write"
+    end
+
+    def close
+      raise NotImplementedError, "#{self.class} needs to implement #close"
+    end
+  end
+end