flat_kit 0.1.0

data/lib/flat_kit/command/merge.rb

@@ -0,0 +1,88 @@
+require 'csv'
+module FlatKit
+  class Command
+    class Merge < ::FlatKit::Command
+
+      def self.name
+        "merge"
+      end
+
+      def self.description
+        "Merge sorted files together that have the same structure."
+      end
+
+      def self.parser
+        ::Optimist::Parser.new do
+          banner "#{Merge.description}"
+          banner ""
+
+          banner <<~BANNER
+          Given a set of input files that have the same structure, and are already
+          sorted by a set of keys. The Merge command will merge all those files
+          into a single output file.
+
+          The --key parameter is required, and it must be a comma separated list
+          of field nams on the input on which to use as the sort key for the merge
+          process.
+
+          There must also be at least 2 input files. Merging only 1 file into an
+          output file is the same as the 'cat' command.
+
+          The flatfile type(s) will be automatically determined by the file name.
+          If the output is not a file, but to stdout then the output type will
+          be the same as the first input file, or it can be specified as a commandline
+          switch.
+
+          The merge will do a single pass through the input to generate the
+          output.
+          BANNER
+
+          banner <<~USAGE
+
+          Usage:
+            fk merge --key surname,given_name file1.csv file2.csv > all.csv
+            fk merge --key surname,given_name --output-format json file1.csv file2.csv > all.json
+            fk merge --key field1,field2 --output-format json input*.csv | gzip -c > all.json.gz
+            fk merge --key field12 file*.json.gz -o all.json.gz
+
+          USAGE
+
+          banner <<~OPTIONS
+
+          Options:
+
+          OPTIONS
+
+          opt :output, "Send the output to the given path instead of standard out.", default: "<stdout>"
+          opt :input_format, "Input format, csv or json", default: "auto", short: :none
+          opt :output_format, "Output format, csv or json", default: "auto", short: :none
+          opt :key, "The comma separted list of field(s) to use for sorting the input", required: true, type: :string
+        end
+      end
+
+      attr_reader :compare_keys
+
+      def parse
+        parser = self.class.parser
+        ::Optimist::with_standard_exception_handling(parser) do
+          begin
+            @opts = parser.parse(argv)
+            @compare_keys = CSV.parse_line(opts[:key])
+            paths = parser.leftovers
+            raise ::Optimist::CommandlineError, "At least 2 input files are required" if paths.size < 2
+
+            @merge = ::FlatKit::Merge.new(inputs: paths, input_fallback: opts[:input_format],
+                                          compare_fields: @compare_keys,
+                                          output: opts[:output], output_fallback: opts[:output_format])
+          rescue ::FlatKit::Error => e
+            raise ::Optimist::CommandlineError, e.message
+          end
+        end
+      end
+
+      def call
+        @merge.call
+      end
+    end
+  end
+end

data/lib/flat_kit/command/sort.rb

@@ -0,0 +1,88 @@
+require 'csv'
+module FlatKit
+  class Command
+    class Sort < ::FlatKit::Command
+
+      def self.name
+        "sort"
+      end
+
+      def self.description
+        "Sort a given file by a set of fields."
+      end
+
+      def self.parser
+        ::Optimist::Parser.new do
+          banner "#{Sort.description}"
+          banner ""
+
+          banner <<~BANNER
+          Given an input file and a sort key, order the records in that file by that
+          key. If no input file is given the stdin is assumed. If no output file
+          is given then stdout is assumed.
+
+          The --key parameter is required, and it must be a comma separated list
+          of field nams on the input on which to use as the sort key for the merge
+          process.
+
+          There must also be only 1 input files.
+
+          The flatfile type(s) will be automatically determined by the file name.
+          If the output is not a file, but to stdout then the output type will
+          be the same as the first input file, or it can be specified as a commandline
+          switch.
+
+          BANNER
+
+          banner <<~USAGE
+
+          Usage:
+            fk sort --key surname,given_name file.csv > sorted.csv
+            fk sort --key surname,given_name --output-format json file.csv > sorted.json
+            fk sort --key field1,field2 --output-format json input.csv | gzip -c > sorted.json.gz
+            fk sort --key field1 file.json.gz -o sorted.json.gz
+            gunzip -c file.json.gz | fk sort --key field1 --input-format json --output-format json > gzip -c sorted.json.gz
+
+          USAGE
+
+          banner <<~OPTIONS
+
+          Options:
+
+          OPTIONS
+
+          opt :output, "Send the output to the given path instead of standard out.", default: "<stdout>"
+          opt :input_format, "Input format, csv or json", default: "auto", short: :none
+          opt :output_format, "Output format, csv or json", default: "auto", short: :none
+          opt :key, "The comma separted list of field(s) to use for sorting the input", required: true, type: :string
+        end
+      end
+
+      attr_reader :compare_keys
+      attr_reader :reader
+      attr_reader :sort
+
+      def parse
+        parser = self.class.parser
+        ::Optimist::with_standard_exception_handling(parser) do
+          begin
+            @opts = parser.parse(argv)
+            @compare_keys = CSV.parse_line(opts[:key])
+            paths = parser.leftovers
+            raise ::Optimist::CommandlineError, "1 and only 1 input file is allowed" if paths.size > 1
+            path = paths.first || "-" # default to stdin
+            @sort = ::FlatKit::Sort.new(input: path, input_fallback: opts[:input_format],
+                                        output: opts[:output], output_fallback: opts[:output_format],
+                                        compare_fields: @compare_keys)
+          rescue ::FlatKit::Error => e
+            raise ::Optimist::CommandlineError, e.message
+          end
+        end
+      end
+
+      def call
+        sort.call
+      end
+    end
+  end
+end

data/lib/flat_kit/descendant_tracker.rb

@@ -0,0 +1,27 @@
+require 'set'
+
+module FlatKit
+  module DescendantTracker
+    def inherited(klass)
+      super
+      return unless klass.instance_of?(Class)
+      self.children << klass
+    end
+
+    def children
+      unless defined? @_children
+        @_children = Set.new
+      end
+      @_children
+    end
+
+    #
+    # Find the first child that returns truthy from the given method with args
+    #
+    def find_child(method, *args)
+      children.find do |child_klass|
+        child_klass.send(method, *args)
+      end
+    end
+  end
+end

data/lib/flat_kit/error.rb

@@ -0,0 +1,5 @@
+module FlatKit
+  class Error < ::StandardError
+    class UnknownFormat < ::FlatKit::Error; end
+  end
+end

data/lib/flat_kit/format.rb

@@ -0,0 +1,34 @@
+module FlatKit
+  class Format
+    extend DescendantTracker
+
+    def self.format_name
+      raise NotImplementedError, "#{self.class} must implemente #{self.class}.format_name"
+    end
+
+    def format_name
+      self.class.format_name
+    end
+
+    def self.for(path)
+      find_child(:handles?, path.to_s)
+    end
+
+    def self.for_with_fallback(path:, fallback: "auto")
+      # test by path
+      format = ::FlatKit::Format.for(path)
+      return format unless format.nil?
+
+      # now try the fallback
+      format = ::FlatKit::Format.for(fallback)
+      return format
+    end
+
+    def self.for_with_fallback!(path:, fallback: "auto")
+      format = for_with_fallback(path: path, fallback: fallback)
+      raise ::FlatKit::Error::UnknownFormat, "Unable to figure out format for '#{path}' with fallback '#{fallback}'" if format.nil?
+
+      return format
+    end
+  end
+end

data/lib/flat_kit/input.rb

@@ -0,0 +1,32 @@
+module FlatKit
+  class Input
+    extend DescendantTracker
+
+    def self.from(input)
+      return input if input.kind_of?(::FlatKit::Input)
+
+      in_klass = find_child(:handles?, input)
+      if in_klass then
+        return in_klass.new(input)
+      end
+
+      raise FlatKit::Error, "Unable to create input from #{input.class} : #{input.inspect}"
+    end
+
+    def name
+      raise NotImplementedError, "#{self.class} must implement #name"
+    end
+
+    #
+    def io
+      raise NotImplementedError, "#{self.class} must implement #io"
+    end
+
+    def close
+      raise NotImplementedError, "#{self.class} must implement #close"
+    end
+  end
+end
+
+require 'flat_kit/input/io'
+require 'flat_kit/input/file'

data/lib/flat_kit/input/file.rb

@@ -0,0 +1,53 @@
+require 'zlib'
+
+module FlatKit
+  class Input
+    class File < Input
+      attr_reader :path
+      attr_reader :count
+
+      def self.handles?(obj)
+        return true if obj.instance_of?(Pathname)
+        return false unless obj.instance_of?(String)
+
+        # incase these get loaded in different orders
+        return false if ::FlatKit::Input::IO.is_stdin?(obj)
+
+        return true
+      end
+
+      def initialize(obj)
+        @count = 0
+        @path = Pathname.new(obj)
+        raise FlatKit::Error, "Input #{obj} is not readable" unless @path.readable?
+        @io = open_input(path)
+      end
+
+      def name
+        path.to_s
+      end
+
+      def close
+        @io.close
+      end
+
+      def io
+        @io
+      end
+
+      private
+
+      # open the opropriate input type depending on the source file name
+      #
+      # TODO: add in bzip
+      def open_input(path)
+        case path.extname
+        when ".gz"
+          Zlib::GzipReader.open(path.to_s)
+        else
+          path.open("rb")
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/input/io.rb

@@ -0,0 +1,54 @@
+module FlatKit
+  class Input
+    class IO < Input
+      STDINS = %w[ stdin STDIN - <stdin> ]
+
+      def self.handles?(obj)
+        return true if is_stdin?(obj)
+        return true if [ ::File, ::StringIO, ::IO ].any? { |klass| obj.kind_of?(klass) }
+        return false
+      end
+
+      def self.is_stdin?(obj)
+        case obj
+        when String
+          return true if STDINS.include?(obj)
+        when ::IO
+          return true if obj == ::STDIN
+        end
+        return false
+      end
+
+      def initialize(obj)
+        if self.class.is_stdin?(obj) then
+          @name = "<STDIN>"
+          @io = $stdin
+        elsif obj.kind_of?(::File) then
+          @name = obj.path
+          @io = obj
+        elsif obj.kind_of?(::StringIO) then
+          @name = obj.inspect
+          @io = obj
+        elsif obj.kind_of?(::IO) then
+          @name = obj.inspect
+          @io = obj
+        else
+          raise ::FlatKit::Error, "Unable to create #{self.class} from #{obj.class} : #{obj.inspect}"
+        end
+      end
+
+      def name
+        @name
+      end
+
+      # this goes to an io stream and we are not in charge of opening it
+      def close
+        @io.close
+      end
+
+      def io
+        @io
+      end
+    end
+  end
+end

data/lib/flat_kit/internal_node.rb

@@ -0,0 +1,84 @@
+module FlatKit
+  # Private: This is a class used internally by MergeTree and should not be used
+  # outside of that context.
+  #
+  # The InternalNode represents a single element of the tournament tree
+  # altorithm holding references to the to other internal nodes that competed in
+  # this node and which one is the winner.
+  #
+  # A reference to the leaf node that is associated with the winner is also kept
+  # here.
+  #
+  class InternalNode
+
+    include Comparable
+
+    attr_accessor :left        # Internal Node
+    attr_accessor :right       # Internal Node
+    attr_accessor :winner      # Internal Node
+    attr_accessor :next_level  # Who to tell
+    attr_accessor :leaf        # winning leaf node
+
+    def initialize(left:, right:)
+      @left       = left
+      @left.next_level = self
+
+      @right      = right
+      @right.next_level = self
+      @next_level  = nil
+
+      play
+    end
+
+    def value
+      winner.value
+    end
+
+    def sentinel?
+      false
+    end
+
+    def leaf?
+      false
+    end
+
+    # We are being told that the passed in node no longer has data in it and is
+    # to be removed from the tree.
+    #
+    # We replace our reference to this node with a sentinal node so that
+    # comparisons work correctly.
+    #
+    # After updating the node, we then need to check and see if both of our
+    # child nodes are sentinels, and if so, then tell our parent to remove us
+    # from the tree.
+    #
+    def player_finished(node)
+      if left.object_id == node.object_id then
+        @left = SentinelInternalNode.new
+        @left.next_level = self
+      elsif right.object_id == node.object_id then
+        @right = SentinelInternalNode.new
+        @right.next_level = self
+      else
+        raise FlatKit::Error, "Unknown player #{node}"
+      end
+
+      if @right.sentinel? && @left.sentinel? then
+        next_level.player_finished(self) if next_level
+      end
+    end
+
+    def play
+      @winner = left <= right ? left : right
+      if !@winner.sentinel? then
+        @leaf = winner.leaf
+      end
+      next_level.play if next_level
+    end
+
+    def <=>(other)
+      return -1 if other.sentinel?
+      value.<=>(other.value)
+    end
+  end
+end