flat_kit 0.2.0 → 1.0.0

data/lib/flat_kit/jsonl/record.rb

@@ -1,8 +1,12 @@
-require 'oj'
-require 'flat_kit/record'
+# frozen_string_literal: true
+
+require "oj"
+require "flat_kit/record"
 
 module FlatKit
   module Jsonl
+    # Internal: Class that exposes data from a JSONL format record to the flatkit api
+    #
     class Record < ::FlatKit::Record
       attr_reader :compare_data
 
@@ -11,7 +15,7 @@ module FlatKit
       end
 
       def self.from_record(record)
-        if record.instance_of?(FlatKit::Jsonl::Record) then
+        if record.instance_of?(FlatKit::Jsonl::Record)
 
           structured = record.complete_structured_data? ? record.complete_structured_data : nil
 
@@ -25,22 +29,20 @@ module FlatKit
       end
 
       def initialize(data:, compare_fields: :none,
-                     compare_data: Hash.new,
+                     compare_data: {},
                      complete_structured_data: nil)
         super(data: data, compare_fields: compare_fields)
 
         @complete_structured_data = complete_structured_data
 
-        if complete_structured_data? && (compare_data.nil? || compare_data.empty?) then
-          @compare_data = complete_structured_data
-        else
-          @compare_data = compare_data
-        end
+        @compare_data = if complete_structured_data? && (compare_data.nil? || compare_data.empty?)
+                          complete_structured_data
+                        else
+                          compare_data
+                        end
 
         # only load compare data if it dosn't exist
-        if data && compare_data.empty? then
-          quick_parse
-        end
+        quick_parse if data && compare_data.empty?
       end
 
       def [](key)
@@ -48,7 +50,7 @@ module FlatKit
       end
 
       def complete_structured_data
-        @complete_structured_data ||= Oj.load(data)
+        @complete_structured_data ||= Oj.load(data, mode: :strict)
       end
       alias to_hash complete_structured_data
 
@@ -59,9 +61,7 @@ module FlatKit
       # overriding parent accessor since we may be initialized without raw bytes
       # to parse
       def data
-        if @data.nil? && complete_structured_data? then
-          @data = Oj.dump(complete_structured_data)
-        end
+        @data = Oj.dump(complete_structured_data, mode: :json) if @data.nil? && complete_structured_data?
         @data
       end
       alias to_s data
@@ -79,6 +79,3 @@ module FlatKit
     end
   end
 end
-
-
-

data/lib/flat_kit/jsonl/writer.rb

@@ -1,19 +1,16 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Jsonl
+    # Internal: Class that writes flatkit records to JSONL files
+    #
     class Writer < ::FlatKit::Writer
-      attr_reader :output
-      attr_reader :count
-
       def self.format_name
         ::FlatKit::Jsonl::Format.format_name
       end
 
-      def initialize(destination:)
-        super
-        @output = ::FlatKit::Output.from(@destination)
-        @count = 0
-      end
-
+      # write the record and return the Position the record was written
+      #
       def write(record)
         case record
         when FlatKit::Jsonl::Record
@@ -24,21 +21,31 @@ module FlatKit
         else
           raise FlatKit::Error, "Unable to write records of type #{record.class}"
         end
-      rescue FlatKit::Error => fe
-        raise fe
-      rescue => e
-        ::FlatKit.logger.error "Error reading jsonl records from #{output.name}: #{e}"
+      rescue FlatKit::Error => e
+        raise e
+      rescue StandardError => e
+        ::FlatKit.logger.error "Error writing jsonl records to #{output.name}: #{e}"
         raise ::FlatKit::Error, e
       end
 
-      def close
-        @output.close
-      end
-
       def write_record(record)
-        # enforces ending in newlin if it doesn't already have one
+        # the index of the record being written is the same as the count of records written so far
+        record_index = @count
+
+        # get the current output stream position to calculate bytes written
+        start_offset = output.io.tell
+
+        # enforces ending in newline if it doesn't already have one
         output.io.puts record.to_s
+
+        ending_offset = output.io.tell
+        bytes_written = (ending_offset - start_offset)
+
         @count += 1
+
+        @last_position = ::FlatKit::Position.new(index: record_index,
+                                                 offset: start_offset,
+                                                 bytesize: bytes_written)
       end
     end
   end

data/lib/flat_kit/jsonl.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Top level namespace for the newline-oriented JSON format
+  #
   module Jsonl
   end
 end
-require 'flat_kit/jsonl/record'
-require 'flat_kit/jsonl/reader'
-require 'flat_kit/jsonl/writer'
-require 'flat_kit/jsonl/format'
+require "flat_kit/jsonl/record"
+require "flat_kit/jsonl/reader"
+require "flat_kit/jsonl/writer"
+require "flat_kit/jsonl/format"

data/lib/flat_kit/leaf_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: The LeafNode is a wrapper around a Reader object to enable
   # a consistent api for use in the MergeTree
@@ -9,11 +11,9 @@ module FlatKit
   # If all the data is used up from the reader, it also notifies the next level
   # of that so the next level can remove it from the tree.
   class LeafNode
-
     include Comparable
 
-    attr_reader :reader
-    attr_reader :value
+    attr_reader :reader, :value
 
     attr_accessor :next_level
 
@@ -43,7 +43,7 @@ module FlatKit
 
     def update_and_replay
       self.next
-      if finished? then
+      if finished?
         ::FlatKit.logger.debug "#{reader.source} has finished reading #{reader.count} records"
         next_level.player_finished(self)
       end
@@ -65,7 +65,8 @@ module FlatKit
 
     def <=>(other)
       return -1 if other.sentinel?
-      self.value.<=>(other.value)
+
+      value <=> (other.value)
     end
   end
 end

data/lib/flat_kit/log_formatter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module FlatKit
+  # Internal: Log formatting class for FlatKit
+  #
+  class LogFormatter < ::Logger::Formatter
+    FORMAT          = "%s %5d %05s : %s\n"
+    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+    def initialize
+      super
+      self.datetime_format = DATETIME_FORMAT
+    end
+
+    def call(severity, time, _progname, msg)
+      format(FORMAT, format_datetime(time.utc), Process.pid, severity, msg2str(msg))
+    end
+  end
+end

data/lib/flat_kit/logger.rb

@@ -1,19 +1,12 @@
-require 'logger'
+# frozen_string_literal: true
 
-module FlatKit
-  class LogFormatter < ::Logger::Formatter
-    FORMAT          = "%s %5d %05s : %s\n".freeze
-    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ".freeze
-    def initialize
-      super
-      self.datetime_format = DATETIME_FORMAT
-    end
-
-    def call(severity, time, progname, msg)
-      FORMAT % [format_datetime(time.utc), Process.pid, severity, msg2str(msg)]
-    end
-  end
+require "logger"
 
+# Public: Top level namespace for the gem
+#
+module FlatKit
+  # Internal: Logger class
+  #
   class Logger
     def self.for_io(io)
       ::Logger.new(io, formatter: LogFormatter.new)
@@ -26,11 +19,11 @@ module FlatKit
   end
 
   def self.log_to(destination = $stderr)
-    if destination.kind_of?(::IO) then
-      @logger = ::FlatKit::Logger.for_io(destination)
-    else
-      @logger = ::FlatKit::Logger.for_path(destination)
-    end
+    @logger = if destination.is_a?(::IO)
+                ::FlatKit::Logger.for_io(destination)
+              else
+                ::FlatKit::Logger.for_path(destination)
+              end
   end
 
   def self.logger

data/lib/flat_kit/merge.rb

@@ -1,15 +1,14 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Class implementing merging from N inputs and output to 1 output.
+  #
   class Merge
-
     include ::FlatKit::EventEmitter
 
-    attr_reader :readers
-    attr_reader :writer
-    attr_reader :compare_fields
+    attr_reader :readers, :writer, :compare_fields
 
-    def initialize(inputs:, input_fallback: "auto",
-                   output:, output_fallback: "auto",
-                   compare_fields:)
+    def initialize(inputs:, output:, compare_fields:, input_fallback: "auto", output_fallback: "auto")
       @compare_fields = compare_fields
       @readers = ::FlatKit::Reader.create_readers_from_paths(paths: inputs, compare_fields: @compare_fields,
                                                              fallback: input_fallback)
@@ -19,19 +18,12 @@ module FlatKit
 
     def call
       ::FlatKit.logger.debug "Merging the following files into #{writer.destination}"
-      ::FlatKit.logger.debug "Using this key for sorting: #{compare_fields.join(", ")}"
+      ::FlatKit.logger.debug "Using this key for sorting: #{compare_fields.join(', ')}"
       readers.each do |r|
         ::FlatKit.logger.debug "  #{r.source}"
       end
 
-      merge_tree = ::FlatKit::MergeTree.new(readers)
-
-      notify_listeners(name: :start, data: :start)
-      merge_tree.each do |record|
-        writer.write(record)
-        notify_listeners(name: :record, data: record)
-      end
-      notify_listeners(name: :stop, data: :stop)
+      run_merge(readers)
 
       readers.each do |r|
         ::FlatKit.logger.debug "  #{r.source} produced #{r.count} records"
@@ -40,5 +32,18 @@ module FlatKit
       writer.close
       ::FlatKit.logger.debug "Wrote #{writer.count} records to #{writer.destination}"
     end
+
+    private
+
+    def run_merge(readers)
+      tree = ::FlatKit::MergeTree.new(readers)
+      notify_listeners(name: :start, data: :start)
+      tree.each do |record|
+        position = writer.write(record)
+        meta = { position: position }
+        notify_listeners(name: :record, data: record, meta: meta)
+      end
+      notify_listeners(name: :stop, data: :stop)
+    end
   end
 end

data/lib/flat_kit/merge_tree.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: Merge a list of sorted records from Readers into a single output Writer
   #
@@ -29,9 +31,7 @@ module FlatKit
   class MergeTree
     include Enumerable
 
-    attr_reader :leaves
-    attr_reader :levels
-    attr_reader :readers
+    attr_reader :leaves, :levels, :readers
 
     def initialize(readers)
       @readers = readers
@@ -44,9 +44,7 @@ module FlatKit
 
       # Need to pad the leaves to an even number so that the slicing by 2 for
       # the tournament will work
-      if @leaves.size.odd? then
-        @leaves << SentinelLeafNode.new
-      end
+      @leaves << SentinelLeafNode.new if @leaves.size.odd?
 
       init_tree
     end
@@ -94,6 +92,7 @@ module FlatKit
     def each
       loop do
         break if root.leaf.finished?
+
         yield root.value
         # consume the yielded value and have the tournament tree replay those
         # brackets affected

data/lib/flat_kit/output/file.rb

@@ -1,22 +1,31 @@
-require 'zlib'
+# frozen_string_literal: true
+
+require "zlib"
+require "pathname"
 
 module FlatKit
   class Output
+    # Internal: File output implementation
+    #
     class File < Output
       attr_reader :path
 
+      # internal api method for testing purposes
+      attr_reader :io
+
       def self.handles?(obj)
         return true if obj.instance_of?(Pathname)
         return false unless obj.instance_of?(String)
 
         # incase these get loaded in different orders
-        return false if ::FlatKit::Output::IO.is_stdout?(obj)
-        return false if ::FlatKit::Output::IO.is_stderr?(obj)
+        return false if ::FlatKit::Output::IO.stdout?(obj)
+        return false if ::FlatKit::Output::IO.stderr?(obj)
 
-        return true
+        true
       end
 
       def initialize(obj)
+        super()
         @path = Pathname.new(obj)
         path.dirname.mkpath
         @io = open_output(path)
@@ -30,11 +39,6 @@ module FlatKit
         @io.close
       end
 
-      # internal api method for testing purposes
-      def io
-        @io
-      end
-
       private
 
       # open the opropriate otuput type depending on the destination file name

data/lib/flat_kit/output/io.rb

@@ -1,73 +1,78 @@
+# frozen_string_literal: true
+
 module FlatKit
   class Output
+    # Internal: Non-file Output impelementation - this is genrally to stdout or stderr
+    #
     class IO < Output
-      attr_reader :count
+      attr_reader :count, :name
+
+      # internal api method for testing
+      attr_reader :io
 
-      STDOUTS = %w[ stdout STDOUT - <stdout> ]
-      STDERRS = %w[ stderr STDERR <stderr> ]
+      STDOUTS = %w[stdout STDOUT - <stdout>].freeze
+      STDERRS = %w[stderr STDERR <stderr>].freeze
 
       def self.handles?(obj)
-        return true if is_stderr?(obj)
-        return true if is_stdout?(obj)
-        return true if [ ::File, ::StringIO, ::IO ].any? { |klass| obj.kind_of?(klass) }
-        return false
+        return true if stderr?(obj)
+        return true if stdout?(obj)
+        return true if [::File, ::StringIO, ::IO].any? { |klass| obj.is_a?(klass) }
+
+        false
       end
 
-      def self.is_stderr?(obj)
+      def self.stderr?(obj)
         case obj
         when String
           return true if STDERRS.include?(obj)
         when ::IO
-          return true if obj == ::STDERR
+          return true if obj == $stderr
         end
-        return false
+        false
       end
 
-      def self.is_stdout?(obj)
+      def self.stdout?(obj)
         case obj
         when String
           return true if STDOUTS.include?(obj)
         when ::IO
-          return true if obj == ::STDOUT
+          return true if obj == $stdout
         end
-        return false
+        false
       end
 
       def initialize(obj)
+        super()
         @count = 0
-        if self.class.is_stdout?(obj) then
+        @name = nil
+        @io = nil
+        init_name_and_io(obj)
+      end
+
+      # this goes to an io stream and we are not in charge of opening it
+      def close
+        @io.close
+      end
+
+      private
+
+      def init_name_and_io(obj)
+        if self.class.stdout?(obj)
           @name = "<STDOUT>"
           @io = $stdout
-        elsif self.class.is_stderr?(obj) then
+        elsif self.class.stderr?(obj)
           @name = "<STDERR>"
           @io = $stderr
-        elsif obj.kind_of?(::File) then
-          @name = obj.path
+        elsif obj.is_a?(::IO)
+          @name = (obj.respond_to?(:path) && obj.path) || obj.inspect
           @io = obj
-        elsif obj.kind_of?(::StringIO) then
-          @name = obj.inspect
-          @io = obj
-        elsif obj.kind_of?(::IO) then
+        elsif obj.is_a?(::StringIO)
           @name = obj.inspect
           @io = obj
         else
           raise ::FlatKit::Error, "Unable to create #{self.class} from #{obj.class} : #{obj.inspect}"
         end
       end
-
-      def name
-        @name
-      end
-
-      # this goes to an io stream and we are not in charge of opening it
-      def close
-        @io.close
-      end
-
-      # internal api method for testing
-      def io
-        @io
-      end
     end
   end
 end

data/lib/flat_kit/output.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Base clases for all output handlers
+  #
   class Output
     extend DescendantTracker
 
     def self.from(out)
-      return out if out.kind_of?(::FlatKit::Output)
+      return out if out.is_a?(::FlatKit::Output)
 
       out_klass = find_child(:handles?, out)
-      if out_klass then
-        return out_klass.new(out)
-      end
+      return out_klass.new(out) if out_klass
 
       raise FlatKit::Error, "Unable to create output from #{out.class} : #{out.inspect}"
     end
@@ -17,16 +19,19 @@ module FlatKit
       raise NotImplementedError, "#{self.class} must implement #name"
     end
 
-    #
     def io
       raise NotImplementedError, "#{self.class} must implement #io"
     end
 
+    def tell
+      io.tell
+    end
+
     def close
       raise NotImplementedError, "#{self.class} must implement #close"
     end
   end
 end
 
-require 'flat_kit/output/io'
-require 'flat_kit/output/file'
+require "flat_kit/output/io"
+require "flat_kit/output/file"

data/lib/flat_kit/position.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module FlatKit
+  # The information about the position of a record in an IO stream
+  #
+  # Generally this is going to be returned by a write_record method to return
+  # information about the record that was just written
+  #
+  class Position
+    attr_reader :index, :offset, :bytesize # zero based   # byte offset in the IO stream # byte length of the record
+
+    def initialize(index: nil, offset: nil, bytesize: nil)
+      @index    = index
+      @offset   = offset
+      @bytesize = bytesize
+    end
+  end
+end

data/lib/flat_kit/reader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: the base class for all format readers.
   #
@@ -14,24 +16,21 @@ module FlatKit
   # API:
   #
   #   initialize(source:, compare_fields:)
-  #   each -> Yields / returns 
+  #   each -> Yields / returns
   #
   class Reader
     include Enumerable
 
-    attr_reader :source
-    attr_reader :compare_fields
+    attr_reader :source, :compare_fields
 
     def self.create_reader_from_path(path: "-", fallback: "auto", compare_fields: :none)
       format = ::FlatKit::Format.for_with_fallback!(path: path, fallback: fallback)
-      return format.reader.new(source: path, compare_fields: compare_fields)
+      format.reader.new(source: path, compare_fields: compare_fields)
     end
 
     def self.create_readers_from_paths(paths:, fallback: "auto", compare_fields: :none)
       # default to stdin if there are no paths
-      if paths.empty? then
-        paths << "-"
-      end
+      paths << "-" if paths.empty?
 
       paths.map do |path|
         create_reader_from_path(path: path, fallback: fallback, compare_fields: compare_fields)
@@ -55,7 +54,8 @@ module FlatKit
 
     def resolve_compare_fields(value)
       return [] if value == :none
-      return value
+
+      value
     end
   end
 end

data/lib/flat_kit/record.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: The base class that all record classes should inherit from.
   #
@@ -35,11 +37,9 @@ module FlatKit
   #   # the initialize method must call super(data:, compare_fields:) to
   #     initializa the root data structures
   class Record
-
     include Comparable
 
-    attr_reader :data
-    attr_reader :compare_fields
+    attr_reader :data, :compare_fields
 
     def initialize(data:, compare_fields:)
       @data = data
@@ -57,15 +57,15 @@ module FlatKit
         my_val         = self[field]
         other_val      = other[field]
 
-        if my_val.nil? && other_val.nil? then
-          compare_result = 0
-        elsif my_val.nil?
-          compare_result = -1
-        elsif other_val.nil?
-          compare_result = 1
-        else
-          compare_result = my_val.<=>(other_val)
-        end
+        compare_result = if my_val.nil? && other_val.nil?
+                           0
+                         elsif my_val.nil?
+                           -1
+                         elsif other_val.nil?
+                           1
+                         else
+                           my_val <=> (other_val)
+                         end
 
         return compare_result unless compare_result.zero?
       end