flat_kit 0.2.0 → 1.0.0

data/lib/flat_kit/xsv/writer.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Xsv
+    # Internal: Write that takes flatkit records and writes them to XSV Output
+    #
     class Writer < ::FlatKit::Writer
-      attr_reader :output
-      attr_reader :count
-      attr_reader :fields
+      attr_reader :fields, :header_bytes
 
       def self.format_name
         ::FlatKit::Xsv::Format.format_name
@@ -12,29 +14,33 @@ module FlatKit
       def self.default_csv_options
         {
           headers: nil,
-          write_headers: true
+          write_headers: true,
         }
       end
 
       def initialize(destination:, fields: :auto, **csv_options)
         super(destination: destination)
         @fields = fields
-        @output = ::FlatKit::Output.from(@destination)
-        @count = 0
         @we_write_the_header = nil
         @csv_options = Writer.default_csv_options.dup
 
-        if @fields == :auto then
+        if @fields == :auto
           @we_write_the_header = true
         else
-          @csv_options.merge!(headers: fields)
+          @csv_options[:headers] = fields
           @we_write_the_header = false
         end
 
+        @header_bytes = 0
         @csv_options.merge!(csv_options)
         @csv = CSV.new(output.io, **@csv_options)
       end
 
+      # write the record and return the Position the record was written
+      #
+      # In the case of the header being written automatcially, the Postion returned is the
+      # position of the reocrd, not the header
+      #
       def write(record)
         case record
         when FlatKit::Xsv::Record
@@ -45,25 +51,37 @@ module FlatKit
         else
           raise FlatKit::Error, "Unable to write records of type #{record.class}"
         end
-      rescue FlatKit::Error => fe
-        raise fe
-      rescue => e
-        ::FlatKit.logger.error "Error reading jsonl records from #{output.name}: #{e}"
+      rescue FlatKit::Error => e
+        raise e
+      rescue StandardError => e
+        ::FlatKit.logger.error "Error writing xsv records to #{output.name}: #{e}"
         raise ::FlatKit::Error, e
       end
 
-      def close
-        @output.close
-      end
-
       private
 
       def write_record(record)
-        if @we_write_the_header && @count == 0 then
+        if @we_write_the_header && @count.zero?
           @csv << record.ordered_fields
+          @header_bytes = output.tell
         end
-        @count += 1
+
+        # the index of the record being written is the same as the count of records written so far
+        record_index = @count
+
+        # get the current output stream position to calculate bytes written
+        start_offset = output.tell
+
         @csv << record.to_a
+
+        ending_offset = output.io.tell
+        bytes_written = (ending_offset - start_offset)
+
+        @count += 1
+
+        @last_position = ::FlatKit::Position.new(index: record_index,
+                                                 offset: start_offset,
+                                                 bytesize: bytes_written)
       end
     end
   end

data/lib/flat_kit/xsv.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Namespace for the xsv file fromat handling
   module Xsv
   end
 end
-require 'flat_kit/xsv/record'
-require 'flat_kit/xsv/reader'
-require 'flat_kit/xsv/writer'
-require 'flat_kit/xsv/format'
+require "flat_kit/xsv/record"
+require "flat_kit/xsv/reader"
+require "flat_kit/xsv/writer"
+require "flat_kit/xsv/format"

data/lib/flat_kit.rb

@@ -1,24 +1,36 @@
+# frozen_string_literal: true
+
+# Public: Top level namespace for this gem
+#
 module FlatKit
-  VERSION = "0.2.0"
+  VERSION = "1.0.0"
 end
-require 'flat_kit/error'
-require 'flat_kit/logger'
-require 'flat_kit/event_emitter'
-require 'flat_kit/descendant_tracker'
-require 'flat_kit/format'
-require 'flat_kit/record'
-require 'flat_kit/reader'
-require 'flat_kit/writer'
-require 'flat_kit/input'
-require 'flat_kit/output'
-require 'flat_kit/cli'
-require 'flat_kit/xsv'
-require 'flat_kit/jsonl'
-require 'flat_kit/merge'
-require 'flat_kit/sort'
+require "flat_kit/error"
+require "flat_kit/descendant_tracker"
+require "flat_kit/log_formatter"
+require "flat_kit/logger"
+require "flat_kit/event_emitter"
+
+require "flat_kit/field_type"
+require "flat_kit/format"
+require "flat_kit/position"
+require "flat_kit/record"
+require "flat_kit/reader"
+require "flat_kit/writer"
+require "flat_kit/input"
+require "flat_kit/output"
+require "flat_kit/cli"
+require "flat_kit/xsv"
+require "flat_kit/jsonl"
+require "flat_kit/merge"
+require "flat_kit/sort"
+require "flat_kit/stats"
+
+require "flat_kit/stat_type"
+require "flat_kit/field_stats"
 
-require 'flat_kit/merge_tree'
-require 'flat_kit/internal_node'
-require 'flat_kit/sentinel_internal_node'
-require 'flat_kit/sentinel_leaf_node'
-require 'flat_kit/leaf_node'
+require "flat_kit/merge_tree"
+require "flat_kit/internal_node"
+require "flat_kit/sentinel_internal_node"
+require "flat_kit/sentinel_leaf_node"
+require "flat_kit/leaf_node"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flat_kit
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Jeremy Hinegardner
 autorequire:
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2021-02-03 00:00:00.000000000 Z
+date: 2024-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -39,75 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-- !ruby/object:Gem::Dependency
-  name: minitest-focus
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.3'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.21'
-  type: :development
+        version: '3.3'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.21'
+        version: '3.3'
 description: A library and commandline program for reading, writing, indexing,  sorting,
   and merging CSV, TSV, JSON and other flat-file formats.
 email: jeremy@copiousfreetime.org
@@ -126,17 +70,29 @@ files:
 - LICENSE.txt
 - Manifest.txt
 - README.md
-- Rakefile
-- bin/fk
+- exe/fk
+- flat_kit.gemspec
 - lib/flat_kit.rb
 - lib/flat_kit/cli.rb
 - lib/flat_kit/command.rb
 - lib/flat_kit/command/cat.rb
 - lib/flat_kit/command/merge.rb
 - lib/flat_kit/command/sort.rb
+- lib/flat_kit/command/stats.rb
 - lib/flat_kit/descendant_tracker.rb
 - lib/flat_kit/error.rb
 - lib/flat_kit/event_emitter.rb
+- lib/flat_kit/field_stats.rb
+- lib/flat_kit/field_type.rb
+- lib/flat_kit/field_type/boolean_type.rb
+- lib/flat_kit/field_type/date_type.rb
+- lib/flat_kit/field_type/float_type.rb
+- lib/flat_kit/field_type/guess_type.rb
+- lib/flat_kit/field_type/integer_type.rb
+- lib/flat_kit/field_type/null_type.rb
+- lib/flat_kit/field_type/string_type.rb
+- lib/flat_kit/field_type/timestamp_type.rb
+- lib/flat_kit/field_type/unknown_type.rb
 - lib/flat_kit/format.rb
 - lib/flat_kit/input.rb
 - lib/flat_kit/input/file.rb
@@ -148,51 +104,40 @@ files:
 - lib/flat_kit/jsonl/record.rb
 - lib/flat_kit/jsonl/writer.rb
 - lib/flat_kit/leaf_node.rb
+- lib/flat_kit/log_formatter.rb
 - lib/flat_kit/logger.rb
 - lib/flat_kit/merge.rb
 - lib/flat_kit/merge_tree.rb
 - lib/flat_kit/output.rb
 - lib/flat_kit/output/file.rb
 - lib/flat_kit/output/io.rb
+- lib/flat_kit/position.rb
 - lib/flat_kit/reader.rb
 - lib/flat_kit/record.rb
 - lib/flat_kit/sentinel_internal_node.rb
 - lib/flat_kit/sentinel_leaf_node.rb
 - lib/flat_kit/sort.rb
+- lib/flat_kit/stat_type.rb
+- lib/flat_kit/stat_type/nominal_stats.rb
+- lib/flat_kit/stat_type/numerical_stats.rb
+- lib/flat_kit/stat_type/ordinal_stats.rb
+- lib/flat_kit/stats.rb
 - lib/flat_kit/writer.rb
 - lib/flat_kit/xsv.rb
 - lib/flat_kit/xsv/format.rb
 - lib/flat_kit/xsv/reader.rb
 - lib/flat_kit/xsv/record.rb
 - lib/flat_kit/xsv/writer.rb
-- tasks/default.rake
-- tasks/extension.rake
-- tasks/man.rake
-- tasks/this.rb
-- test/device_dataset.rb
-- test/input/test_file.rb
-- test/input/test_io.rb
-- test/jsonl/test_format.rb
-- test/jsonl/test_reader.rb
-- test/jsonl/test_record.rb
-- test/jsonl/test_writer.rb
-- test/output/test_file.rb
-- test/output/test_io.rb
-- test/test_conversions.rb
-- test/test_event_emitter.rb
-- test/test_format.rb
-- test/test_helper.rb
-- test/test_merge.rb
-- test/test_merge_tree.rb
-- test/test_version.rb
-- test/xsv/test_format.rb
-- test/xsv/test_reader.rb
-- test/xsv/test_record.rb
-- test/xsv/test_writer.rb
 homepage: http://github.com/copiousfreetime/flat_kit
 licenses:
 - MIT
-metadata: {}
+metadata:
+  bug_tracker_uri: https://github.com/copiousfreetime/flat_kit/issues
+  changelog_uri: https://github.com/copiousfreetime/flat_kit/blob/master/HISTORY.md
+  homepage_uri: https://github.com/copiousfreetime/flat_kit
+  source_code_uri: https://github.com/copiousfreetime/flat_kit
+  label: flat_kit
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options:
 - "--main"
@@ -205,36 +150,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: A library and commandline program for reading, writing, indexing, sorting,
   and merging CSV, TSV, JSON and other flat-file formats.
-test_files:
-- test/device_dataset.rb
-- test/input/test_file.rb
-- test/input/test_io.rb
-- test/jsonl/test_format.rb
-- test/jsonl/test_reader.rb
-- test/jsonl/test_record.rb
-- test/jsonl/test_writer.rb
-- test/output/test_file.rb
-- test/output/test_io.rb
-- test/test_conversions.rb
-- test/test_event_emitter.rb
-- test/test_format.rb
-- test/test_helper.rb
-- test/test_merge.rb
-- test/test_merge_tree.rb
-- test/test_version.rb
-- test/xsv/test_format.rb
-- test/xsv/test_reader.rb
-- test/xsv/test_record.rb
-- test/xsv/test_writer.rb
+test_files: []

data/Rakefile

@@ -1,20 +0,0 @@
-# vim: syntax=ruby
-load 'tasks/this.rb'
-
-This.name     = "flat_kit"
-This.author   = "Jeremy Hinegardner"
-This.email    = "jeremy@copiousfreetime.org"
-This.homepage = "http://github.com/copiousfreetime/#{ This.name }"
-
-This.ruby_gemspec do |spec|
-  spec.add_dependency('oj', '~> 3.0')
-  spec.add_dependency('optimist', '~> 3.0')
-
-  spec.add_development_dependency( 'rake'     , '~> 13.0')
-  spec.add_development_dependency( 'minitest' , '~> 5.11' )
-  spec.add_development_dependency( 'minitest-focus' , '~> 1.2' )
-  spec.add_development_dependency( 'rdoc'     , '~> 6.3' )
-  spec.add_development_dependency( 'simplecov', '~> 0.21')
-end
-
-load 'tasks/default.rake'

data/tasks/default.rake

@@ -1,242 +0,0 @@
-# vim: syntax=ruby
-require 'rake/clean'
-require 'digest'
-#------------------------------------------------------------------------------
-# If you want to Develop on this project just run 'rake develop' and you'll
-# have all you need to get going. If you want to use bundler for development,
-# then run 'rake develop:using_bundler'
-#------------------------------------------------------------------------------
-namespace :develop do
-
-  # Install all the development and runtime dependencies of this gem using the
-  # gemspec.
-  task :default => 'Gemfile' do
-    require 'rubygems/dependency_installer'
-    installer = ::Gem::DependencyInstaller.new
-    puts "Installing bundler..."
-    installer.install 'bundler'
-    sh 'bundle install'
-    puts "\n\nNow run 'rake test'"
-  end
-
-  # Create a Gemfile that just references the gemspec
-  file 'Gemfile' => :gemspec do
-    File.open( "Gemfile", "w+" ) do |f|
-      f.puts "# DO NOT EDIT - This file is automatically generated"
-      f.puts "# Make changes to Manifest.txt and/or Rakefile and regenerate"
-      f.puts 'source "https://rubygems.org"'
-      f.puts 'gemspec'
-    end
-  end
-end
-desc "Bootstrap development"
-task :develop => "develop:default"
-
-#------------------------------------------------------------------------------
-# Minitest - standard TestTask
-#------------------------------------------------------------------------------
-begin
-  require 'rake/testtask'
-  Rake::TestTask.new( :test ) do |t|
-    t.ruby_opts    = %w[ -w ]
-    t.libs         = %w[ lib spec test ]
-    t.pattern      = "{test,spec}/**/{test_*,*_spec}.rb"
-  end
-
-  task :test_requirements
-  task :test => :test_requirements
-  task :default => :test
-rescue LoadError
-  This.task_warning( 'test' )
-end
-
-#------------------------------------------------------------------------------
-# RDoc - standard rdoc rake task, although we must make sure to use a more
-#        recent version of rdoc since it is the one that has 'tomdoc' markup
-#------------------------------------------------------------------------------
-begin
-  gem 'rdoc' # otherwise we get the wrong task from stdlib
-  require 'rdoc/task'
-  RDoc::Task.new do |t|
-    t.markup   = 'tomdoc'
-    t.rdoc_dir = 'doc'
-    t.main     = 'README.md'
-    t.title    = "#{This.name} #{This.version}"
-    t.rdoc_files.include( FileList['*.{rdoc,md,txt}'], FileList['ext/**/*.c'],
-                          FileList['lib/**/*.rb'] )
-  end
-rescue StandardError, LoadError
-  This.task_warning( 'rdoc' )
-end
-
-#------------------------------------------------------------------------------
-# Coverage - optional code coverage, rcov for 1.8 and simplecov for 1.9, so
-#            for the moment only rcov is listed.
-#------------------------------------------------------------------------------
-begin
-  require 'simplecov'
-  desc 'Run tests with code coverage'
-  task :coverage do
-    ENV['COVERAGE'] = 'true'
-    Rake::Task[:test].execute
-  end
-  CLOBBER << 'coverage' if File.directory?( 'coverage' )
-rescue LoadError
-  This.task_warning( 'simplecov' )
-end
-
-#------------------------------------------------------------------------------
-# Manifest - We want an explicit list of thos files that are to be packaged in
-#            the gem. Most of this is from Hoe.
-#------------------------------------------------------------------------------
-namespace 'manifest' do
-  desc "Check the manifest"
-  task :check => :clean do
-    files = FileList["**/*", ".*"].exclude( This.exclude_from_manifest ).to_a.sort
-    files = files.select{ |f| File.file?( f ) }
-
-    tmp = "Manifest.tmp"
-    File.open( tmp, 'w' ) do |f|
-      f.puts files.join("\n")
-    end
-
-    begin
-      sh "diff -du Manifest.txt #{tmp}"
-    ensure
-      rm tmp
-    end
-    puts "Manifest looks good"
-  end
-
-  desc "Generate the manifest"
-  task :generate => :clean do
-    files = %x[ git ls-files ].split("\n").sort
-    files.reject! { |f| f =~ This.exclude_from_manifest }
-    File.open( "Manifest.txt", "w" ) do |f|
-      f.puts files.join("\n")
-    end
-  end
-end
-
-#------------------------------------------------------------------------------
-# Fixme - look for fixmes and report them
-#------------------------------------------------------------------------------
-namespace :fixme do
-  task :default => 'manifest:check' do
-    This.manifest.each do |file|
-      next if file == __FILE__
-      next unless file =~ %r/(txt|rb|md|rdoc|css|html|xml|css)\Z/
-      puts "FIXME: Rename #{file}" if file =~ /fixme/i
-      IO.readlines( file ).each_with_index do |line, idx|
-        prefix = "FIXME: #{file}:#{idx+1}".ljust(42)
-        puts "#{prefix} => #{line.strip}" if line =~ /fixme/i
-      end
-    end
-  end
-
-  def fixme_project_root
-    This.project_path( '../fixme' )
-  end
-
-  def fixme_project_path( subtree )
-    fixme_project_root.join( subtree )
-  end
-
-  def local_fixme_files
-    This.manifest.select { |p| p =~ %r|^tasks/| }
-  end
-
-  def outdated_fixme_files
-    local_fixme_files.select do |local|
-      upstream     = fixme_project_path( local )
-      upstream.exist? &&
-        ( Digest::SHA256.file( local ) != Digest::SHA256.file( upstream ) )
-    end
-  end
-
-  def fixme_up_to_date?
-    outdated_fixme_files.empty?
-  end
-
-  desc "See if the fixme tools are outdated"
-  task :outdated do
-    if fixme_up_to_date? then
-      puts "Fixme files are up to date."
-    else
-      outdated_fixme_files.each do |f|
-        puts "#{f} is outdated"
-      end
-    end
-  end
-
-  desc "Update outdated fixme files"
-  task :update do
-    if fixme_up_to_date? then
-      puts "Fixme files are already up to date."
-    else
-      puts "Updating fixme files:"
-      outdated_fixme_files.each do |local|
-        upstream = fixme_project_path( local )
-        puts "  * #{local}"
-        FileUtils.cp( upstream, local )
-      end
-      puts "Use your git commands as appropriate."
-    end
-  end
-end
-desc "Look for fixmes and report them"
-task :fixme => "fixme:default"
-
-#------------------------------------------------------------------------------
-# Gem Specification
-#------------------------------------------------------------------------------
-# Really this is only here to support those who use bundler
-desc "Build the #{This.name}.gemspec file"
-task :gemspec do
-  File.open( This.gemspec_file, "wb+" ) do |f|
-    f.puts "# DO NOT EDIT - This file is automatically generated"
-    f.puts "# Make changes to Manifest.txt and/or Rakefile and regenerate"
-    f.write This.platform_gemspec.to_ruby
-  end
-end
-
-# .rbc files from ruby 2.0
-CLOBBER << FileList["**/*.rbc"]
-
-# The standard gem packaging task, everyone has it.
-require 'rubygems/package_task'
-::Gem::PackageTask.new( This.platform_gemspec ) do
-  # nothing
-end
-
-#------------------------------------------------------------------------------
-# Release - the steps we go through to do a final release, this is pulled from
-#           a compbination of mojombo's rakegem, hoe and hoe-git
-#
-# 1) make sure we are on the main branch
-# 2) make sure there are no uncommitted items
-# 3) check the manifest and make sure all looks good
-# 4) build the gem
-# 5) do an empty commit to have the commit message of the version
-# 6) tag that commit as the version
-# 7) push main
-# 8) push the tag
-# 7) pus the gem
-#------------------------------------------------------------------------------
-task :release_check do
-  unless `git branch` =~ /^\* main/
-    abort "You must be on the main branch to release!"
-  end
-  unless `git status` =~ /^nothing to commit/m
-    abort "Nope, sorry, you have unfinished business"
-  end
-end
-
-desc "Create tag v#{This.version}, build and push #{This.platform_gemspec.full_name} to rubygems.org"
-task :release => [ :release_check, 'manifest:check', :gem ] do
-  sh "git commit --allow-empty -a -m 'Release #{This.version}'"
-  sh "git tag -a -m 'v#{This.version}' v#{This.version}"
-  sh "git push origin main"
-  sh "git push origin v#{This.version}"
-  sh "gem push pkg/#{This.platform_gemspec.full_name}.gem"
-end