flat_kit 0.1.0

data/lib/flat_kit/xsv.rb

@@ -0,0 +1,8 @@
+module FlatKit
+  module Xsv
+  end
+end
+require 'flat_kit/xsv/record'
+require 'flat_kit/xsv/reader'
+require 'flat_kit/xsv/writer'
+require 'flat_kit/xsv/format'

data/lib/flat_kit/xsv/format.rb

@@ -0,0 +1,25 @@
+module FlatKit
+  module Xsv
+    class Format < ::FlatKit::Format
+      def self.format_name
+        "xsv"
+      end
+
+      def self.handles?(filename)
+        parts = filename.split(".")
+        %w[ csv tsv txt ].each do |ext|
+          return true if parts.include?(ext)
+        end
+        return false
+      end
+
+      def self.reader
+        ::FlatKit::Xsv::Reader
+      end
+
+      def self.writer
+        ::FlatKit::Xsv::Writer
+      end
+    end
+  end
+end

data/lib/flat_kit/xsv/reader.rb

@@ -0,0 +1,45 @@
+require 'csv'
+
+module FlatKit
+  module Xsv
+    class Reader < ::FlatKit::Reader
+      attr_reader :input
+      attr_reader :count
+      attr_reader :fields
+
+      def self.format_name
+        ::FlatKit::Xsv::Format.format_name
+      end
+
+      def self.default_csv_options
+        {
+          headers: :first_row,
+          converters: :numeric,
+          return_headers: false
+        }
+      end
+
+      def initialize(source:, compare_fields: :none, **csv_options)
+        super(source: source, compare_fields: compare_fields)
+        @input = ::FlatKit::Input.from(source)
+        @count = 0
+        @csv_options = Reader.default_csv_options.merge(csv_options)
+        @fields = nil
+        @csv = CSV.new(input.io, **@csv_options)
+      end
+
+      def each
+        @csv.each do |row|
+          @fields = row.headers if @fields.nil?
+          record = ::FlatKit::Xsv::Record.new(data: row, compare_fields: compare_fields)
+          @count += 1
+          yield record
+        end
+        input.close
+      rescue => e
+        ::FlatKit.logger.error "Error reading xsv records from #{input.name}: #{e}"
+        raise ::FlatKit::Error, e
+      end
+    end
+  end
+end

data/lib/flat_kit/xsv/record.rb

@@ -0,0 +1,90 @@
+require 'csv'
+require 'flat_kit/record'
+
+module FlatKit
+  module Xsv
+    class Record < ::FlatKit::Record
+      attr_reader :ordered_fields
+
+      def self.format_name
+        ::FlatKit::Xsv::Format.format_name
+      end
+
+      def self.from_record(record, ordered_fields: nil)
+        if record.instance_of?(FlatKit::Xsv::Record) then
+          new(data: record.data, compare_fields: record.compare_fields)
+        else
+          new(data: nil, compare_fields: record.compare_fields,
+              ordered_fields: nil,
+              complete_structured_data: record.to_hash)
+        end
+      end
+
+      def initialize(data:, compare_fields: :none,
+                     ordered_fields: :auto,
+                     complete_structured_data: nil)
+        super(data: data, compare_fields: compare_fields)
+
+        @complete_structured_data = complete_structured_data
+        @ordered_fields = ordered_fields
+
+        if data.nil? && (complete_structured_data.nil? || complete_structured_data.empty?) then
+          raise FlatKit::Error,
+            "#{self.class} requires initialization from data: or complete_structured_data:"
+        end
+
+        resolve_ordered_fields
+      end
+
+      def [](key)
+        return nil unless @compare_fields.include?(key)
+        if data.nil? && !@complete_structured_data.nil? then
+          @complete_structured_data[key]
+        else
+          data[key]
+        end
+      end
+
+      def complete_structured_data
+        @complete_structured_data ||= data.to_hash
+      end
+      alias to_hash complete_structured_data
+
+      def to_a
+        return data.fields unless data.nil?
+
+        Array.new.tap do |a|
+          @ordered_fields.each do |field|
+            a << @complete_structured_data[field]
+          end
+        end
+      end
+
+      # convert to a csv line,
+      #
+      # First we use data if it is there since that should be a CSV::Row
+      #
+      # Next, if that doesn't work - iterate over the ordered fields and use the
+      # yield the values from complete_structured_data in that order
+      #
+      # And finally, if that doesn'twork, then just use complete structured data
+      # values in that order.
+      def to_s
+        return data.to_csv unless data.nil?
+        CSV.generate_line(to_a)
+      end
+
+      private
+
+      def resolve_ordered_fields
+        if (@ordered_fields == :auto) || (@ordered_fields.nil? || @ordered_fields.empty?) then
+          if @data.nil? || @data.empty? then
+            @ordered_fields = complete_structured_data.keys
+          else
+            @ordered_fields = @data.headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/xsv/writer.rb

@@ -0,0 +1,70 @@
+module FlatKit
+  module Xsv
+    class Writer < ::FlatKit::Writer
+      attr_reader :output
+      attr_reader :count
+      attr_reader :fields
+
+      def self.format_name
+        ::FlatKit::Xsv::Format.format_name
+      end
+
+      def self.default_csv_options
+        {
+          headers: nil,
+          write_headers: true
+        }
+      end
+
+      def initialize(destination:, fields: :auto, **csv_options)
+        super(destination: destination)
+        @fields = fields
+        @output = ::FlatKit::Output.from(@destination)
+        @count = 0
+        @we_write_the_header = nil
+        @csv_options = Writer.default_csv_options.dup
+
+        if @fields == :auto then
+          @we_write_the_header = true
+        else
+          @csv_options.merge!(headers: fields)
+          @we_write_the_header = false
+        end
+
+        @csv_options.merge!(csv_options)
+        @csv = CSV.new(output.io, **@csv_options)
+      end
+
+      def write(record)
+        case record
+        when FlatKit::Xsv::Record
+          write_record(record)
+        when FlatKit::Record
+          converted_record = ::FlatKit::Xsv::Record.from_record(record, ordered_fields: @fields)
+          write_record(converted_record)
+        else
+          raise FlatKit::Error, "Unable to write records of type #{record.class}"
+        end
+      rescue FlatKit::Error => fe
+        raise fe
+      rescue => e
+        ::FlatKit.logger.error "Error reading jsonl records from #{output.name}: #{e}"
+        raise ::FlatKit::Error, e
+      end
+
+      def close
+        @output.close
+      end
+
+      private
+
+      def write_record(record)
+        if @we_write_the_header && @count == 0 then
+          @csv << record.ordered_fields
+        end
+        @count += 1
+        @csv << record.to_a
+      end
+    end
+  end
+end

data/tasks/default.rake

@@ -0,0 +1,242 @@
+# vim: syntax=ruby
+require 'rake/clean'
+require 'digest'
+#------------------------------------------------------------------------------
+# If you want to Develop on this project just run 'rake develop' and you'll
+# have all you need to get going. If you want to use bundler for development,
+# then run 'rake develop:using_bundler'
+#------------------------------------------------------------------------------
+namespace :develop do
+
+  # Install all the development and runtime dependencies of this gem using the
+  # gemspec.
+  task :default => 'Gemfile' do
+    require 'rubygems/dependency_installer'
+    installer = ::Gem::DependencyInstaller.new
+    puts "Installing bundler..."
+    installer.install 'bundler'
+    sh 'bundle install'
+    puts "\n\nNow run 'rake test'"
+  end
+
+  # Create a Gemfile that just references the gemspec
+  file 'Gemfile' => :gemspec do
+    File.open( "Gemfile", "w+" ) do |f|
+      f.puts "# DO NOT EDIT - This file is automatically generated"
+      f.puts "# Make changes to Manifest.txt and/or Rakefile and regenerate"
+      f.puts 'source "https://rubygems.org"'
+      f.puts 'gemspec'
+    end
+  end
+end
+desc "Bootstrap development"
+task :develop => "develop:default"
+
+#------------------------------------------------------------------------------
+# Minitest - standard TestTask
+#------------------------------------------------------------------------------
+begin
+  require 'rake/testtask'
+  Rake::TestTask.new( :test ) do |t|
+    t.ruby_opts    = %w[ -w ]
+    t.libs         = %w[ lib spec test ]
+    t.pattern      = "{test,spec}/**/{test_*,*_spec}.rb"
+  end
+
+  task :test_requirements
+  task :test => :test_requirements
+  task :default => :test
+rescue LoadError
+  This.task_warning( 'test' )
+end
+
+#------------------------------------------------------------------------------
+# RDoc - standard rdoc rake task, although we must make sure to use a more
+#        recent version of rdoc since it is the one that has 'tomdoc' markup
+#------------------------------------------------------------------------------
+begin
+  gem 'rdoc' # otherwise we get the wrong task from stdlib
+  require 'rdoc/task'
+  RDoc::Task.new do |t|
+    t.markup   = 'tomdoc'
+    t.rdoc_dir = 'doc'
+    t.main     = 'README.md'
+    t.title    = "#{This.name} #{This.version}"
+    t.rdoc_files.include( FileList['*.{rdoc,md,txt}'], FileList['ext/**/*.c'],
+                          FileList['lib/**/*.rb'] )
+  end
+rescue StandardError, LoadError
+  This.task_warning( 'rdoc' )
+end
+
+#------------------------------------------------------------------------------
+# Coverage - optional code coverage, rcov for 1.8 and simplecov for 1.9, so
+#            for the moment only rcov is listed.
+#------------------------------------------------------------------------------
+begin
+  require 'simplecov'
+  desc 'Run tests with code coverage'
+  task :coverage do
+    ENV['COVERAGE'] = 'true'
+    Rake::Task[:test].execute
+  end
+  CLOBBER << 'coverage' if File.directory?( 'coverage' )
+rescue LoadError
+  This.task_warning( 'simplecov' )
+end
+
+#------------------------------------------------------------------------------
+# Manifest - We want an explicit list of thos files that are to be packaged in
+#            the gem. Most of this is from Hoe.
+#------------------------------------------------------------------------------
+namespace 'manifest' do
+  desc "Check the manifest"
+  task :check => :clean do
+    files = FileList["**/*", ".*"].exclude( This.exclude_from_manifest ).to_a.sort
+    files = files.select{ |f| File.file?( f ) }
+
+    tmp = "Manifest.tmp"
+    File.open( tmp, 'w' ) do |f|
+      f.puts files.join("\n")
+    end
+
+    begin
+      sh "diff -du Manifest.txt #{tmp}"
+    ensure
+      rm tmp
+    end
+    puts "Manifest looks good"
+  end
+
+  desc "Generate the manifest"
+  task :generate => :clean do
+    files = %x[ git ls-files ].split("\n").sort
+    files.reject! { |f| f =~ This.exclude_from_manifest }
+    File.open( "Manifest.txt", "w" ) do |f|
+      f.puts files.join("\n")
+    end
+  end
+end
+
+#------------------------------------------------------------------------------
+# Fixme - look for fixmes and report them
+#------------------------------------------------------------------------------
+namespace :fixme do
+  task :default => 'manifest:check' do
+    This.manifest.each do |file|
+      next if file == __FILE__
+      next unless file =~ %r/(txt|rb|md|rdoc|css|html|xml|css)\Z/
+      puts "FIXME: Rename #{file}" if file =~ /fixme/i
+      IO.readlines( file ).each_with_index do |line, idx|
+        prefix = "FIXME: #{file}:#{idx+1}".ljust(42)
+        puts "#{prefix} => #{line.strip}" if line =~ /fixme/i
+      end
+    end
+  end
+
+  def fixme_project_root
+    This.project_path( '../fixme' )
+  end
+
+  def fixme_project_path( subtree )
+    fixme_project_root.join( subtree )
+  end
+
+  def local_fixme_files
+    This.manifest.select { |p| p =~ %r|^tasks/| }
+  end
+
+  def outdated_fixme_files
+    local_fixme_files.select do |local|
+      upstream     = fixme_project_path( local )
+      upstream.exist? &&
+        ( Digest::SHA256.file( local ) != Digest::SHA256.file( upstream ) )
+    end
+  end
+
+  def fixme_up_to_date?
+    outdated_fixme_files.empty?
+  end
+
+  desc "See if the fixme tools are outdated"
+  task :outdated do
+    if fixme_up_to_date? then
+      puts "Fixme files are up to date."
+    else
+      outdated_fixme_files.each do |f|
+        puts "#{f} is outdated"
+      end
+    end
+  end
+
+  desc "Update outdated fixme files"
+  task :update do
+    if fixme_up_to_date? then
+      puts "Fixme files are already up to date."
+    else
+      puts "Updating fixme files:"
+      outdated_fixme_files.each do |local|
+        upstream = fixme_project_path( local )
+        puts "  * #{local}"
+        FileUtils.cp( upstream, local )
+      end
+      puts "Use your git commands as appropriate."
+    end
+  end
+end
+desc "Look for fixmes and report them"
+task :fixme => "fixme:default"
+
+#------------------------------------------------------------------------------
+# Gem Specification
+#------------------------------------------------------------------------------
+# Really this is only here to support those who use bundler
+desc "Build the #{This.name}.gemspec file"
+task :gemspec do
+  File.open( This.gemspec_file, "wb+" ) do |f|
+    f.puts "# DO NOT EDIT - This file is automatically generated"
+    f.puts "# Make changes to Manifest.txt and/or Rakefile and regenerate"
+    f.write This.platform_gemspec.to_ruby
+  end
+end
+
+# .rbc files from ruby 2.0
+CLOBBER << FileList["**/*.rbc"]
+
+# The standard gem packaging task, everyone has it.
+require 'rubygems/package_task'
+::Gem::PackageTask.new( This.platform_gemspec ) do
+  # nothing
+end
+
+#------------------------------------------------------------------------------
+# Release - the steps we go through to do a final release, this is pulled from
+#           a compbination of mojombo's rakegem, hoe and hoe-git
+#
+# 1) make sure we are on the main branch
+# 2) make sure there are no uncommitted items
+# 3) check the manifest and make sure all looks good
+# 4) build the gem
+# 5) do an empty commit to have the commit message of the version
+# 6) tag that commit as the version
+# 7) push main
+# 8) push the tag
+# 7) pus the gem
+#------------------------------------------------------------------------------
+task :release_check do
+  unless `git branch` =~ /^\* main/
+    abort "You must be on the main branch to release!"
+  end
+  unless `git status` =~ /^nothing to commit/m
+    abort "Nope, sorry, you have unfinished business"
+  end
+end
+
+desc "Create tag v#{This.version}, build and push #{This.platform_gemspec.full_name} to rubygems.org"
+task :release => [ :release_check, 'manifest:check', :gem ] do
+  sh "git commit --allow-empty -a -m 'Release #{This.version}'"
+  sh "git tag -a -m 'v#{This.version}' v#{This.version}"
+  sh "git push origin main"
+  sh "git push origin v#{This.version}"
+  sh "gem push pkg/#{This.platform_gemspec.full_name}.gem"
+end