flat_kit 0.2.0 → 1.0.0

data/tasks/extension.rake

@@ -1,38 +0,0 @@
-# To be used if the gem has extensions.
-# If this task set is inclueded then you will need to also have
-#
-#   spec.add_development_dependency( 'rake-compiler', '~> 0.8.1' )
-#
-# in your top level rakefile
-begin
-  require 'rake/extensiontask'
-  require 'rake/javaextensiontask'
-
-  if RUBY_PLATFORM == "java" then
-
-    Rake::JavaExtensionTask.new( This.name) do |ext|
-      ext.ext_dir  = File.join( 'ext', This.name, "java" )
-      ext.lib_dir  = File.join( 'lib', This.name )
-      ext.gem_spec = This.java_gemspec
-    end
-
-  else
-
-    Rake::ExtensionTask.new( This.name ) do |ext|
-      ext.ext_dir   = File.join( 'ext', This.name, "c" )
-      ext.lib_dir   = File.join( 'lib', This.name )
-      ext.gem_spec  = This.ruby_gemspec
-
-      ext.cross_compile  = true                         # enable cross compilation (requires cross compile toolchain)
-      ext.cross_platform = %w[x86-mingw32 x64-mingw32]  # forces the Windows platform instead of the default one
-                                                        # configure options only for cross compile
-    end
-  end
-
-  task :test_requirements => :compile
-rescue LoadError
-  This.task_warning( 'extension' )
-end
-
-CLOBBER << FileList["lib/**/*.{jar,so,bundle}"]
-CLOBBER << FileList["lib/#{This.name}/{1,2}.*/"]

data/tasks/man.rake

@@ -1,7 +0,0 @@
-#------------------------------------------------------------------------------
-# man pages
-#------------------------------------------------------------------------------
-desc "Create the man pages"
-task :man do
-  sh "ronn --roff #{FileList["man/*.ronn"]}"
-end

data/tasks/this.rb

@@ -1,208 +0,0 @@
-require 'pathname'
-
-# Public: A Class containing all the metadata and utilities needed to manage a
-# ruby project.
-class ThisProject
-  # The name of this project
-  attr_accessor :name
-
-  # The author's name
-  attr_accessor :author
-
-  # The email address of the author(s)
-  attr_accessor :email
-
-  # The homepage of this project
-  attr_accessor :homepage
-
-  # The regex of files to exclude from the manifest
-  attr_accessor :exclude_from_manifest
-
-  # The hash of Gem::Specifications keyed' by platform
-  attr_accessor :gemspecs
-
-  # Public: Initialize ThisProject
-  #
-  # Yields self
-  def initialize(&block)
-    @exclude_from_manifest = Regexp.union(/\.(git|DS_Store)/,
-                                          /^(doc|coverage|pkg|tmp|Gemfile(\.lock)?)/,
-                                          /^[^\/]+\.gemspec/,
-                                          /\.(swp|jar|bundle|so|rvmrc|travis.yml|byebug_history|fossa.yml|ruby-version)$/,
-                                          /~$/)
-    @gemspecs              = Hash.new
-    yield self if block_given?
-  end
-
-  # Public: return the version of ThisProject
-  #
-  # Search the ruby files in the project looking for the one that has the
-  # version string in it. This does not eval any code in the project, it parses
-  # the source code looking for the string.
-  #
-  # Returns a String version
-  def version
-    [ "lib/#{ name }.rb", "lib/#{ name }/version.rb" ].each do |v|
-      path = project_path( v )
-      line = path.read[/^\s*VERSION\s*=\s*.*/]
-      if line then
-        return line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
-      end
-    end
-  end
-
-  # Internal: Return a section of an RDoc file with the given section name
-  #
-  # path         - the relative path in the project of the file to parse
-  # section_name - the section out of the file from which to parse data
-  #
-  # Retuns the text of the section as an array of paragrphs.
-  def section_of( file, section_name )
-    re    = /^[=#]+ (.*)$/
-    sectional = project_path( file )
-    parts = sectional.read.split( re )[1..-1]
-    parts.map! { |p| p.strip }
-
-    sections = Hash.new
-    Hash[*parts].each do |k,v|
-      sections[k] = v.split("\n\n")
-    end
-    return sections[section_name]
-  end
-
-  # Internal: print out a warning about the give task
-  def task_warning( task )
-    warn "WARNING: '#{task}' tasks are not defined. Please run 'rake develop'"
-  end
-
-  # Internal: Return the full path to the file that is relative to the project
-  # root.
-  #
-  # path - the relative path of the file from the project root
-  #
-  # Returns the Pathname of the file
-  def project_path( *relative_path )
-    project_root.join( *relative_path )
-  end
-
-  # Internal: The absolute path of this file
-  #
-  # Returns the Pathname of this file.
-  def this_file_path
-    Pathname.new( __FILE__ ).expand_path
-  end
-
-  # Internal: The root directory of this project
-  #
-  # This is defined as being the directory that is in the path of this project
-  # that has the first Rakefile
-  #
-  # Returns the Pathname of the directory
-  def project_root
-    this_file_path.ascend do |p|
-      rakefile = p.join( 'Rakefile' )
-      return p if rakefile.exist?
-    end
-  end
-
-  # Internal: Returns the contents of the Manifest.txt file as an array
-  #
-  # Returns an Array of strings
-  def manifest
-    manifest_file = project_path( "Manifest.txt" )
-    abort "You need a Manifest.txt" unless manifest_file.readable?
-    manifest_file.readlines.map { |l| l.strip }
-  end
-
-  # Internal: Return the files that define the extensions
-  #
-  # Returns an Array
-  def extension_conf_files
-    manifest.grep( /extconf.rb\Z/ )
-  end
-
-  # Internal: Returns the gemspace associated with the current ruby platform
-  def platform_gemspec
-    gemspecs.fetch(platform) { This.ruby_gemspec }
-  end
-
-  def core_gemspec
-    Gem::Specification.new do |spec|
-      spec.name        = name
-      spec.version     = version
-      spec.author      = author
-      spec.email       = email
-      spec.homepage    = homepage
-
-      spec.summary     = summary
-      spec.description = description
-      spec.license     = license
-
-      spec.files       = manifest
-      spec.executables = spec.files.grep(/^bin/) { |f| File.basename(f) }
-      spec.test_files  = spec.files.grep(/^(spec|test)/)
-
-      spec.extra_rdoc_files += spec.files.grep(/(txt|rdoc|md)$/)
-      spec.rdoc_options = [ "--main"  , 'README.md',
-                            "--markup", "tomdoc" ]
-
-      spec.required_ruby_version = '>= 2.2.2'
-    end
-  end
-
-  # Internal: Return the gemspec for the ruby platform
-  def ruby_gemspec( core = core_gemspec, &block )
-    yielding_gemspec( 'ruby', core, &block )
-  end
-
-  # Internal: Return the gemspec for the jruby platform
-  def java_gemspec( core = core_gemspec, &block )
-    yielding_gemspec( 'java', core, &block )
-  end
-
-  # Internal: give an initial spec and a key, create a new gemspec based off of
-  # it.
-  #
-  # This will force the new gemspecs 'platform' to be that of the key, since the
-  # only reason you would have multiple gemspecs at this point is to deal with
-  # different platforms.
-  def yielding_gemspec( key, core )
-    spec = gemspecs[key] ||= core.dup
-    spec.platform = key
-    yield spec if block_given?
-    return spec
-  end
-
-  # Internal: Return the platform of ThisProject at the current moment in time.
-  def platform
-    (RUBY_PLATFORM == "java") ? 'java' : Gem::Platform::RUBY
-  end
-
-  # Internal: Return the DESCRIPTION section of the README.rdoc file
-  def description_section
-    section_of( 'README.md', 'DESCRIPTION')
-  end
-
-  # Internal: Return the summary text from the README
-  def summary
-    description_section.first
-  end
-
-  # Internal: Return the full description text from the README
-  def description
-    description_section.join(" ").tr("\n", ' ').gsub(/[{}]/,'').gsub(/\[[^\]]+\]/,'') # strip rdoc
-  end
-
-  def license
-    license_file = project_path("LICENSE.txt")
-    line = license_file.readlines.first
-    line.split(/\s+/).first
-  end
-
-  # Internal: The path to the gemspec file
-  def gemspec_file
-    project_path( "#{ name }.gemspec" )
-  end
-end
-
-This = ThisProject.new

data/test/device_dataset.rb

@@ -1,117 +0,0 @@
-class DeviceDataset
-
-  include TestHelper
-
-  attr_reader :count
-  attr_reader :compare_fields
-  attr_reader :fields
-
-  attr_reader :filename_jsonl
-  attr_reader :filename_sorted_jsonl
-  attr_reader :filename_csv
-  attr_reader :filename_sorted_csv
-
-  def initialize(count:, compare_fields: [ "manufacturer", "model_name", "slug" ])
-    @count = count
-    @compare_fields = compare_fields
-    @fields = %w[
-      build_number
-      manufacturer
-      model_name
-      platform
-      serial
-      slug
-    ]
-    @filename_sorted_jsonl = nil
-    @filename_jsonl = nil
-    @filename_sorted_csv = nil
-    @filename_csv = nil
-    @slug = generate_slug
-  end
-
-  def persist_records_as_jsonl
-    @filename_jsonl = scratch_file(prefix: "unsorted_", slug: @slug)
-    @filename_jsonl.open("w+") do |f|
-      f.write(records_as_jsonl)
-    end
-  end
-
-  def persist_sorted_records_as_jsonl
-    @filename_sorted_jsonl = scratch_file(prefix: "sorted_", slug: @slug)
-    @filename_sorted_jsonl.open("w+") do |f|
-      f.write(sorted_records_as_jsonl)
-    end
-  end
-
-  def cleanup_files
-    [ @filename_sorted_jsonl, @filename_jsonl, @filename_sorted_csv, @filename_csv ].each do |p|
-      next if p.nil?
-      p.unlink if p.exist?
-    end
-  end
-
-  def records
-    @records ||= Array.new.tap do |a|
-      count.times do
-        a << Hash.new.tap do |h|
-          fields.each do |f|
-            value = (f == 'slug') ? generate_slug : ::Faker::Device.send(f)
-            h[f] = value
-          end
-        end
-      end
-    end
-  end
-
-  def sorted_records
-    @sorted_records ||= records.sort_by do |r|
-      compare_fields.map { |field| r[field] }
-    end
-  end
-
-  def records_as_jsonl
-    @jsonl_records ||= as_jsonl(list: records)
-  end
-
-  def records_as_csv
-    @csv_records ||= as_csv(list: records)
-  end
-
-  def records_as_csv_rows
-    @csv_rows ||= as_csv_rows(records_as_csv)
-  end
-
-  def sorted_records_as_jsonl
-    @jsonl_sorted_records ||= as_jsonl(list: sorted_records)
-  end
-
-  def sorted_records_as_csv
-    @csv_sorted_records ||= as_csv(list: sorted_records)
-  end
-
-  def sorted_records_as_csv_rows
-    @csv_sorted_rows ||= as_csv_rows(sorted_records_as_csv)
-  end
-
-  private
-
-  def as_jsonl(list:)
-    list.map { |r| Oj.dump(r) }.join("\n") + "\n"
-  end
-
-  def as_csv(list:, headers: fields)
-    CSV.generate('', headers: headers , write_headers: true) do |csv|
-      list.each do |r|
-        csv << fields.map { |f| r[f] }
-      end
-    end
-  end
-
-  def as_csv_rows(text)
-    Array.new.tap do |a|
-      CSV.new(text, converters: :numeric, headers: :first_row, return_headers: false).each do |row|
-        a << row
-      end
-    end
-  end
-end

data/test/input/test_file.rb

@@ -1,73 +0,0 @@
-require_relative '../test_helper'
-
-module TestInput
-  class TestFile < ::Minitest::Test
-    def test_does_not_handle_stdin_text
-      ::FlatKit::Input::IO::STDINS.each do |e|
-        refute(::FlatKit::Input::File.handles?(e), "#{e} is not stdin text")
-      end
-    end
-
-    def test_handles_existing_file
-      test_path = "tmp/test_handles_existing_file.txt"
-      begin
-        IO.write(test_path,"test handles existing file")
-        assert(::FlatKit::Input::File.handles?(test_path))
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_only_handles_string
-      refute(::FlatKit::Input::File.handles?(Object.new))
-    end
-
-    def test_raises_error_if_not_readable
-      assert_raises(FlatKit::Error) { ::FlatKit::Input::File.new("tmp/does-not-exist") }
-    end
-
-    def test_init_from_path
-      test_path = "tmp/test_init_from_path.txt"
-      begin
-        IO.write(test_path,"nothing to see here")
-        io = ::FlatKit::Input::File.new(test_path)
-        assert_equal(test_path, io.name)
-        assert_instance_of(::File, io.io)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_reads_from_file
-      test_path = "tmp/test_reads_from_file.txt"
-      begin
-        text = "test_reads_from_file"
-        IO.write(test_path,text)
-
-        input = ::FlatKit::Input::File.new(test_path)
-        content = input.io.read
-        assert_equal(text, content)
-
-        input.close
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_reads_from_gzfile
-      test_path = "tmp/test_reads_from_gzfile.txt.gz"
-      begin
-        text = "this is something to read"
-        system("echo '#{text}' | gzip > #{test_path}")
-
-        input = ::FlatKit::Input::File.new(test_path)
-        content = input.io.read
-        assert_equal(text + "\n", content)
-
-        input.close
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-  end
-end

data/test/input/test_io.rb

@@ -1,93 +0,0 @@
-require_relative '../test_helper'
-
-module TestInput
-  class NullIO < ::IO
-    def initialize()
-    end
-  end
-
-  class TestIO < ::Minitest::Test
-
-    def test_handles_stdin_text
-      ::FlatKit::Input::IO::STDINS.each do |e|
-        assert(::FlatKit::Input::IO.handles?(e), "#{e} is not stdin text")
-      end
-    end
-
-    def test_handles_stdin_io
-      x = $stdin
-      assert(::FlatKit::Input::IO.handles?(x), "is not stdin")
-    end
-
-    def test_handles_stringio
-      assert(::FlatKit::Input::IO.handles?(StringIO.new))
-    end
-
-    def test_does_not_handle_other
-      x = Object.new
-      refute(::FlatKit::Input::IO.handles?(x))
-    end
-
-    def test_init_from_dash
-      io = ::FlatKit::Input::IO.new("-")
-      assert_equal("<STDIN>", io.name)
-      assert_equal(::STDIN, io.io)
-    end
-
-    def test_init_from_file_object
-      test_path = "tmp/test_init_from_file_object.txt"
-      begin
-        File.open(test_path, "w+") do |f|
-          io = ::FlatKit::Input::IO.new(f)
-          assert_equal(test_path, io.name)
-          assert_instance_of(::File, io.io)
-        end
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_init_from_string_io_object
-      sio = StringIO.new
-      io = ::FlatKit::Input::IO.new(sio)
-      assert_match(/StringIO/, io.name)
-      assert_instance_of(::StringIO, io.io)
-    end
-
-    def test_init_from_io_object
-      null_io = NullIO.new
-      io = ::FlatKit::Input::IO.new(null_io)
-      assert_match(/NullIO/, io.name)
-      assert_instance_of(::TestInput::NullIO, io.io)
-    end
-
-    def test_init_from_stdin
-      io  = ::FlatKit::Input::IO.new($stdin)
-      assert_equal("<STDIN>", io.name)
-      assert_equal(::STDIN, io.io)
-    end
-
-    def test_init_from_invalid
-      assert_raises(::FlatKit::Error) { ::FlatKit::Input::IO.new(Object.new) }
-    end
-
-    def test_reads_from_io
-      test_path = "tmp/test_reads_from_io.txt"
-      begin
-        line = "This is a line to read"
-        File.open(test_path, "w+") do |f|
-          f.write(line)
-          f.rewind
-
-          io = ::FlatKit::Input::IO.new(f)
-          content = io.io.read
-          io.close
-          assert_equal(content, line)
-        end
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-  end
-end

data/test/jsonl/test_format.rb

@@ -1,22 +0,0 @@
-require_relative '../test_helper'
-
-module TestJsonl
-  class TestFormat < ::Minitest::Test
-
-    def test_handles_json
-      assert(::FlatKit::Jsonl::Format.handles?("data.json.gz"))
-    end
-
-    def test_handles_jsonl
-      assert(::FlatKit::Jsonl::Format.handles?("data.jsonl"))
-    end
-
-    def test_handles_ndjson
-      assert(::FlatKit::Jsonl::Format.handles?("log.ndjson"))
-    end
-
-    def test_does_not_handle_csv
-      refute(::FlatKit::Jsonl::Format.handles?("data.csv"))
-    end
-  end
-end

data/test/jsonl/test_reader.rb

@@ -1,49 +0,0 @@
-require_relative '../test_helper'
-
-module TestJsonl
-  class TestReader < ::Minitest::Test
-    def setup
-      @count = 20
-      @dataset = DeviceDataset.new(count: @count)
-      @compare_fields = @dataset.compare_fields
-      @test_path = "tmp/test_reads_from_io.jsonl"
-
-      File.open(@test_path, "wb") do |f|
-        f.write(@dataset.records_as_jsonl)
-      end
-    end
-
-    def teardown
-      File.unlink(@test_path) if File.exist?(@test_path)
-    end
-
-    def test_raises_error_on_invalid_source
-      assert_raises(::FlatKit::Error) {
-        ::FlatKit::Jsonl::Reader.new(source: Object.new, compare_fields: nil)
-      }
-    end
-
-    def test_reads_from_pathname
-      reader = ::FlatKit::Jsonl::Reader.new(source: @test_path, compare_fields: @compare_fields)
-      all = reader.to_a
-      assert_equal(@count, reader.count)
-      assert_equal(@count, all.size)
-    end
-
-    def test_reads_from_io
-      File.open(@test_path) do |f|
-        reader = ::FlatKit::Jsonl::Reader.new(source: f, compare_fields: @compare_fields)
-        all = reader.to_a
-        assert_equal(@count, reader.count)
-        assert_equal(@count, all.size)
-      end
-    end
-
-    def test_raises_error_on_io_error
-      s = StringIO.new
-      s.close_read
-      reader = ::FlatKit::Jsonl::Reader.new(source: s, compare_fields: @compare_fields)
-      assert_raises(::FlatKit::Error) { reader.to_a }
-    end
-  end
-end

data/test/jsonl/test_record.rb

@@ -1,61 +0,0 @@
-require_relative '../test_helper'
-require 'faker'
-require 'byebug'
-
-module TestJsonl
-  class TestRecord < ::Minitest::Test
-    def setup
-      @one_row_dataset = DeviceDataset.new(count: 1)
-      @src_record = @one_row_dataset.records.first
-      @src_json = JSON.generate(@src_record)
-      @compare_fields = @one_row_dataset.compare_fields
-    end
-
-    def test_initializes_from_data
-      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-      @compare_fields.each do |k|
-        assert_equal(@src_record[k], record[k])
-      end
-    end
-
-    def test_ignores_non_compare_fields_values
-      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-
-      refute(record["version"])
-    end
-
-    def test_is_sortable
-      dataset = DeviceDataset.new(count: 20)
-      fk_records = Array.new.tap do |a|
-        dataset.records.each do |r|
-          data = JSON.generate(r)
-          record = FlatKit::Jsonl::Record.new(data: data, compare_fields: @compare_fields)
-          a << record
-        end
-      end
-
-      sorted = fk_records.sort
-
-      sio = StringIO.new
-      sorted.each do |r|
-        sio.puts(r.to_s)
-      end
-
-      sorted_string = sio.string
-      assert_equal(dataset.sorted_records_as_jsonl, sorted_string)
-    end
-
-    def test_converts_to_hash
-      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-      h      = record.to_hash
-
-      assert_equal(@src_record, h)
-    end
-
-    def test_converts_from_record
-      rec1 = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-      rec2 = FlatKit::Jsonl::Record.from_record(rec1)
-      assert_equal(rec1, rec2)
-    end
-  end
-end