flat_kit 0.1.0

data/tasks/extension.rake

@@ -0,0 +1,38 @@
+# To be used if the gem has extensions.
+# If this task set is inclueded then you will need to also have
+#
+#   spec.add_development_dependency( 'rake-compiler', '~> 0.8.1' )
+#
+# in your top level rakefile
+begin
+  require 'rake/extensiontask'
+  require 'rake/javaextensiontask'
+
+  if RUBY_PLATFORM == "java" then
+
+    Rake::JavaExtensionTask.new( This.name) do |ext|
+      ext.ext_dir  = File.join( 'ext', This.name, "java" )
+      ext.lib_dir  = File.join( 'lib', This.name )
+      ext.gem_spec = This.java_gemspec
+    end
+
+  else
+
+    Rake::ExtensionTask.new( This.name ) do |ext|
+      ext.ext_dir   = File.join( 'ext', This.name, "c" )
+      ext.lib_dir   = File.join( 'lib', This.name )
+      ext.gem_spec  = This.ruby_gemspec
+
+      ext.cross_compile  = true                         # enable cross compilation (requires cross compile toolchain)
+      ext.cross_platform = %w[x86-mingw32 x64-mingw32]  # forces the Windows platform instead of the default one
+                                                        # configure options only for cross compile
+    end
+  end
+
+  task :test_requirements => :compile
+rescue LoadError
+  This.task_warning( 'extension' )
+end
+
+CLOBBER << FileList["lib/**/*.{jar,so,bundle}"]
+CLOBBER << FileList["lib/#{This.name}/{1,2}.*/"]

data/tasks/man.rake

@@ -0,0 +1,7 @@
+#------------------------------------------------------------------------------
+# man pages
+#------------------------------------------------------------------------------
+desc "Create the man pages"
+task :man do
+  sh "ronn --roff #{FileList["man/*.ronn"]}"
+end

data/tasks/this.rb

@@ -0,0 +1,208 @@
+require 'pathname'
+
+# Public: A Class containing all the metadata and utilities needed to manage a
+# ruby project.
+class ThisProject
+  # The name of this project
+  attr_accessor :name
+
+  # The author's name
+  attr_accessor :author
+
+  # The email address of the author(s)
+  attr_accessor :email
+
+  # The homepage of this project
+  attr_accessor :homepage
+
+  # The regex of files to exclude from the manifest
+  attr_accessor :exclude_from_manifest
+
+  # The hash of Gem::Specifications keyed' by platform
+  attr_accessor :gemspecs
+
+  # Public: Initialize ThisProject
+  #
+  # Yields self
+  def initialize(&block)
+    @exclude_from_manifest = Regexp.union(/\.(git|DS_Store)/,
+                                          /^(doc|coverage|pkg|tmp|Gemfile(\.lock)?)/,
+                                          /^[^\/]+\.gemspec/,
+                                          /\.(swp|jar|bundle|so|rvmrc|travis.yml|byebug_history|fossa.yml|ruby-version)$/,
+                                          /~$/)
+    @gemspecs              = Hash.new
+    yield self if block_given?
+  end
+
+  # Public: return the version of ThisProject
+  #
+  # Search the ruby files in the project looking for the one that has the
+  # version string in it. This does not eval any code in the project, it parses
+  # the source code looking for the string.
+  #
+  # Returns a String version
+  def version
+    [ "lib/#{ name }.rb", "lib/#{ name }/version.rb" ].each do |v|
+      path = project_path( v )
+      line = path.read[/^\s*VERSION\s*=\s*.*/]
+      if line then
+        return line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+      end
+    end
+  end
+
+  # Internal: Return a section of an RDoc file with the given section name
+  #
+  # path         - the relative path in the project of the file to parse
+  # section_name - the section out of the file from which to parse data
+  #
+  # Retuns the text of the section as an array of paragrphs.
+  def section_of( file, section_name )
+    re    = /^[=#]+ (.*)$/
+    sectional = project_path( file )
+    parts = sectional.read.split( re )[1..-1]
+    parts.map! { |p| p.strip }
+
+    sections = Hash.new
+    Hash[*parts].each do |k,v|
+      sections[k] = v.split("\n\n")
+    end
+    return sections[section_name]
+  end
+
+  # Internal: print out a warning about the give task
+  def task_warning( task )
+    warn "WARNING: '#{task}' tasks are not defined. Please run 'rake develop'"
+  end
+
+  # Internal: Return the full path to the file that is relative to the project
+  # root.
+  #
+  # path - the relative path of the file from the project root
+  #
+  # Returns the Pathname of the file
+  def project_path( *relative_path )
+    project_root.join( *relative_path )
+  end
+
+  # Internal: The absolute path of this file
+  #
+  # Returns the Pathname of this file.
+  def this_file_path
+    Pathname.new( __FILE__ ).expand_path
+  end
+
+  # Internal: The root directory of this project
+  #
+  # This is defined as being the directory that is in the path of this project
+  # that has the first Rakefile
+  #
+  # Returns the Pathname of the directory
+  def project_root
+    this_file_path.ascend do |p|
+      rakefile = p.join( 'Rakefile' )
+      return p if rakefile.exist?
+    end
+  end
+
+  # Internal: Returns the contents of the Manifest.txt file as an array
+  #
+  # Returns an Array of strings
+  def manifest
+    manifest_file = project_path( "Manifest.txt" )
+    abort "You need a Manifest.txt" unless manifest_file.readable?
+    manifest_file.readlines.map { |l| l.strip }
+  end
+
+  # Internal: Return the files that define the extensions
+  #
+  # Returns an Array
+  def extension_conf_files
+    manifest.grep( /extconf.rb\Z/ )
+  end
+
+  # Internal: Returns the gemspace associated with the current ruby platform
+  def platform_gemspec
+    gemspecs.fetch(platform) { This.ruby_gemspec }
+  end
+
+  def core_gemspec
+    Gem::Specification.new do |spec|
+      spec.name        = name
+      spec.version     = version
+      spec.author      = author
+      spec.email       = email
+      spec.homepage    = homepage
+
+      spec.summary     = summary
+      spec.description = description
+      spec.license     = license
+
+      spec.files       = manifest
+      spec.executables = spec.files.grep(/^bin/) { |f| File.basename(f) }
+      spec.test_files  = spec.files.grep(/^(spec|test)/)
+
+      spec.extra_rdoc_files += spec.files.grep(/(txt|rdoc|md)$/)
+      spec.rdoc_options = [ "--main"  , 'README.md',
+                            "--markup", "tomdoc" ]
+
+      spec.required_ruby_version = '>= 2.2.2'
+    end
+  end
+
+  # Internal: Return the gemspec for the ruby platform
+  def ruby_gemspec( core = core_gemspec, &block )
+    yielding_gemspec( 'ruby', core, &block )
+  end
+
+  # Internal: Return the gemspec for the jruby platform
+  def java_gemspec( core = core_gemspec, &block )
+    yielding_gemspec( 'java', core, &block )
+  end
+
+  # Internal: give an initial spec and a key, create a new gemspec based off of
+  # it.
+  #
+  # This will force the new gemspecs 'platform' to be that of the key, since the
+  # only reason you would have multiple gemspecs at this point is to deal with
+  # different platforms.
+  def yielding_gemspec( key, core )
+    spec = gemspecs[key] ||= core.dup
+    spec.platform = key
+    yield spec if block_given?
+    return spec
+  end
+
+  # Internal: Return the platform of ThisProject at the current moment in time.
+  def platform
+    (RUBY_PLATFORM == "java") ? 'java' : Gem::Platform::RUBY
+  end
+
+  # Internal: Return the DESCRIPTION section of the README.rdoc file
+  def description_section
+    section_of( 'README.md', 'DESCRIPTION')
+  end
+
+  # Internal: Return the summary text from the README
+  def summary
+    description_section.first
+  end
+
+  # Internal: Return the full description text from the README
+  def description
+    description_section.join(" ").tr("\n", ' ').gsub(/[{}]/,'').gsub(/\[[^\]]+\]/,'') # strip rdoc
+  end
+
+  def license
+    license_file = project_path("LICENSE.txt")
+    line = license_file.readlines.first
+    line.split(/\s+/).first
+  end
+
+  # Internal: The path to the gemspec file
+  def gemspec_file
+    project_path( "#{ name }.gemspec" )
+  end
+end
+
+This = ThisProject.new

data/test/device_dataset.rb

@@ -0,0 +1,117 @@
+class DeviceDataset
+
+  include TestHelper
+
+  attr_reader :count
+  attr_reader :compare_fields
+  attr_reader :fields
+
+  attr_reader :filename_jsonl
+  attr_reader :filename_sorted_jsonl
+  attr_reader :filename_csv
+  attr_reader :filename_sorted_csv
+
+  def initialize(count:, compare_fields: [ "manufacturer", "model_name", "slug" ])
+    @count = count
+    @compare_fields = compare_fields
+    @fields = %w[
+      build_number
+      manufacturer
+      model_name
+      platform
+      serial
+      slug
+    ]
+    @filename_sorted_jsonl = nil
+    @filename_jsonl = nil
+    @filename_sorted_csv = nil
+    @filename_csv = nil
+    @slug = generate_slug
+  end
+
+  def persist_records_as_jsonl
+    @filename_jsonl = scratch_file(prefix: "unsorted_", slug: @slug)
+    @filename_jsonl.open("w+") do |f|
+      f.write(records_as_jsonl)
+    end
+  end
+
+  def persist_sorted_records_as_jsonl
+    @filename_sorted_jsonl = scratch_file(prefix: "sorted_", slug: @slug)
+    @filename_sorted_jsonl.open("w+") do |f|
+      f.write(sorted_records_as_jsonl)
+    end
+  end
+
+  def cleanup_files
+    [ @filename_sorted_jsonl, @filename_jsonl, @filename_sorted_csv, @filename_csv ].each do |p|
+      next if p.nil?
+      p.unlink if p.exist?
+    end
+  end
+
+  def records
+    @records ||= Array.new.tap do |a|
+      count.times do
+        a << Hash.new.tap do |h|
+          fields.each do |f|
+            value = (f == 'slug') ? generate_slug : ::Faker::Device.send(f)
+            h[f] = value
+          end
+        end
+      end
+    end
+  end
+
+  def sorted_records
+    @sorted_records ||= records.sort_by do |r|
+      compare_fields.map { |field| r[field] }
+    end
+  end
+
+  def records_as_jsonl
+    @jsonl_records ||= as_jsonl(list: records)
+  end
+
+  def records_as_csv
+    @csv_records ||= as_csv(list: records)
+  end
+
+  def records_as_csv_rows
+    @csv_rows ||= as_csv_rows(records_as_csv)
+  end
+
+  def sorted_records_as_jsonl
+    @jsonl_sorted_records ||= as_jsonl(list: sorted_records)
+  end
+
+  def sorted_records_as_csv
+    @csv_sorted_records ||= as_csv(list: sorted_records)
+  end
+
+  def sorted_records_as_csv_rows
+    @csv_sorted_rows ||= as_csv_rows(sorted_records_as_csv)
+  end
+
+  private
+
+  def as_jsonl(list:)
+    list.map { |r| Oj.dump(r) }.join("\n") + "\n"
+  end
+
+  def as_csv(list:, headers: fields)
+    CSV.generate('', headers: headers , write_headers: true) do |csv|
+      list.each do |r|
+        csv << fields.map { |f| r[f] }
+      end
+    end
+  end
+
+  def as_csv_rows(text)
+    Array.new.tap do |a|
+      CSV.new(text, converters: :numeric, headers: :first_row, return_headers: false).each do |row|
+        a << row
+      end
+    end
+  end
+end

data/test/input/test_file.rb

@@ -0,0 +1,73 @@
+require_relative '../test_helper'
+
+module TestInput
+  class TestFile < ::Minitest::Test
+    def test_does_not_handle_stdin_text
+      ::FlatKit::Input::IO::STDINS.each do |e|
+        refute(::FlatKit::Input::File.handles?(e), "#{e} is not stdin text")
+      end
+    end
+
+    def test_handles_existing_file
+      test_path = "tmp/test_handles_existing_file.txt"
+      begin
+        IO.write(test_path,"test handles existing file")
+        assert(::FlatKit::Input::File.handles?(test_path))
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_only_handles_string
+      refute(::FlatKit::Input::File.handles?(Object.new))
+    end
+
+    def test_raises_error_if_not_readable
+      assert_raises(FlatKit::Error) { ::FlatKit::Input::File.new("tmp/does-not-exist") }
+    end
+
+    def test_init_from_path
+      test_path = "tmp/test_init_from_path.txt"
+      begin
+        IO.write(test_path,"nothing to see here")
+        io = ::FlatKit::Input::File.new(test_path)
+        assert_equal(test_path, io.name)
+        assert_instance_of(::File, io.io)
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_reads_from_file
+      test_path = "tmp/test_reads_from_file.txt"
+      begin
+        text = "test_reads_from_file"
+        IO.write(test_path,text)
+
+        input = ::FlatKit::Input::File.new(test_path)
+        content = input.io.read
+        assert_equal(text, content)
+
+        input.close
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_reads_from_gzfile
+      test_path = "tmp/test_reads_from_gzfile.txt.gz"
+      begin
+        text = "this is something to read"
+        system("echo '#{text}' | gzip > #{test_path}")
+
+        input = ::FlatKit::Input::File.new(test_path)
+        content = input.io.read
+        assert_equal(text + "\n", content)
+
+        input.close
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+  end
+end

data/test/input/test_io.rb

@@ -0,0 +1,93 @@
+require_relative '../test_helper'
+
+module TestInput
+  class NullIO < ::IO
+    def initialize()
+    end
+  end
+
+  class TestIO < ::Minitest::Test
+
+    def test_handles_stdin_text
+      ::FlatKit::Input::IO::STDINS.each do |e|
+        assert(::FlatKit::Input::IO.handles?(e), "#{e} is not stdin text")
+      end
+    end
+
+    def test_handles_stdin_io
+      x = $stdin
+      assert(::FlatKit::Input::IO.handles?(x), "is not stdin")
+    end
+
+    def test_handles_stringio
+      assert(::FlatKit::Input::IO.handles?(StringIO.new))
+    end
+
+    def test_does_not_handle_other
+      x = Object.new
+      refute(::FlatKit::Input::IO.handles?(x))
+    end
+
+    def test_init_from_dash
+      io = ::FlatKit::Input::IO.new("-")
+      assert_equal("<STDIN>", io.name)
+      assert_equal(::STDIN, io.io)
+    end
+
+    def test_init_from_file_object
+      test_path = "tmp/test_init_from_file_object.txt"
+      begin
+        File.open(test_path, "w+") do |f|
+          io = ::FlatKit::Input::IO.new(f)
+          assert_equal(test_path, io.name)
+          assert_instance_of(::File, io.io)
+        end
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_init_from_string_io_object
+      sio = StringIO.new
+      io = ::FlatKit::Input::IO.new(sio)
+      assert_match(/StringIO/, io.name)
+      assert_instance_of(::StringIO, io.io)
+    end
+
+    def test_init_from_io_object
+      null_io = NullIO.new
+      io = ::FlatKit::Input::IO.new(null_io)
+      assert_match(/NullIO/, io.name)
+      assert_instance_of(::TestInput::NullIO, io.io)
+    end
+
+    def test_init_from_stdin
+      io  = ::FlatKit::Input::IO.new($stdin)
+      assert_equal("<STDIN>", io.name)
+      assert_equal(::STDIN, io.io)
+    end
+
+    def test_init_from_invalid
+      assert_raises(::FlatKit::Error) { ::FlatKit::Input::IO.new(Object.new) }
+    end
+
+    def test_reads_from_io
+      test_path = "tmp/test_reads_from_io.txt"
+      begin
+        line = "This is a line to read"
+        File.open(test_path, "w+") do |f|
+          f.write(line)
+          f.rewind
+
+          io = ::FlatKit::Input::IO.new(f)
+          content = io.io.read
+          io.close
+          assert_equal(content, line)
+        end
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+  end
+end