embulk 0.8.35-java → 0.10.24-java

data/lib/embulk/data/package_data.rb

@@ -1,59 +0,0 @@
-module Embulk
-
-  class PackageData
-    def initialize(base_name, dest_dir, erb_binding=nil)
-      require 'fileutils'
-      @base_name = base_name
-      @dest_dir = dest_dir
-      @erb_binding = erb_binding
-    end
-
-    def path(src)
-      Embulk.lib_path("embulk/data/#{@base_name}/#{src}")
-    end
-
-    def content(src)
-      File.read(path(src))
-    end
-
-    def bincontent(src)
-      File.binread(path(src))
-    end
-
-    def erb(src)
-      require 'erb'
-      ERB.new(content(src), nil, '%').result(@erb_binding)
-    end
-
-    def cp(src, dest_name)
-      dest = dest_path_message(dest_name)
-      File.open(dest, "wb") do |dst_io|
-        File.open(path(src), "rb") do |src_io|
-          FileUtils.copy_stream src_io, dst_io
-        end
-      end
-    end
-
-    def cp_erb(src, dest_name)
-      dest = dest_path_message(dest_name)
-      File.open(dest, "wb") {|f| f.write erb(src) }
-    end
-
-    def dest_path(dest_name)
-      File.join(@dest_dir, *dest_name.split('/'))
-    end
-
-    def dest_path_message(dest_name)
-      path = dest_path(dest_name)
-      puts "  Creating #{path}"
-      FileUtils.mkdir_p File.dirname(path)
-      path
-    end
-
-    def set_executable(dest_name)
-      dest = dest_path(dest_name)
-      File.chmod(File.stat(dest).mode | 0111, dest)
-    end
-  end
-
-end

data/lib/embulk/java/time_helper.rb

@@ -1,79 +0,0 @@
-module Embulk
-  module Java
-    #require 'embulk/java/imports'
-    require 'time'  # Date._strptime
-
-    class TimeParserHelper
-      #include Java::JRubyTimeParserHelper
-      include org.embulk.spi.time.JRubyTimeParserHelper
-
-      class Factory
-        #include Java::JRubyTimeParserHelperFactory
-        include org.embulk.spi.time.JRubyTimeParserHelperFactory
-
-        def newInstance(format_string, year, mon, day, hour, min, sec, usec)
-          default_time = Time.utc(year, mon, day, hour, min, sec, usec)
-          TimeParserHelper.new(format_string, default_time)
-        end
-      end
-
-      def initialize(format_string, default_time)
-        @format_string = format_string
-        @default_time = default_time
-      end
-
-      # Override
-      def strptimeUsec(text)
-        hash = Date._strptime(text, @format_string)
-        unless hash
-          raise Java::TimestampParseException.new("Failed to parse '" + text + "'")
-        end
-
-        if seconds = hash[:seconds] # if %s, it's Integer. if %Q, Rational.
-          sec_fraction = hash[:sec_fraction]  # Rational
-          usec = sec_fraction * 1_000_000 if sec_fraction
-          return (seconds * 1_000_000).to_i + usec.to_i
-
-        else
-          year = hash[:year]
-          mon = hash[:mon]
-          day = hash[:mday]
-          hour = hash[:hour]
-          min = hash[:min]
-          sec = hash[:sec]
-          sec_fraction = hash[:sec_fraction]
-          usec = sec_fraction * 1_000_000 if sec_fraction
-          zone = hash[:zone]
-
-          now = @default_time
-          begin
-            break if year; year = now.year
-            break if mon; mon = now.mon
-            break if day; day = now.day
-            break if hour; hour = now.hour
-            break if min; min = now.min
-            break if sec; sec = now.sec
-            break if sec_fraction; usec = now.tv_usec
-          end until true
-
-          year ||= 1970
-          mon ||= 1
-          day ||= 1
-          hour ||= 0
-          min ||= 0
-          sec ||= 0
-          usec ||= 0
-
-          @zone = zone
-          time = Time.utc(year, mon, day, hour, min, sec, usec)
-          return time.tv_sec * 1_000_000 + time.tv_usec
-        end
-      end
-
-      # Override
-      def getZone
-        @zone
-      end
-    end
-  end
-end

data/lib/embulk/runner.rb

@@ -1,53 +0,0 @@
-module Embulk
-
-  class EmbulkRunner
-    def initialize(runner_java)
-      @runner_java = runner_java  # Java::org.embulk.EmbulkRunner
-    end
-
-    def guess(config, options={})
-      output_path =
-        (options[:next_config_output_path] ? Java::java.lang.String.new(options[:next_config_output_path]) : nil)
-
-      case config
-      when String
-        @runner_java.guess(Java::java.lang.String.new(config), output_path)
-      when Hash, DataSource
-        @runner_java.guess(DataSource.new(config).to_java, output_path)
-      end
-
-      nil
-    end
-
-    def preview(config, options={})
-      format = (options[:format] ? Java::java.lang.String.new(options[:format]) : nil)
-
-      case config
-      when String
-        @runner_java.preview(Java::java.lang.String.new(config), format)
-      when Hash, DataSource
-        @runner_java.preview(DataSource.new(config).to_java, format)
-      end
-
-      nil
-    end
-
-    def run(config, options={})
-      config_diff_path =
-        (options[:next_config_diff_path] ? Java::java.lang.String.new(options[:next_config_diff_path]) : nil)
-      output_path =  # deprecated
-        (options[:next_config_output_path] ? Java::java.lang.String.new(options[:next_config_output_path]) : nil)
-      resume_state_path =
-        (options[:resume_state_path] ? Java::java.lang.String.new(options[:resume_state_path]) : nil)
-
-      case config
-      when String
-        @runner_java.run(Java::java.lang.String.new(config), config_diff_path, output_path, resume_state_path)
-      when Hash, DataSource
-        @runner_java.run(DataSource.new(config).to_java, config_diff_path, output_path, resume_state_path)
-      end
-
-      nil
-    end
-  end
-end

data/settings.gradle

@@ -1,8 +0,0 @@
-rootProject.name = 'embulk'
-
-include 'embulk-core'
-include 'embulk-standards'
-include 'embulk-cli'
-include 'embulk-test'
-include 'embulk-jruby-strptime'
-include 'embulk-docs'

data/test/guess/test_csv_all_strings.rb

@@ -1,43 +0,0 @@
-require 'helper'
-require 'time'
-require 'embulk/guess/csv_all_strings'
-
-class CsvAllStringsGuessTest < ::Test::Unit::TestCase
-  class TestAllStrings < self
-    def test_columns_without_header
-      actual = guess([
-        "1\tfoo\t2000-01-01T00:00:00+0900",
-        "2\tbar\t2000-01-01T00:00:00+0900",
-      ])
-      expected = [
-        {"name" => "c0", "type" => "string"},
-        {"name" => "c1", "type" => "string"},
-        {"name" => "c2", "type" => "string"},
-      ]
-      assert_equal expected, actual["parser"]["columns"]
-    end
-
-    def test_columns_with_header
-      actual = guess([
-        "num\tstr\ttime",
-        "1\tfoo\t2000-01-01T00:00:00+0900",
-        "2\tbar\t2000-01-01T00:00:00+0900",
-      ])
-      expected = [
-        {"name" => "num", "type" => "string"},
-        {"name" => "str", "type" => "string"},
-        {"name" => "time", "type" => "string"},
-      ]
-      assert_equal expected, actual["parser"]["columns"]
-    end
-  end
-
-  def guess(texts)
-    conf = Embulk::DataSource.new({
-      parser: {
-        type: "csv"
-      }
-    })
-    Embulk::Guess::CsvAllStringsGuessPlugin.new.guess_lines(conf, Array(texts))
-  end
-end

data/test/guess/test_csv_guess.rb

@@ -1,170 +0,0 @@
-require 'helper'
-require 'time'
-require 'embulk/guess/csv'
-
-class CsvGuessTest < ::Test::Unit::TestCase
-  class TestDelimiter < self
-    data(
-      "\t" => "\t",
-      "," => ",",
-      "|" => "|",
-    )
-    def test_delimiter_detection(delim)
-      actual = guess([
-        ["1", "foo"].join(delim),
-        ["2", "bar"].join(delim),
-      ])
-      assert_equal delim, actual["parser"]["delimiter"]
-    end
-  end
-
-  class TestQuote < self
-    data(
-      "'" => "'",
-      '"' => '"',
-      nil => nil,
-    )
-    def test_quote(quotation)
-      actual = guess([
-        %w(1 foo).map{|str| %Q(#{quotation}#{str}#{quotation})}.join("\t"),
-        %w(2 bar).map{|str| %Q(#{quotation}#{str}#{quotation})}.join("\t"),
-      ])
-      assert_equal quotation, actual["parser"]["quote"]
-    end
-  end
-
-  class TestEscape < self
-    data(
-      "\\" => "\\",
-      '"' => '"',
-    )
-    def test_escape(char)
-      actual = guess([
-        %Q('1'\t'F#{char}'OO'),
-        %Q('2'\t'FOOOOOOOO#{char}'OO'),
-      ])
-      assert_equal char, actual["parser"]["escape"]
-    end
-  end
-
-  class TestSkipHeaderLines < self
-    def test_skip_header_lines_one
-      actual = guess([
-        "col1\tcol2",
-        "1\tfoo",
-        "2\tbar",
-      ])
-      assert_equal 1, actual["parser"]["skip_header_lines"]
-    end
-
-    def test_skip_header_lines_three
-      actual = guess([
-        "this is a CSV",
-        "created for a test",
-        "col1\tcol2",
-        "1\tfoo",
-        "2\tbar",
-      ])
-      assert_equal 3, actual["parser"]["skip_header_lines"]
-    end
-  end
-
-  class TestNullString < self
-    data(
-      "\\N" => "\\N",
-      "null" => "null",
-      "NULL" => "NULL",
-      "#N/A" => "#N/A",
-      nil => nil,
-    )
-    def test_null_string(null)
-      actual = guess([
-        "1\tfoo\t#{null}",
-        "2\tbar\t#{null}",
-      ])
-      assert_equal null, actual["parser"]["null_string"]
-    end
-  end
-
-  class TestTrim < self
-    def test_trim_flag_when_will_be_long_if_strip_arround_space
-      actual = guess([
-        "  1 \tfoo",
-        "  2 \tfoo",
-        "  3 \tfoo",
-      ])
-      assert_equal true, actual["parser"]["trim_if_not_quoted"]
-    end
-  end
-
-  class TestCommentLineMarker < self
-    data(
-      "#" => "#",
-      "//" => "//",
-    )
-    def test_comment_line_marker(marker)
-      actual = guess([
-        "foo\t 1\tother",
-        "#{marker} foo\t 2\tother",
-        "foo\t 3\tother",
-      ])
-      assert_equal marker, actual["parser"]["comment_line_marker"]
-    end
-  end
-
-  class TestColumns < self
-    def test_columns_without_header
-      actual = guess([
-        "1\tfoo\t2000-01-01T00:00:00+0900",
-        "2\tbar\t2000-01-01T00:00:00+0900",
-      ])
-      expected = [
-        {"name" => "c0", "type" => "long"},
-        {"name" => "c1", "type" => "string"},
-        {"name" => "c2", "type" => "timestamp", "format"=>"%Y-%m-%dT%H:%M:%S%z"},
-      ]
-      assert_equal expected, actual["parser"]["columns"]
-    end
-
-    def test_columns_with_header
-      actual = guess([
-        "num\tstr\ttime",
-        "1\tfoo\t2000-01-01T00:00:00+0900",
-        "2\tbar\t2000-01-01T00:00:00+0900",
-      ])
-      expected = [
-        {"name" => "num", "type" => "long"},
-        {"name" => "str", "type" => "string"},
-        {"name" => "time", "type" => "timestamp", "format"=>"%Y-%m-%dT%H:%M:%S%z"},
-      ]
-      assert_equal expected, actual["parser"]["columns"]
-    end
-
-    def test_complex_line
-      actual = guess([
-        %Q(this is useless header),
-        %Q(and more),
-        %Q(num,str,quoted_num,time),
-        %Q(1, "value with space "" and quote in it", "123",21150312000000Z),
-        %Q(2),
-        %Q(# 3, "this is commented out" ,"1",21150312000000Z),
-      ])
-      expected = [
-        {"name" => "num", "type" => "long"},
-        {"name" => "str", "type" => "string"},
-        {"name" => "quoted_num", "type" => "long"},
-        {"name" => "time", "type" => "timestamp", "format"=>"%Y%m%d%H%M%S%z"},
-      ]
-      assert_equal expected, actual["parser"]["columns"]
-    end
-  end
-
-  def guess(texts)
-    conf = Embulk::DataSource.new({
-      parser: {
-        type: "csv"
-      }
-    })
-    Embulk::Guess::CsvGuessPlugin.new.guess_lines(conf, Array(texts))
-  end
-end

data/test/guess/test_schema_guess.rb

@@ -1,29 +0,0 @@
-require 'helper'
-require 'time'
-require 'embulk/guess/schema_guess'
-
-class SchemaGuessTest < ::Test::Unit::TestCase
-  G = Embulk::Guess::SchemaGuess
-  C = Embulk::Column
-
-  def test_guess
-    G.from_hash_records([{"int" => "1", "str" => "a"}])
-  end
-
-  def test_coalesce
-    assert_equal(
-      [C.new(0, "a", :timestamp, "%Y%m%d")],
-      G.from_hash_records([
-        {"a" => "20160101"},
-        {"a" => "20160101"},
-      ]))
-
-    assert_equal(
-      [C.new(0, "a", :long)],
-      G.from_hash_records([
-        {"a" => "20160101"},
-        {"a" => "20160101"},
-        {"a" => "12345678"},
-      ]))
-  end
-end