embulk 0.7.11 → 0.8.0

data/lib/embulk/data/new/java/config/checkstyle/default.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip

data/lib/embulk/data/new/java/gradlew

@@ -42,11 +42,6 @@ case "`uname`" in
     ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
 # Attempt to set APP_HOME
 # Resolve links: $0 may be a link
 PRG="$0"
@@ -61,9 +56,9 @@ while [ -h "$PRG" ] ; do
     fi
 done
 SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
+cd "`dirname \"$PRG\"`/" >/dev/null
 APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
+cd "$SAVED" >/dev/null
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
@@ -114,6 +109,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`

data/lib/embulk/data/new/ruby/.ruby-version

@@ -1 +1 @@
-jruby-9.0.0.0
+jruby-9.0.4.0

data/lib/embulk/guess/csv.rb

@@ -72,8 +72,8 @@ module Embulk
               else
                 # disable escaping (set null)
               end
-              parser_guessed["escape"] = escape
             end
+            parser_guessed["escape"] = escape
           else
             # escape does nothing if quote is disabled
           end

data/lib/embulk/guess/schema_guess.rb

@@ -68,6 +68,12 @@ module Embulk::Guess
           return nil
         end
 
+        begin
+          JSON.parse(str)
+          return "json"
+        rescue
+        end
+
         return "string"
       end
 

data/lib/embulk/guess_plugin.rb

@@ -27,7 +27,7 @@ module Embulk
     end
 
     def self.from_java(java_class)
-      JavaPlugin.ruby_adapter(java_class, GuessPlugin, RubyAdapter)
+      JavaPlugin.ruby_adapter_class(java_class, GuessPlugin, RubyAdapter)
     end
 
     module RubyAdapter

data/lib/embulk/java/imports.rb

@@ -22,6 +22,10 @@ module Embulk::Java
       include_package 'org.embulk.time'
     end
 
+    module Json
+      include_package 'org.embulk.json'
+    end
+
     module Type
       include_package 'org.embulk.type'
     end

data/lib/embulk/plugin_registry.rb

@@ -27,9 +27,9 @@ module Embulk
         if value = @map[type]
           return value
         end
-        raise PluginLoadError, "Unknown #{@category} plugin '#{type}'. #{@search_prefix}#{type}.rb is installed but it does not correctly register plugin."
+        raise PluginLoadError.new "Unknown #{@category} plugin '#{type}'. #{@search_prefix}#{type}.rb is installed but it does not correctly register plugin."
       else
-        raise PluginLoadError, "Unknown #{@category} plugin '#{type}'. #{@search_prefix}#{type}.rb is not installed. Run 'embulk gem search -rd embulk-#{@category}' command to find plugins."
+        raise PluginLoadError.new "Unknown #{@category} plugin '#{type}'. #{@search_prefix}#{type}.rb is not installed. Run 'embulk gem search -rd embulk-#{@category}' command to find plugins."
       end
     end
 
@@ -58,10 +58,8 @@ module Embulk
 
       # search gems
       if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
-        specs = Kernel::RUBYGEMS_ACTIVATION_MONITOR.synchronize do  # this lock is added as a workaround of https://github.com/jruby/jruby/issues/3652
-          Gem::Specification.find_all do |spec|
-            spec.contains_requirable_file? name
-          end
+        specs = Gem::Specification.find_all do |spec|
+          spec.contains_requirable_file? name
         end
 
         # prefer newer version
@@ -80,12 +78,10 @@ module Embulk
     def require_and_show(path, spec=nil)
       require path
       unless spec
-        name, spec = Kernel::RUBYGEMS_ACTIVATION_MONITOR.synchronize do  # this lock is added as a workaround of https://github.com/jruby/jruby/issues/3652
-          Gem.loaded_specs.find {|name,spec|
-            #spec.files.include?(path)
-            spec.contains_requirable_file?(path)
-          }
-        end
+        name, spec = Gem.loaded_specs.find {|name,spec|
+          #spec.files.include?(path)
+          spec.contains_requirable_file?(path)
+        }
       end
       if spec
         unless @loaded_gems[spec.name]

data/lib/embulk/schema.rb

@@ -19,6 +19,7 @@ module Embulk
       each do |column|
         idx = column.index
         column_script =
+          "value_api = ::Embulk::Java::SPI::Json::RubyValueApi\n" <<
           "if reader.isNull(#{idx})\n" <<
           "record << nil\n" <<
           "else\n" <<
@@ -33,6 +34,8 @@ module Embulk
             "record << reader.getString(#{idx})"
           when :timestamp
             "record << reader.getTimestamp(#{idx}).getRubyTime(JRuby.runtime)"
+          when :json
+            "record << MessagePack.unpack(value_api.toMessagePack(JRuby.runtime, reader.getJson(#{idx})))"
           else
             raise "Unknown type #{column.type.inspect}"
           end <<
@@ -45,6 +48,7 @@ module Embulk
 
       record_writer_script = "lambda do |builder,record|\n"
       record_writer_script << "java_timestamp_class = ::Embulk::Java::Timestamp\n"
+      record_writer_script << "value_api = ::Embulk::Java::SPI::Json::RubyValueApi\n"
       each do |column|
         idx = column.index
         column_script =
@@ -62,6 +66,8 @@ module Embulk
             "builder.setString(#{idx}, record[#{idx}])"
           when :timestamp
             "builder.setTimestamp(#{idx}, java_timestamp_class.fromRubyTime(record[#{idx}]))"
+          when :json
+            "builder.setJson(#{idx}, value_api.fromMessagePack(MessagePack.pack(record[#{idx}])))"
           else
             raise "Unknown type #{column.type.inspect}"
           end <<

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.11'
+  VERSION = '0.8.0'
 end

data/test/guess/test_csv_guess.rb

@@ -0,0 +1,170 @@
+require 'helper'
+require 'time'
+require 'embulk/guess/csv'
+
+class CsvGuessTest < ::Test::Unit::TestCase
+  class TestDelimiter < self
+    data(
+      "\t" => "\t",
+      "," => ",",
+      "|" => "|",
+    )
+    def test_delimiter_detection(delim)
+      actual = guess([
+        ["1", "foo"].join(delim),
+        ["2", "bar"].join(delim),
+      ])
+      assert_equal delim, actual["parser"]["delimiter"]
+    end
+  end
+
+  class TestQuote < self
+    data(
+      "'" => "'",
+      '"' => '"',
+      nil => nil,
+    )
+    def test_quote(quotation)
+      actual = guess([
+        %w(1 foo).map{|str| %Q(#{quotation}#{str}#{quotation})}.join("\t"),
+        %w(2 bar).map{|str| %Q(#{quotation}#{str}#{quotation})}.join("\t"),
+      ])
+      assert_equal quotation, actual["parser"]["quote"]
+    end
+  end
+
+  class TestEscape < self
+    data(
+      "\\" => "\\",
+      '"' => '"',
+    )
+    def test_escape(char)
+      actual = guess([
+        %Q('1'\t'F#{char}'OO'),
+        %Q('2'\t'FOOOOOOOO#{char}'OO'),
+      ])
+      assert_equal char, actual["parser"]["escape"]
+    end
+  end
+
+  class TestSkipHeaderLines < self
+    def test_skip_header_lines_one
+      actual = guess([
+        "col1\tcol2",
+        "1\tfoo",
+        "2\tbar",
+      ])
+      assert_equal 1, actual["parser"]["skip_header_lines"]
+    end
+
+    def test_skip_header_lines_three
+      actual = guess([
+        "this is a CSV",
+        "created for a test",
+        "col1\tcol2",
+        "1\tfoo",
+        "2\tbar",
+      ])
+      assert_equal 3, actual["parser"]["skip_header_lines"]
+    end
+  end
+
+  class TestNullString < self
+    data(
+      "\\N" => "\\N",
+      "null" => "null",
+      "NULL" => "NULL",
+      "#N/A" => "#N/A",
+      nil => nil,
+    )
+    def test_null_string(null)
+      actual = guess([
+        "1\tfoo\t#{null}",
+        "2\tbar\t#{null}",
+      ])
+      assert_equal null, actual["parser"]["null_string"]
+    end
+  end
+
+  class TestTrim < self
+    def test_trim_flag_when_will_be_long_if_strip_arround_space
+      actual = guess([
+        "  1 \tfoo",
+        "  2 \tfoo",
+        "  3 \tfoo",
+      ])
+      assert_equal true, actual["parser"]["trim_if_not_quoted"]
+    end
+  end
+
+  class TestCommentLineMarker < self
+    data(
+      "#" => "#",
+      "//" => "//",
+    )
+    def test_comment_line_marker(marker)
+      actual = guess([
+        "foo\t 1\tother",
+        "#{marker} foo\t 2\tother",
+        "foo\t 3\tother",
+      ])
+      assert_equal marker, actual["parser"]["comment_line_marker"]
+    end
+  end
+
+  class TestColumns < self
+    def test_columns_without_header
+      actual = guess([
+        "1\tfoo\t2000-01-01T00:00:00+0900",
+        "2\tbar\t2000-01-01T00:00:00+0900",
+      ])
+      expected = [
+        {"name" => "c0", "type" => "long"},
+        {"name" => "c1", "type" => "string"},
+        {"name" => "c2", "type" => "timestamp", "format"=>"%Y-%m-%dT%H:%M:%S%z"},
+      ]
+      assert_equal expected, actual["parser"]["columns"]
+    end
+
+    def test_columns_with_header
+      actual = guess([
+        "num\tstr\ttime",
+        "1\tfoo\t2000-01-01T00:00:00+0900",
+        "2\tbar\t2000-01-01T00:00:00+0900",
+      ])
+      expected = [
+        {"name" => "num", "type" => "long"},
+        {"name" => "str", "type" => "string"},
+        {"name" => "time", "type" => "timestamp", "format"=>"%Y-%m-%dT%H:%M:%S%z"},
+      ]
+      assert_equal expected, actual["parser"]["columns"]
+    end
+
+    def test_complex_line
+      actual = guess([
+        %Q(this is useless header),
+        %Q(and more),
+        %Q(num,str,quoted_num,time),
+        %Q(1, "value with space "" and quote in it", "123",21150312000000Z),
+        %Q(2),
+        %Q(# 3, "this is commented out" ,"1",21150312000000Z),
+      ])
+      expected = [
+        {"name" => "num", "type" => "long"},
+        {"name" => "str", "type" => "string"},
+        {"name" => "quoted_num", "type" => "long"},
+        {"name" => "time", "type" => "timestamp", "format"=>"%Y%m%d%H%M%S%z"},
+      ]
+      assert_equal expected, actual["parser"]["columns"]
+    end
+  end
+
+  def guess(texts)
+    conf = Embulk::DataSource.new({
+      parser: {
+        type: "csv"
+      }
+    })
+    Embulk::Guess::CsvGuessPlugin.new.guess_lines(conf, Array(texts))
+  end
+end