embulk 0.8.15 → 0.8.16

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/gradlew

@@ -6,12 +6,30 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,26 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -85,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -157,4 +161,9 @@ function splitJvmOpts() {
 eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
 JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
 
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+  cd "$(dirname "$0")"
+fi
+
 exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

data/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/lib/embulk/command/embulk_migrate_plugin.rb

@@ -43,8 +43,8 @@ module Embulk
     end
 
     # upgrade gradle version
-    if migrator.match("gradle/wrapper/gradle-wrapper.properties", /gradle-2\.\d-/)
-      # gradle < 2.10 (\d matches one digit)
+    if migrator.match("gradle/wrapper/gradle-wrapper.properties", /gradle-[23]\.\d+(\.\d+)?-/)
+      # gradle < 3.2.1
       require 'embulk/data/package_data'
       data = PackageData.new("new", migrator.path)
       migrator.write "gradle/wrapper/gradle-wrapper.properties", data.content("java/gradle/wrapper/gradle-wrapper.properties")

data/lib/embulk/data/new/java/build.gradle.erb

@@ -60,9 +60,11 @@ task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
     script "pkg/${project.name}-${project.version}.gem"
 }
 
-task "package"(dependsOn: ["gemspec", "classpath"]) << {
-    println "> Build succeeded."
-    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
 }
 
 task gemspec {

data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/lib/embulk/data/new/java/gradlew

@@ -6,12 +6,30 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,26 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -85,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -157,4 +161,9 @@ function splitJvmOpts() {
 eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
 JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
 
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+  cd "$(dirname "$0")"
+fi
+
 exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

data/lib/embulk/data/new/java/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/lib/embulk/guess/csv.rb

@@ -6,7 +6,7 @@ module Embulk
       Plugin.register_guess('csv', self)
 
       DELIMITER_CANDIDATES = [
-        ",", "\t", "|"
+        ",", "\t", "|", ";"
       ]
 
       QUOTE_CANDIDATES = [
@@ -41,8 +41,8 @@ module Embulk
         else
           delim = guess_delimiter(sample_lines)
           unless delim
-            # not CSV file
-            return {}
+            # assuming single column CSV
+            delim = DELIMITER_CANDIDATES.first
           end
         end
 
@@ -104,26 +104,48 @@ module Embulk
 
         sample_records = split_lines(parser_guessed, true, sample_lines, delim, {})
 
-        first_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
-        other_types = SchemaGuess.types_from_array_records(sample_records[1..-1] || [])
+        if sample_lines.size == 1
+          # The file contains only 1 line. Assume that there are no header line.
+          header_line = false
 
-        if first_types.size <= 1 || other_types.size <= 1
-          # guess failed
-          return {}
-        end
+          column_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
 
-        unless parser_guessed.has_key?("trim_if_not_quoted")
-          sample_records_trimmed = split_lines(parser_guessed, true, sample_lines, delim, {"trim_if_not_quoted" => true})
-          other_types_trimmed = SchemaGuess.types_from_array_records(sample_records_trimmed[1..-1] || [])
-          if other_types != other_types_trimmed
-            parser_guessed["trim_if_not_quoted"] = true
-            other_types = other_types_trimmed
-          else
-            parser_guessed["trim_if_not_quoted"] = false
+          unless parser_guessed.has_key?("trim_if_not_quoted")
+            sample_records_trimmed = split_lines(parser_guessed, true, sample_lines, delim, {"trim_if_not_quoted" => true})
+            column_types_trimmed = SchemaGuess.types_from_array_records(sample_records_trimmed)
+            if column_types != column_types_trimmed
+              parser_guessed["trim_if_not_quoted"] = true
+              column_types = column_types_trimmed
+            else
+              parser_guessed["trim_if_not_quoted"] = false
+            end
+          end
+        else
+          # The file contains more than 1 line. If guessed first line's column types are all strings or boolean, and the types are
+          # different from the other lines, assume that the first line is column names.
+          first_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
+          other_types = SchemaGuess.types_from_array_records(sample_records[1..-1] || [])
+
+          unless parser_guessed.has_key?("trim_if_not_quoted")
+            sample_records_trimmed = split_lines(parser_guessed, true, sample_lines, delim, {"trim_if_not_quoted" => true})
+            other_types_trimmed = SchemaGuess.types_from_array_records(sample_records_trimmed[1..-1] || [])
+            if other_types != other_types_trimmed
+              parser_guessed["trim_if_not_quoted"] = true
+              other_types = other_types_trimmed
+            else
+              parser_guessed["trim_if_not_quoted"] = false
+            end
           end
+
+          header_line = (first_types != other_types && first_types.all? {|t| ["string", "boolean"].include?(t) }) || guess_string_header_line(sample_records)
+          column_types = other_types
         end
 
-        header_line = (first_types != other_types && first_types.all? {|t| ["string", "boolean"].include?(t) }) || guess_string_header_line(sample_records)
+        if column_types.empty?
+          # TODO here is making the guessing failed if the file doesn't contain any columns. However,
+          #      this may not be convenient for users.
+          return {}
+        end
 
         if header_line
           parser_guessed["skip_header_lines"] = skip_header_lines + 1
@@ -135,12 +157,12 @@ module Embulk
         parser_guessed["allow_optional_columns"] = false unless parser_guessed.has_key?("allow_optional_columns")
 
         if header_line
-          column_names = sample_records.first
+          column_names = sample_records.first.map(&:strip)
         else
-          column_names = (0..other_types.size).to_a.map {|i| "c#{i}" }
+          column_names = (0..column_types.size).to_a.map {|i| "c#{i}" }
         end
         schema = []
-        column_names.zip(other_types).each do |name,type|
+        column_names.zip(column_types).each do |name,type|
           if name && type
             schema << new_column(name, type)
           end
@@ -313,7 +335,7 @@ module Embulk
         first = sample_records.first
         first.count.times do |column_index|
           lengths = sample_records.map {|row| row[column_index] }.compact.map {|v| v.to_s.size }
-          if lengths.size > 2
+          if lengths.size > 1
             if array_variance(lengths[1..-1]) <= 0.2
               avg = array_avg(lengths[1..-1])
               if avg == 0.0 ? lengths[0] > 1 : (avg - lengths[0]).abs / avg > 0.7

data/lib/embulk/guess/newline.rb

@@ -4,10 +4,16 @@ module Embulk
     class NewlineGuessPlugin < TextGuessPlugin
       Plugin.register_guess('newline', self)
 
-      def guess_text(config, sample_text)
-        cr_count = sample_text.count("\r")
-        lf_count = sample_text.count("\n")
-        crlf_count = sample_text.scan(/\r\n/).length
+      def guess(config, sample)
+        if config.fetch('parser', {}).fetch('charset', nil).nil?
+          require 'embulk/guess/charset'
+          charset_guess = Guess::CharsetGuessPlugin.new
+          return charset_guess.guess(config, sample)
+        end
+
+        cr_count = sample.count("\r")
+        lf_count = sample.count("\n")
+        crlf_count = sample.scan(/\r\n/).length
         if crlf_count > cr_count / 2 && crlf_count > lf_count / 2
           return {"parser" => {"newline" => "CRLF"}}
         elsif cr_count > lf_count / 2

data/lib/embulk/guess_plugin.rb

@@ -113,7 +113,9 @@ module Embulk
         while line = decoder.poll
           sample_lines << line
         end
-        sample_lines.pop unless sample_lines.empty?  # last line can be partial
+        unless sample.end_with?(parser_task.getNewline.getString)
+          sample_lines.pop if sample_lines.empty? # last line is partial
+        end
       end
 
       return guess_lines(config, sample_lines);