embulk-parser-poi_excel 0.1.4 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be106894c6b2167dd2f275cf8696003cc82195c3
-  data.tar.gz: c6f3718249f56278a0a29dd50521216765838c51
+  metadata.gz: 3ac332670f656bcf9bbf5fe659b53a9a5d61ed34
+  data.tar.gz: b08187d453742b61242f3c0a414b859e7b3df605
 SHA512:
-  metadata.gz: 6c88d6b4604a07b40681a44290bdf75c7f8c10e6e857da5ed254fe7828fc796e54f84e0f37de1e5c9f0089c934dd91283783b95849180b620cf1180a70d8a419
-  data.tar.gz: 772b8d21d4e1587acd97886a79b284a1db42d8a12ac30660a0d2eba9db61292839ceeef39f65d8756e0ee4aea58c7a3c9e3785fcda31c51f01708b116497c6d7
+  metadata.gz: d10568439d7728ce805c8a4945dce146b3c5b3983b1be2433b13986a07be0ac0cb52763daebe5c2cc0e0154403e3ec3a3fbbc57e1ff6112afb01a23157ad11da
+  data.tar.gz: e3b6f6efa059db9b8994ae78700cc8c50f67d80059b76eb41dafbc1b5ce26f1b133c1def929a7e4aa1d1f0fdaaff0d174683d2dba86956948276d2038275dd2e

data/README.md

@@ -28,12 +28,12 @@ in:
 ```
 
 if omit **value**, specified `cell_value`.  
-if omit **column_number** when **valus** is `cell_value`, specified next column.  
+if omit **column_number** when **value** is `cell_value`, specified next column.  
 
 
 ## Configuration
 
-* **sheets**: sheet name. (list of string, required)
+* **sheets**: sheet name. can use wildcards `*`, `?`. (list of string, required)
 * **skip_header_lines**: skip rows. (integer, default: `0`)
 * **columns**: column definition. see below. (hash, required)
 * **sheet_options**: sheet option. see below. (hash, default: null)
@@ -44,9 +44,15 @@ if omit **column_number** when **valus** is `cell_value`, specified next column.
 * **type**: Embulk column type. (string, required)
 * **value**: value type. see below. (string, defualt: `cell_value`)
 * **column_number**: Excel column number. see below. (string, default: next column)
+* **cell_address**: Excel cell address such as `A1`, `Sheet1!B3`. only one of `column_number`, `cell_address` can be specified. (string, not required)
+* **numeric_format**: format of numeric(double) to string such as `%4.2f`. (default: Java's Double.toString())
 * **attribute_name**: use with value `cell_style`, `cell_font`, etc. see below. (list of string)
 * **on_cell_error**: processing method of Cell error. see below. (string, default: `constant`)
+* **formula_handling**: processing method of formula. see below. (`evaluate` or `cashed_value`. default: `evaluate`)
+* **on_evaluate_error**: processing method of evaluate formula error. see below. (string, default: `exception`)
+* **formula_replace**: replace formula before evaluate. see below.
 * **on_convert_error**: processing method of convert error. see below. (string, default: `exception`)
+* **search_merged_cell**: search merged cell when cell is BLANK. (`none`, `linear_search`, `tree_search` or `hash_search`, default: `hash_search`)
 
 ### value
 
@@ -55,6 +61,8 @@ if omit **column_number** when **valus** is `cell_value`, specified next column.
 * `cell_style`: all cell style attributes. returned json string. see **attribute_name**. (**type** required `string`)
 * `cell_font`: all cell font attributes. returned json string. see **attribute_name**. (**type** required `string`)
 * `cell_comment`: all cell comment attributes. returned json string. see **attribute_name**. (**type** required `string`)
+* `cell_type`: cell type. returned Cell.getCellType() of POI.
+* `cell_cached_type`: cell cached formula result type. returned Cell.getCachedFormulaResultType() of POI when CellType==FORMULA, otherwise same as `cell_type` (returned Cell.getCellType()).
 * `sheet_name`: sheet name.
 * `row_number`: row number(1 origin).
 * `column_number`: column number(1 origin).
@@ -120,11 +128,50 @@ Processing method of Cell error (`#DIV/0!`, `#REF!`, etc).
 ```
 
 * `constant`: set null. (default)
-* `constant.`*value*: set value.
+* `constant.`*value*: set specified value.
 * `error_code`: set error code.
 * `exception`: throw exception.
 
 
+### formula_handling
+
+Processing method of formula.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, formula_handling: cashed_value}
+```
+
+* `evaluate`: evaluate formula. (default)
+* `cashed_value`: cashed value in cell.
+
+
+### on_evaluate_error
+
+Processing method of evaluate formula error.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, on_evaluate_error: constant}
+```
+
+* `constant`: set null.
+* `constant.`*value*: set value.
+* `exception`: throw exception. (default)
+
+
+### formula_replace
+
+Replace formula before evaluate.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, formula_replace: [{regex: aaa, to: "A${row}"}, {regex: bbb, to: "B${row}"}]}
+```
+
+`${row}` is replaced with the current row number.
+
+
 ### on_convert_error
 
 Processing method of convert error. ex) Excel boolean to Embulk timestamp
@@ -141,7 +188,7 @@ Processing method of convert error. ex) Excel boolean to Embulk timestamp
 
 ### sheet_options
 
-Options of indivisual sheet.
+Options of individual sheet.
 
 ```yaml
   parser:
@@ -165,7 +212,7 @@ Options of indivisual sheet.
 ```
 
 **sheet_options** is map of sheet name.  
-Map values are **skip_header_lines**, **colums**.
+Map values are **skip_header_lines**, **columns**.
 
 **columns** is map of column name.  
 Map values are same **columns** in **parser** (excluding `name`, `type`).

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.github.jruby-gradle.base" version "1.5.0"
     id "java"
     id "eclipse"
 }
@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.11"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -23,8 +23,10 @@ dependencies {
     provided "org.embulk:embulk-core:0.7.5"
     compile  "org.embulk:embulk-standards:0.7.5"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    compile group: 'org.apache.poi', name : 'poi', version: '3.13'
-    compile group: 'org.apache.poi', name : 'poi-ooxml', version: '3.13'
+    compile group: 'org.apache.poi', name : 'poi', version: '3.17'
+    compile(group: 'org.apache.poi', name : 'poi-ooxml', version: '3.17') {
+        exclude group: 'stax', module: 'stax-api'
+    }
     testCompile "junit:junit:4.+"
 }
 
@@ -36,19 +38,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 clean { delete "classpath" }
 
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "${project.name}.gemspec"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
 task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
-    script "pkg/${project.name}-${project.version}.gem"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
 }
 
-task "package"(dependsOn: ["gemspec", "classpath"]) << {
-    println "> Build succeeded."
-    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
 }
 
 task gemspec {

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Feb 04 13:46:12 PST 2015
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,20 +6,38 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelColumnValueType.java

@@ -11,6 +11,10 @@ public enum PoiExcelColumnValueType {
 	CELL_FONT(true, false),
 	/** cell comment */
 	CELL_COMMENT(true, false),
+	/** cell type */
+	CELL_TYPE(true, false),
+	/** cell CachedFormulaResultType */
+	CELL_CACHED_TYPE(true, false),
 	/** sheet name */
 	SHEET_NAME(false, false),
 	/** row number (1 origin) */

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -2,8 +2,11 @@ package org.embulk.parser.poi_excel;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
@@ -93,6 +96,11 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		@ConfigDefault("null")
 		public Optional<String> getColumnNumber();
 
+		// A1,B2,... or Sheet1!A1
+		@Config("cell_address")
+		@ConfigDefault("null")
+		public Optional<String> getCellAddress();
+
 		// use when value_type=cell_style, cell_font, ...
 		@Config("attribute_name")
 		@ConfigDefault("null")
@@ -100,11 +108,19 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 	}
 
 	public interface ColumnCommonOptionTask extends Task {
+		// format of numeric(double) to string
+		@Config("numeric_format")
+		@ConfigDefault("null")
+		public Optional<String> getNumericFormat();
 
 		// search merged cell if cellType=BLANK
 		@Config("search_merged_cell")
 		@ConfigDefault("null")
-		public Optional<Boolean> getSearchMergedCell();
+		public Optional<String> getSearchMergedCell();
+
+		@Config("formula_handling")
+		@ConfigDefault("null")
+		public Optional<String> getFormulaHandling();
 
 		@Config("formula_replace")
 		@ConfigDefault("null")
@@ -166,9 +182,60 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 					throw new RuntimeException(e);
 				}
 
-				run(task, schema, workbook, sheetNames, output);
+				List<String> list = resolveSheetName(workbook, sheetNames);
+				if (log.isDebugEnabled()) {
+					log.debug("resolved sheet names={}", list);
+				}
+				run(task, schema, workbook, list, output);
+			}
+		}
+	}
+
+	private List<String> resolveSheetName(Workbook workbook, List<String> sheetNames) {
+		Set<String> set = new LinkedHashSet<>();
+		for (String s : sheetNames) {
+			if (s.contains("*") || s.contains("?")) {
+				int length = s.length();
+				StringBuilder sb = new StringBuilder(length * 2);
+				StringBuilder buf = new StringBuilder(32);
+				for (int i = 0; i < length;) {
+					int c = s.codePointAt(i);
+					switch (c) {
+					case '*':
+						if (buf.length() > 0) {
+							sb.append(Pattern.quote(buf.toString()));
+							buf.setLength(0);
+						}
+						sb.append(".*");
+						break;
+					case '?':
+						if (buf.length() > 0) {
+							sb.append(Pattern.quote(buf.toString()));
+							buf.setLength(0);
+						}
+						sb.append(".");
+						break;
+					default:
+						buf.appendCodePoint(c);
+						break;
+					}
+					i += Character.charCount(c);
+				}
+				if (buf.length() > 0) {
+					sb.append(Pattern.quote(buf.toString()));
+				}
+				String regex = sb.toString();
+				for (Sheet sheet : workbook) {
+					String name = sheet.getSheetName();
+					if (name.matches(regex)) {
+						set.add(name);
+					}
+				}
+			} else {
+				set.add(s);
 			}
 		}
+		return new ArrayList<>(set);
 	}
 
 	protected void run(PluginTask task, Schema schema, Workbook workbook, List<String> sheetNames, PageOutput output) {
@@ -193,18 +260,28 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 
 				int count = 0;
 				for (Row row : sheet) {
-					if (row.getRowNum() < skipHeaderLines) {
+					int rowIndex = row.getRowNum();
+					if (rowIndex < skipHeaderLines) {
+						log.debug("row({}) skipped", rowIndex);
 						continue;
 					}
+					if (log.isDebugEnabled()) {
+						log.debug("row({}) start", rowIndex);
+					}
 
 					visitor.setRow(row);
 					schema.visitColumns(visitor);
 					pageBuilder.addRecord();
 
 					if (++count >= flushCount) {
+						log.trace("flush");
 						pageBuilder.flush();
 						count = 0;
 					}
+
+					if (log.isDebugEnabled()) {
+						log.debug("row({}) end", rowIndex);
+					}
 				}
 				pageBuilder.flush();
 			}