embulk-parser-poi_excel 0.1.4 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be106894c6b2167dd2f275cf8696003cc82195c3
-  data.tar.gz: c6f3718249f56278a0a29dd50521216765838c51
+  metadata.gz: '058cae76dc2aff62982491ddd1d29f0bc707d030'
+  data.tar.gz: 4b7652888f66faa0b3b3653cda9c3ba4b9d49887
 SHA512:
-  metadata.gz: 6c88d6b4604a07b40681a44290bdf75c7f8c10e6e857da5ed254fe7828fc796e54f84e0f37de1e5c9f0089c934dd91283783b95849180b620cf1180a70d8a419
-  data.tar.gz: 772b8d21d4e1587acd97886a79b284a1db42d8a12ac30660a0d2eba9db61292839ceeef39f65d8756e0ee4aea58c7a3c9e3785fcda31c51f01708b116497c6d7
+  metadata.gz: b700c88e34a05edd16d1d7444053540fae8f8ef7d596c709505f070ca4d3d0582e9c69da6cc253b97a595cbe5d3aad60f1a1bbb75b90110e573575ba32c4d752
+  data.tar.gz: a36bc92ebd495a908f98c53cb92ebd3c84c294df3cde9d2ea926f0c452c9a6d2fcce7646f9f906a6ed25b21fb14de1a2ecc6ea5ee6e8aeec10c8b46346efd070

data/README.md

@@ -28,12 +28,12 @@ in:
 ```
 
 if omit **value**, specified `cell_value`.  
-if omit **column_number** when **valus** is `cell_value`, specified next column.  
+if omit **column_number** when **value** is `cell_value`, specified next column.  
 
 
 ## Configuration
 
-* **sheets**: sheet name. (list of string, required)
+* **sheets**: sheet name. can use wildcards `*`, `?`. (list of string, required)
 * **skip_header_lines**: skip rows. (integer, default: `0`)
 * **columns**: column definition. see below. (hash, required)
 * **sheet_options**: sheet option. see below. (hash, default: null)
@@ -44,9 +44,15 @@ if omit **column_number** when **valus** is `cell_value`, specified next column.
 * **type**: Embulk column type. (string, required)
 * **value**: value type. see below. (string, defualt: `cell_value`)
 * **column_number**: Excel column number. see below. (string, default: next column)
+* **cell_address**: Excel cell address such as `A1`, `Sheet1!B3`. only one of `column_number`, `cell_address` can be specified. (string, not required)
+* **numeric_format**: format of numeric(double) to string such as `%4.2f`. (default: Java's Double.toString())
 * **attribute_name**: use with value `cell_style`, `cell_font`, etc. see below. (list of string)
 * **on_cell_error**: processing method of Cell error. see below. (string, default: `constant`)
+* **formula_handling**: processing method of formula. see below. (`evaluate` or `cashed_value`. default: `evaluate`)
+* **on_evaluate_error**: processing method of evaluate formula error. see below. (string, default: `exception`)
+* **formula_replace**: replace formula before evaluate. see below.
 * **on_convert_error**: processing method of convert error. see below. (string, default: `exception`)
+* **search_merged_cell**: search merged cell when cell is BLANK. one of `none`, `linear_search`, `tree_search`. (string, default: `tree_search`)
 
 ### value
 
@@ -55,6 +61,8 @@ if omit **column_number** when **valus** is `cell_value`, specified next column.
 * `cell_style`: all cell style attributes. returned json string. see **attribute_name**. (**type** required `string`)
 * `cell_font`: all cell font attributes. returned json string. see **attribute_name**. (**type** required `string`)
 * `cell_comment`: all cell comment attributes. returned json string. see **attribute_name**. (**type** required `string`)
+* `cell_type`: cell type. returned Cell.getCellType() of POI.
+* `cell_cached_type`: cell cached formula result type. returned Cell.getCachedFormulaResultType() of POI when CellType==FORMULA, otherwise same as `cell_type` (returned Cell.getCellType()).
 * `sheet_name`: sheet name.
 * `row_number`: row number(1 origin).
 * `column_number`: column number(1 origin).
@@ -120,11 +128,50 @@ Processing method of Cell error (`#DIV/0!`, `#REF!`, etc).
 ```
 
 * `constant`: set null. (default)
-* `constant.`*value*: set value.
+* `constant.`*value*: set specified value.
 * `error_code`: set error code.
 * `exception`: throw exception.
 
 
+### formula_handling
+
+Processing method of formula.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, formula_handling: cashed_value}
+```
+
+* `evaluate`: evaluate formula. (default)
+* `cashed_value`: cashed value in cell.
+
+
+### on_evaluate_error
+
+Processing method of evaluate formula error.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, on_evaluate_error: constant}
+```
+
+* `constant`: set null.
+* `constant.`*value*: set value.
+* `exception`: throw exception. (default)
+
+
+### formula_replace
+
+Replace formula before evaluate.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, formula_replace: [{regex: aaa, to: "A${row}"}, {regex: bbb, to: "B${row}"}]}
+```
+
+`${row}` is replaced with the current row number.
+
+
 ### on_convert_error
 
 Processing method of convert error. ex) Excel boolean to Embulk timestamp
@@ -141,7 +188,7 @@ Processing method of convert error. ex) Excel boolean to Embulk timestamp
 
 ### sheet_options
 
-Options of indivisual sheet.
+Options of individual sheet.
 
 ```yaml
   parser:
@@ -165,7 +212,7 @@ Options of indivisual sheet.
 ```
 
 **sheet_options** is map of sheet name.  
-Map values are **skip_header_lines**, **colums**.
+Map values are **skip_header_lines**, **columns**.
 
 **columns** is map of column name.  
 Map values are same **columns** in **parser** (excluding `name`, `type`).

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.github.jruby-gradle.base" version "1.5.0"
     id "java"
     id "eclipse"
 }
@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.10"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -23,8 +23,10 @@ dependencies {
     provided "org.embulk:embulk-core:0.7.5"
     compile  "org.embulk:embulk-standards:0.7.5"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    compile group: 'org.apache.poi', name : 'poi', version: '3.13'
-    compile group: 'org.apache.poi', name : 'poi-ooxml', version: '3.13'
+    compile group: 'org.apache.poi', name : 'poi', version: '3.17'
+    compile(group: 'org.apache.poi', name : 'poi-ooxml', version: '3.17') {
+        exclude group: 'stax', module: 'stax-api'
+    }
     testCompile "junit:junit:4.+"
 }
 
@@ -36,14 +38,22 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 clean { delete "classpath" }
 
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "${project.name}.gemspec"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
 task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
-    script "pkg/${project.name}-${project.version}.gem"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
+}
+
+task gemUnpack(type: JRubyExec) {
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "unpack", "pkg/${project.name}-${project.version}.gem"
 }
 
 task "package"(dependsOn: ["gemspec", "classpath"]) << {

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelColumnValueType.java

@@ -11,6 +11,10 @@ public enum PoiExcelColumnValueType {
 	CELL_FONT(true, false),
 	/** cell comment */
 	CELL_COMMENT(true, false),
+	/** cell type */
+	CELL_TYPE(true, false),
+	/** cell CachedFormulaResultType */
+	CELL_CACHED_TYPE(true, false),
 	/** sheet name */
 	SHEET_NAME(false, false),
 	/** row number (1 origin) */

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -2,8 +2,11 @@ package org.embulk.parser.poi_excel;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
@@ -93,6 +96,11 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		@ConfigDefault("null")
 		public Optional<String> getColumnNumber();
 
+		// A1,B2,... or Sheet1!A1
+		@Config("cell_address")
+		@ConfigDefault("null")
+		public Optional<String> getCellAddress();
+
 		// use when value_type=cell_style, cell_font, ...
 		@Config("attribute_name")
 		@ConfigDefault("null")
@@ -100,11 +108,19 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 	}
 
 	public interface ColumnCommonOptionTask extends Task {
+		// format of numeric(double) to string
+		@Config("numeric_format")
+		@ConfigDefault("null")
+		public Optional<String> getNumericFormat();
 
 		// search merged cell if cellType=BLANK
 		@Config("search_merged_cell")
 		@ConfigDefault("null")
-		public Optional<Boolean> getSearchMergedCell();
+		public Optional<String> getSearchMergedCell();
+
+		@Config("formula_handling")
+		@ConfigDefault("null")
+		public Optional<String> getFormulaHandling();
 
 		@Config("formula_replace")
 		@ConfigDefault("null")
@@ -166,9 +182,60 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 					throw new RuntimeException(e);
 				}
 
-				run(task, schema, workbook, sheetNames, output);
+				List<String> list = resolveSheetName(workbook, sheetNames);
+				if (log.isDebugEnabled()) {
+					log.debug("resolved sheet names={}", list);
+				}
+				run(task, schema, workbook, list, output);
+			}
+		}
+	}
+
+	private List<String> resolveSheetName(Workbook workbook, List<String> sheetNames) {
+		Set<String> set = new LinkedHashSet<>();
+		for (String s : sheetNames) {
+			if (s.contains("*") || s.contains("?")) {
+				int length = s.length();
+				StringBuilder sb = new StringBuilder(length * 2);
+				StringBuilder buf = new StringBuilder(32);
+				for (int i = 0; i < length;) {
+					int c = s.codePointAt(i);
+					switch (c) {
+					case '*':
+						if (buf.length() > 0) {
+							sb.append(Pattern.quote(buf.toString()));
+							buf.setLength(0);
+						}
+						sb.append(".*");
+						break;
+					case '?':
+						if (buf.length() > 0) {
+							sb.append(Pattern.quote(buf.toString()));
+							buf.setLength(0);
+						}
+						sb.append(".");
+						break;
+					default:
+						buf.appendCodePoint(c);
+						break;
+					}
+					i += Character.charCount(c);
+				}
+				if (buf.length() > 0) {
+					sb.append(Pattern.quote(buf.toString()));
+				}
+				String regex = sb.toString();
+				for (Sheet sheet : workbook) {
+					String name = sheet.getSheetName();
+					if (name.matches(regex)) {
+						set.add(name);
+					}
+				}
+			} else {
+				set.add(s);
 			}
 		}
+		return new ArrayList<>(set);
 	}
 
 	protected void run(PluginTask task, Schema schema, Workbook workbook, List<String> sheetNames, PageOutput output) {
@@ -193,18 +260,28 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 
 				int count = 0;
 				for (Row row : sheet) {
-					if (row.getRowNum() < skipHeaderLines) {
+					int rowIndex = row.getRowNum();
+					if (rowIndex < skipHeaderLines) {
+						log.debug("row({}) skipped", rowIndex);
 						continue;
 					}
+					if (log.isDebugEnabled()) {
+						log.debug("row({}) start", rowIndex);
+					}
 
 					visitor.setRow(row);
 					schema.visitColumns(visitor);
 					pageBuilder.addRecord();
 
 					if (++count >= flushCount) {
+						log.trace("flush");
 						pageBuilder.flush();
 						count = 0;
 					}
+
+					if (log.isDebugEnabled()) {
+						log.debug("row({}) end", rowIndex);
+					}
 				}
 				pageBuilder.flush();
 			}

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelColumnBean.java

@@ -11,6 +11,7 @@ import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnCommonOptionTask;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.FormulaReplaceTask;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.ErrorStrategy.Strategy;
+import org.embulk.parser.poi_excel.bean.util.PoiExcelCellAddress;
 import org.embulk.spi.Column;
 
 import com.google.common.base.Optional;
@@ -107,6 +108,25 @@ public class PoiExcelColumnBean {
 		return Optional.absent();
 	}
 
+	private Optional<PoiExcelCellAddress> cellAddress;
+
+	public PoiExcelCellAddress getCellAddress() {
+		if (cellAddress == null) {
+			this.cellAddress = initializeCellAddress();
+		}
+		return cellAddress.orNull();
+	}
+
+	protected Optional<PoiExcelCellAddress> initializeCellAddress() {
+		for (ColumnOptionTask task : columnTaskList) {
+			Optional<String> option = task.getCellAddress();
+			if (option.isPresent()) {
+				return Optional.of(new PoiExcelCellAddress(option.get()));
+			}
+		}
+		return Optional.absent();
+	}
+
 	protected abstract class CacheValue<T> {
 		private T value;
 
@@ -231,23 +251,100 @@ public class PoiExcelColumnBean {
 		return attributeName.get();
 	}
 
-	private CacheValue<Boolean> searchMergedCell = new CacheValue<Boolean>() {
+	private CacheValue<String> numericFormat = new CacheValue<String>() {
 
 		@Override
-		protected Optional<Boolean> getTaskValue(ColumnCommonOptionTask task) {
-			return task.getSearchMergedCell();
+		protected Optional<String> getTaskValue(ColumnCommonOptionTask task) {
+			return task.getNumericFormat();
 		}
 
 		@Override
-		protected Boolean getDefaultValue() {
-			return true;
+		protected String getDefaultValue() {
+			return "";
 		}
 	};
 
-	public boolean getSearchMergedCell() {
+	public String getNumericFormat() {
+		return numericFormat.get();
+	}
+
+	public enum SearchMergedCell {
+		NONE, LINEAR_SEARCH, TREE_SEARCH
+	}
+
+	private CacheValue<SearchMergedCell> searchMergedCell = new CacheValue<SearchMergedCell>() {
+
+		@Override
+		protected Optional<SearchMergedCell> getTaskValue(ColumnCommonOptionTask task) {
+			Optional<String> option = task.getSearchMergedCell();
+			String value = option.or("null").trim();
+			switch (value.toLowerCase()) {
+			case "null":
+				return Optional.absent();
+			case "true": // compatibility ver 0.1.7
+				return Optional.of(getDefaultValue());
+			case "false": // compatibility ver 0.1.7
+				return Optional.of(SearchMergedCell.NONE);
+			default:
+				break;
+			}
+			try {
+				return Optional.of(SearchMergedCell.valueOf(value.toUpperCase()));
+			} catch (Exception e) {
+				List<String> list = new ArrayList<>();
+				for (SearchMergedCell s : SearchMergedCell.values()) {
+					list.add(s.name().toLowerCase());
+				}
+				throw new ConfigException(MessageFormat.format("illegal search_merged_cell={0}. expected={1}", value,
+						list), e);
+			}
+		}
+
+		@Override
+		protected SearchMergedCell getDefaultValue() {
+			return SearchMergedCell.TREE_SEARCH;
+		}
+	};
+
+	public SearchMergedCell getSearchMergedCell() {
 		return searchMergedCell.get();
 	}
 
+	public enum FormulaHandling {
+		EVALUATE, CASHED_VALUE
+	}
+
+	private CacheValue<FormulaHandling> formulaHandling = new CacheValue<FormulaHandling>() {
+
+		@Override
+		protected Optional<FormulaHandling> getTaskValue(ColumnCommonOptionTask task) {
+			Optional<String> option = task.getFormulaHandling();
+			String value = option.or("null");
+			if ("null".equalsIgnoreCase(value)) {
+				return Optional.absent();
+			}
+			try {
+				return Optional.of(FormulaHandling.valueOf(value.trim().toUpperCase()));
+			} catch (Exception e) {
+				List<String> list = new ArrayList<>();
+				for (FormulaHandling s : FormulaHandling.values()) {
+					list.add(s.name().toLowerCase());
+				}
+				throw new ConfigException(MessageFormat.format("illegal formula_handling={0}. expected={1}", value,
+						list), e);
+			}
+		}
+
+		@Override
+		protected FormulaHandling getDefaultValue() {
+			return FormulaHandling.EVALUATE;
+		}
+	};
+
+	public FormulaHandling getFormulaHandling() {
+		return formulaHandling.get();
+	}
+
 	private CacheValue<List<FormulaReplaceTask>> formulaReplace = new CacheValue<List<FormulaReplaceTask>>() {
 
 		@Override