embulk-parser-poi_excel 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6b3e37c337cca59783620b59c08c46eb873188d
-  data.tar.gz: 6648241ed3545ccf5e134412274a5dfa25a41903
+  metadata.gz: 5a6920bea2e679cbcc53f02a2c70485ba3d4b350
+  data.tar.gz: a83e9e1a2cb8ef78da34d7ef5602b66b3f2e48c8
 SHA512:
-  metadata.gz: 3c115aa89954e182773b200e09dd90d091d8f3a8ba274bd1fd2b30b6cfdceb857b65ff2b2cdf375e4cbfc4a73bb46d8ecb3e73ea6ed823d53e1306ce3c4809b8
-  data.tar.gz: 60461a21d343976e53ef726dc6a893572fe041849c7f7ed1cdb3f8511584f54c7956ce3716cbec7a40db7a89717358092c891d487c4e1c1431107180f3ade95a
+  metadata.gz: 33920df4d360e86b5609f3939c63762b2f1d9b87a16f1923cf59ab5f16060ce1a2070c860e66d46fd50631a6a08f7677c7f32cb3dfc33e2084f1aca6fab75fe7
+  data.tar.gz: 043b272a531e072b07033c43a014021c8cf9e1e65a7e12f7079341cff35adf6dc429e30c3da3415f994698fa02aa401fd527771469094db93583408c081c2c31

data/README.md

@@ -28,12 +28,12 @@ in:
 ```
 
 if omit **value**, specified `cell_value`.  
-if omit **column_number** when **valus** is `cell_value`, specified next column.  
+if omit **column_number** when **value** is `cell_value`, specified next column.  
 
 
 ## Configuration
 
-* **sheets**: sheet name. (list of string, required)
+* **sheets**: sheet name. can use wildcards `*`, `?`. (list of string, required)
 * **skip_header_lines**: skip rows. (integer, default: `0`)
 * **columns**: column definition. see below. (hash, required)
 * **sheet_options**: sheet option. see below. (hash, default: null)
@@ -46,6 +46,9 @@ if omit **column_number** when **valus** is `cell_value`, specified next column.
 * **column_number**: Excel column number. see below. (string, default: next column)
 * **attribute_name**: use with value `cell_style`, `cell_font`, etc. see below. (list of string)
 * **on_cell_error**: processing method of Cell error. see below. (string, default: `constant`)
+* **formula_handling** : processing method of formula. see below. (`evaluate` or `cashed_value`. default: `evaluate`)
+* **on_evaluate_error**: processing method of evaluate formula error. see below. (string, default: `exception`)
+* **formula_replace** : replace formula before evaluate. see below.
 * **on_convert_error**: processing method of convert error. see below. (string, default: `exception`)
 
 ### value
@@ -125,6 +128,45 @@ Processing method of Cell error (`#DIV/0!`, `#REF!`, etc).
 * `exception`: throw exception.
 
 
+### formula_handling
+
+Processing method of formula.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, formula_handling: cashed_value}
+```
+
+* `evaluate`: evaluate formula. (default)
+* `cashed_value`: cashed value in cell.
+
+
+### on_evaluate_error
+
+Processing method of evaluate formula error.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, on_evaluate_error: constant}
+```
+
+* `constant`: set null.
+* `constant.`*value*: set value.
+* `exception`: throw exception. (default)
+
+
+### formula_replace
+
+Replace formula before evaluate.
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, formula_replace: [{regex: aaa, to: "A${row}"}, {regex: bbb, to: "B${row}"}]}
+```
+
+`${row}` is replaced with the current row number.
+
+
 ### on_convert_error
 
 Processing method of convert error. ex) Excel boolean to Embulk timestamp

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.5"
+version = "0.1.6"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -2,8 +2,11 @@ package org.embulk.parser.poi_excel;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
@@ -106,6 +109,10 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		@ConfigDefault("null")
 		public Optional<Boolean> getSearchMergedCell();
 
+		@Config("formula_handling")
+		@ConfigDefault("null")
+		public Optional<String> getFormulaHandling();
+
 		@Config("formula_replace")
 		@ConfigDefault("null")
 		public Optional<List<FormulaReplaceTask>> getFormulaReplace();
@@ -166,11 +173,62 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 					throw new RuntimeException(e);
 				}
 
-				run(task, schema, workbook, sheetNames, output);
+				List<String> list = resolveSheetName(workbook, sheetNames);
+				if (log.isDebugEnabled()) {
+					log.debug("resolved sheet names={}", list);
+				}
+				run(task, schema, workbook, list, output);
 			}
 		}
 	}
 
+	private List<String> resolveSheetName(Workbook workbook, List<String> sheetNames) {
+		Set<String> set = new LinkedHashSet<>();
+		for (String s : sheetNames) {
+			if (s.contains("*") || s.contains("?")) {
+				int length = s.length();
+				StringBuilder sb = new StringBuilder(length * 2);
+				StringBuilder buf = new StringBuilder(32);
+				for (int i = 0; i < length;) {
+					int c = s.codePointAt(i);
+					switch (c) {
+					case '*':
+						if (buf.length() > 0) {
+							sb.append(Pattern.quote(buf.toString()));
+							buf.setLength(0);
+						}
+						sb.append(".*");
+						break;
+					case '?':
+						if (buf.length() > 0) {
+							sb.append(Pattern.quote(buf.toString()));
+							buf.setLength(0);
+						}
+						sb.append(".");
+						break;
+					default:
+						buf.appendCodePoint(c);
+						break;
+					}
+					i += Character.charCount(c);
+				}
+				if (buf.length() > 0) {
+					sb.append(Pattern.quote(buf.toString()));
+				}
+				String regex = sb.toString();
+				for (Sheet sheet : workbook) {
+					String name = sheet.getSheetName();
+					if (name.matches(regex)) {
+						set.add(name);
+					}
+				}
+			} else {
+				set.add(s);
+			}
+		}
+		return new ArrayList<>(set);
+	}
+
 	protected void run(PluginTask task, Schema schema, Workbook workbook, List<String> sheetNames, PageOutput output) {
 		final int flushCount = task.getFlushCount();
 
@@ -193,18 +251,28 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 
 				int count = 0;
 				for (Row row : sheet) {
-					if (row.getRowNum() < skipHeaderLines) {
+					int rowIndex = row.getRowNum();
+					if (rowIndex < skipHeaderLines) {
+						log.debug("row({}) skipped", rowIndex);
 						continue;
 					}
+					if (log.isDebugEnabled()) {
+						log.debug("row({}) start", rowIndex);
+					}
 
 					visitor.setRow(row);
 					schema.visitColumns(visitor);
 					pageBuilder.addRecord();
 
 					if (++count >= flushCount) {
+						log.trace("flush");
 						pageBuilder.flush();
 						count = 0;
 					}
+
+					if (log.isDebugEnabled()) {
+						log.debug("row({}) end", rowIndex);
+					}
 				}
 				pageBuilder.flush();
 			}

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelColumnBean.java

@@ -248,6 +248,41 @@ public class PoiExcelColumnBean {
 		return searchMergedCell.get();
 	}
 
+	public enum FormulaHandling {
+		EVALUATE, CASHED_VALUE
+	}
+
+	private CacheValue<FormulaHandling> formulaHandling = new CacheValue<FormulaHandling>() {
+
+		@Override
+		protected Optional<FormulaHandling> getTaskValue(ColumnCommonOptionTask task) {
+			Optional<String> option = task.getFormulaHandling();
+			String value = option.or("null");
+			if ("null".equalsIgnoreCase(value)) {
+				return Optional.absent();
+			}
+			try {
+				return Optional.of(FormulaHandling.valueOf(value.trim().toUpperCase()));
+			} catch (Exception e) {
+				List<String> list = new ArrayList<>();
+				for (FormulaHandling s : FormulaHandling.values()) {
+					list.add(s.name().toLowerCase());
+				}
+				throw new ConfigException(MessageFormat.format("illegal formula_handling={0}. expected={1}", value,
+						list), e);
+			}
+		}
+
+		@Override
+		protected FormulaHandling getDefaultValue() {
+			return FormulaHandling.EVALUATE;
+		}
+	};
+
+	public FormulaHandling getFormulaHandling() {
+		return formulaHandling.get();
+	}
+
 	private CacheValue<List<FormulaReplaceTask>> formulaReplace = new CacheValue<List<FormulaReplaceTask>>() {
 
 		@Override

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellValueVisitor.java

@@ -16,6 +16,7 @@ import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.FormulaReplaceTask;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.ErrorStrategy;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.FormulaHandling;
 import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
@@ -109,6 +110,44 @@ public class PoiExcelCellValueVisitor {
 	protected void visitCellValueFormula(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
 		assert cell.getCellType() == Cell.CELL_TYPE_FORMULA;
 
+		FormulaHandling handling = bean.getFormulaHandling();
+		switch (handling) {
+		case CASHED_VALUE:
+			visitCellValueFormulaCashedValue(bean, cell, visitor);
+			break;
+		default:
+			visitCellValueFormulaEvaluate(bean, cell, visitor);
+			break;
+		}
+	}
+
+	protected void visitCellValueFormulaCashedValue(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
+		Column column = bean.getColumn();
+
+		int cellType = cell.getCachedFormulaResultType();
+		switch (cellType) {
+		case Cell.CELL_TYPE_NUMERIC:
+			visitor.visitCellValueNumeric(column, cell, cell.getNumericCellValue());
+			return;
+		case Cell.CELL_TYPE_STRING:
+			visitor.visitCellValueString(column, cell, cell.getStringCellValue());
+			return;
+		case Cell.CELL_TYPE_BLANK:
+			visitCellValueBlank(bean, cell, visitor);
+			return;
+		case Cell.CELL_TYPE_BOOLEAN:
+			visitor.visitCellValueBoolean(column, cell, cell.getBooleanCellValue());
+			return;
+		case Cell.CELL_TYPE_ERROR:
+			visitCellValueError(bean, cell, cell.getErrorCellValue(), visitor);
+			return;
+		case Cell.CELL_TYPE_FORMULA:
+		default:
+			throw new IllegalStateException(MessageFormat.format("unsupported POI cellType={0}", cellType));
+		}
+	}
+
+	protected void visitCellValueFormulaEvaluate(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
 		Column column = bean.getColumn();
 
 		List<FormulaReplaceTask> list = bean.getFormulaReplace();

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelColumnVisitor.java

@@ -10,9 +10,12 @@ import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
+import org.slf4j.Logger;
 
 public class PoiExcelColumnVisitor implements ColumnVisitor {
+	private final Logger log = Exec.getLogger(getClass());
 
 	protected final PoiExcelVisitorValue visitorValue;
 	protected final PageBuilder pageBuilder;
@@ -56,6 +59,9 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 	}
 
 	protected final void visitCell0(Column column, CellVisitor visitor) {
+		if (log.isTraceEnabled()) {
+			log.trace("{} start", column);
+		}
 		try {
 			visitCell(column, visitor);
 		} catch (Exception e) {
@@ -65,6 +71,9 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 			throw new RuntimeException(MessageFormat.format("error at {0} cell={1}!{2}. {3}", column, sheetName, ref,
 					e.getMessage()), e);
 		}
+		if (log.isTraceEnabled()) {
+			log.trace("{} end", column);
+		}
 	}
 
 	protected void visitCell(Column column, CellVisitor visitor) {

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin.java

@@ -10,7 +10,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.TimeZone;
 
-import org.embulk.config.ConfigSource;
 import org.embulk.parser.EmbulkPluginTester;
 import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
 import org.embulk.parser.EmbulkTestParserConfig;
@@ -114,33 +113,6 @@ public class TestPoiExcelParserPlugin {
 		assertThat(r.getAsString("col-s"), is("A"));
 	}
 
-	@Theory
-	public void testForumlaReplace(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "formula_replace");
-
-			ConfigSource replace0 = tester.newConfigSource();
-			replace0.set("regex", "test1");
-			replace0.set("to", "merged_cell");
-			ConfigSource replace1 = tester.newConfigSource();
-			replace1.set("regex", "B1");
-			replace1.set("to", "B${row}");
-			parser.set("formula_replace", Arrays.asList(replace0, replace1));
-
-			parser.addColumn("text", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(2));
-			assertThat(result.get(0).getAsString("text"), is("test3-a1"));
-			assertThat(result.get(1).getAsString("text"), is("test2-b2"));
-		}
-	}
-
 	@Theory
 	public void testSearchMergedCell_true(String excelFile) throws ParseException {
 		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_formula.java

@@ -0,0 +1,90 @@
+package org.embulk.parser.poi_excel;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.net.URL;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.parser.EmbulkPluginTester;
+import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
+import org.embulk.parser.EmbulkTestParserConfig;
+import org.junit.experimental.theories.DataPoints;
+import org.junit.experimental.theories.Theories;
+import org.junit.experimental.theories.Theory;
+import org.junit.runner.RunWith;
+
+@RunWith(Theories.class)
+public class TestPoiExcelParserPlugin_formula {
+
+	@DataPoints
+	public static String[] FILES = { "test1.xls", "test2.xlsx" };
+
+	@Theory
+	public void testForumlaHandlingCashedValue(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheet", "formula_replace");
+
+			parser.addColumn("text", "string").set("formula_handling", "cashed_value");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(2));
+			assertThat(result.get(0).getAsString("text"), is("boolean"));
+			assertThat(result.get(1).getAsString("text"), is("test2-b1"));
+		}
+	}
+
+	@Theory
+	public void testForumlaHandlingEvaluate(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheet", "formula_replace");
+
+			parser.addColumn("text", "string").set("formula_handling", "evaluate");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(2));
+			assertThat(result.get(0).getAsString("text"), is("boolean"));
+			assertThat(result.get(1).getAsString("text"), is("test2-b1"));
+		}
+	}
+
+	@Theory
+	public void testForumlaReplace(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheet", "formula_replace");
+
+			ConfigSource replace0 = tester.newConfigSource();
+			replace0.set("regex", "test1");
+			replace0.set("to", "merged_cell");
+			ConfigSource replace1 = tester.newConfigSource();
+			replace1.set("regex", "B1");
+			replace1.set("to", "B${row}");
+			parser.set("formula_replace", Arrays.asList(replace0, replace1));
+
+			parser.addColumn("text", "string");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(2));
+			assertThat(result.get(0).getAsString("text"), is("test3-a1"));
+			assertThat(result.get(1).getAsString("text"), is("test2-b2"));
+		}
+	}
+}

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_sheets.java

@@ -89,4 +89,38 @@ public class TestPoiExcelParserPlugin_sheets {
 		assertThat(record.getAsString("text"), is(text));
 		assertThat(record.getAsLong("number"), is(number));
 	}
+
+	@Theory
+	public void testResolveSheetName1(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("*es*"));
+			parser.addColumn("name", "string").set("value", "sheet_name");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			OutputRecord record = result.get(0);
+			assertThat(record.getAsString("name"), is("test1"));
+		}
+	}
+
+	@Theory
+	public void testResolveSheetName2(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test?"));
+			parser.addColumn("name", "string").set("value", "sheet_name");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			OutputRecord record = result.get(0);
+			assertThat(record.getAsString("name"), is("test1"));
+		}
+	}
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-poi_excel
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - hishidama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-11 00:00:00.000000000 Z
+date: 2017-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -88,11 +88,12 @@ files:
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_columnNumber.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_constant.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_convertError.java
+- src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_formula.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_sheets.java
 - src/test/resources/org/embulk/parser/poi_excel/test1.xls
 - src/test/resources/org/embulk/parser/poi_excel/test2.xlsx
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-parser-poi_excel-0.1.5.jar
+- classpath/embulk-parser-poi_excel-0.1.6.jar
 - classpath/embulk-standards-0.7.5.jar
 - classpath/poi-3.13.jar
 - classpath/poi-ooxml-3.13.jar