embulk-parser-poi_excel 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17272dd82557dd34357f3407c8d4eb7658e31ed2
-  data.tar.gz: e135d924eae7de15e5da96c381511a38f2d35426
+  metadata.gz: 3d1be437da5c3e86a63ef4c925d3ba11218c5344
+  data.tar.gz: eb1b92328a4ac0c9619165bf046289538223e2ed
 SHA512:
-  metadata.gz: 7922a700f58a63aefb895334ab5483da14e4f1c36dce2beac9868de28e8c61f88ce34487c660f804d9540b176afcf73258d12c9035867271d6a020ccfe63d217
-  data.tar.gz: 4908655995c8abe3dc4bd552defb4a8142eecd093ebd5bb0b728f06030737d764e2851a31bfa479d87d3469b61a18d8beef57cc0cd40f919c34538aefd9e056a
+  metadata.gz: de797f9fd1895bb79987ae4564d0a4288a7a4c4115b217194462d9f5912912b0c814214c6319a363923953feeeb62f70cc2e4261b2a27695b1a26e6b60324c5c
+  data.tar.gz: 8db340bcd1e3a46f54bd62513620c3d067432db0b5c28f71d9235f2dd3ae1aa5d0c37753ae1f1488f868e1915129c814d61d306dd58e9d58a602dcadd2ef9795

data/README.md

@@ -21,6 +21,8 @@ This plugin uses Apache POI.
 * **value**: value type. see below. (string, defualt: `cell_value`)
 * **column_number**: Excel column number. see below. (string, default: next column)
 * **attribute_name**: use with value `cell_style`, `cell_font`, etc. see below. (list of string)
+* **on_cell_error**: processing method of Cell error. see below. (string, default: `constant`)
+* **on_convert_error**: processing method of convert error. see below. (string, default: `exception`)
 
 ### value
 
@@ -32,6 +34,10 @@ This plugin uses Apache POI.
 * `sheet_name`: sheet name.
 * `row_number`: row number(1 origin).
 * `column_number`: column number(1 origin).
+* `constant`: constant value.
+
+  * `constant.`*value*: specified value.
+  * `constant`: null.
 
 ### column_number
 
@@ -80,6 +86,35 @@ attribute_nameを指定することで、指定された属性だけを取得し
 （`cell_value`では、**column_number**を省略すると次の列に移る）
 
 
+### on_cell_error
+
+Processing method of Cell error (`#DIV/0!`, `#REF!`, etc).
+
+```yaml
+    columns:
+    - {name: foo, type: string, column_number: A, value: cell_value, on_cell_error: error_code}
+```
+
+* `constant`: set null. (default)
+* `constant.`*value*: set value.
+* `error_code`: set error code.
+* `exception`: throw exception.
+
+
+### on_convert_error
+
+Processing method of convert error. ex) Excel boolean to Embulk timestamp
+
+```yaml
+    columns:
+    - {name: foo, type: timestamp, format: "%Y/%m/%d", column_number: A, value: cell_value, on_convert_error: constant.9999/12/31}
+```
+
+* `constant`: set null.
+* `constant.`*value*: set value.
+* `exception`: throw exception. (default)
+
+
 ## Example
 
 ```yaml

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelColumnValueType.java

@@ -17,9 +17,8 @@ public enum PoiExcelColumnValueType {
 	ROW_NUMBER(false, false),
 	/** column number (1 origin) */
 	COLUMN_NUMBER(true, false),
-
-	//
-	;
+	/** constant */
+	CONSTANT(false, false);
 
 	private final boolean useCell;
 	private final boolean nextIndex;

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.parser.poi_excel;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
@@ -38,7 +39,7 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 
 	public static final String TYPE = "poi_excel";
 
-	public interface PluginTask extends Task, TimestampParser.Task {
+	public interface PluginTask extends Task, TimestampParser.Task, SheetOptionTask {
 		@Config("sheet")
 		@ConfigDefault("null")
 		public Optional<String> getSheet();
@@ -51,66 +52,49 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		@ConfigDefault("false")
 		public boolean getIgnoreSheetNotFound();
 
-		@Config("skip_header_lines")
-		@ConfigDefault("0")
-		public int getSkipHeaderLines();
-
-		// search merged cell if cellType=BLANK
-		@Config("search_merged_cell")
-		@ConfigDefault("true")
-		public boolean getSearchMergedCell();
-
-		@Config("formula_replace")
-		@ConfigDefault("null")
-		public Optional<List<FormulaReplaceTask>> getFormulaReplace();
-
-		// true: set null if formula error
-		// false: throw exception if formula error
-		@Config("formula_error_null")
-		@ConfigDefault("false")
-		public boolean getFormulaErrorNull();
-
-		// true: set null if cell value error
-		// false: set error code if cell value error
-		@Config("cell_error_null")
-		@ConfigDefault("true")
-		public boolean getCellErrorNull();
+		@Config("sheet_options")
+		@ConfigDefault("{}")
+		public Map<String, SheetOptionTask> getSheetOptions();
 
 		@Config("flush_count")
 		@ConfigDefault("100")
 		public int getFlushCount();
+	}
+
+	public interface SheetOptionTask extends Task, ColumnCommonOptionTask {
+
+		@Config("skip_header_lines")
+		@ConfigDefault("null")
+		public Optional<Integer> getSkipHeaderLines();
 
 		@Config("columns")
 		public SchemaConfig getColumns();
 	}
 
-	public interface ColumnOptionTask extends Task {
+	public interface ColumnOptionTask extends Task, ColumnCommonOptionTask {
 
 		/**
 		 * @see PoiExcelColumnValueType
 		 * @return value_type
 		 */
 		@Config("value")
-		@ConfigDefault("\"cell_value\"")
-		public String getValueType();
-
-		public void setValueTypeEnum(PoiExcelColumnValueType valueType);
-
-		public PoiExcelColumnValueType getValueTypeEnum();
-
-		public void setValueTypeSuffix(String suffix);
-
-		public String getValueTypeSuffix();
+		@ConfigDefault("null")
+		public Optional<String> getValueType();
 
 		// A,B,... or number(1 origin)
 		@Config("column_number")
 		@ConfigDefault("null")
 		public Optional<String> getColumnNumber();
 
-		public void setColumnIndex(int index);
+		// use when value_type=cell_style, cell_font, ...
+		@Config("attribute_name")
+		@ConfigDefault("null")
+		public Optional<List<String>> getAttributeName();
+	}
 
-		public int getColumnIndex();
+	public interface ColumnCommonOptionTask extends Task {
 
+		// search merged cell if cellType=BLANK
 		@Config("search_merged_cell")
 		@ConfigDefault("null")
 		public Optional<Boolean> getSearchMergedCell();
@@ -119,18 +103,17 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		@ConfigDefault("null")
 		public Optional<List<FormulaReplaceTask>> getFormulaReplace();
 
-		@Config("formula_error_null")
+		@Config("on_evaluate_error")
 		@ConfigDefault("null")
-		public Optional<Boolean> getFormulaErrorNull();
+		public Optional<String> getOnEvaluateError();
 
-		@Config("cell_error_null")
+		@Config("on_cell_error")
 		@ConfigDefault("null")
-		public Optional<Boolean> getCellErrorNull();
+		public Optional<String> getOnCellError();
 
-		// use when value_type=cell_style, cell_font, ...
-		@Config("attribute_name")
+		@Config("on_convert_error")
 		@ConfigDefault("null")
-		public Optional<List<String>> getAttributeName();
+		public Optional<String> getOnConvertError();
 	}
 
 	public interface FormulaReplaceTask extends Task {
@@ -139,7 +122,7 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		public String getRegex();
 
 		// replace string
-		// use variable: "${row}"
+		// can use variable: "${row}"
 		@Config("to")
 		public String getTo();
 	}
@@ -182,7 +165,6 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 	}
 
 	protected void run(PluginTask task, Schema schema, Workbook workbook, List<String> sheetNames, PageOutput output) {
-		int skipHeaderLines = task.getSkipHeaderLines();
 		final int flushCount = task.getFlushCount();
 
 		try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -198,8 +180,9 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 				}
 
 				log.info("sheet={}", sheetName);
-				PoiExcelVisitorFactory factory = newPoiExcelVisitorFactory(task, sheet, pageBuilder);
+				PoiExcelVisitorFactory factory = newPoiExcelVisitorFactory(task, schema, sheet, pageBuilder);
 				PoiExcelColumnVisitor visitor = factory.getPoiExcelColumnVisitor();
+				final int skipHeaderLines = factory.getVisitorValue().getSheetBean().getSkipHeaderLines();
 
 				int count = 0;
 				for (Row row : sheet) {
@@ -222,8 +205,9 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		}
 	}
 
-	protected PoiExcelVisitorFactory newPoiExcelVisitorFactory(PluginTask task, Sheet sheet, PageBuilder pageBuilder) {
-		PoiExcelVisitorValue visitorValue = new PoiExcelVisitorValue(task, sheet, pageBuilder);
+	protected PoiExcelVisitorFactory newPoiExcelVisitorFactory(PluginTask task, Schema schema, Sheet sheet,
+			PageBuilder pageBuilder) {
+		PoiExcelVisitorValue visitorValue = new PoiExcelVisitorValue(task, schema, sheet, pageBuilder);
 		return new PoiExcelVisitorFactory(visitorValue);
 	}
 }

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelColumnBean.java

@@ -0,0 +1,300 @@
+package org.embulk.parser.poi_excel.bean;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.embulk.config.ConfigException;
+import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnCommonOptionTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.FormulaReplaceTask;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.ErrorStrategy.Strategy;
+import org.embulk.spi.Column;
+
+import com.google.common.base.Optional;
+
+public class PoiExcelColumnBean {
+
+	protected final PoiExcelSheetBean sheetBean;
+	protected final Column column;
+	protected final List<ColumnOptionTask> columnTaskList = new ArrayList<>();
+	protected final List<ColumnCommonOptionTask> allTaskList = new ArrayList<>();
+
+	private PoiExcelColumnValueType valueType;
+	private String valueTypeSuffix;
+	private int columnIndex;
+
+	public PoiExcelColumnBean(PoiExcelSheetBean sheetBean, Column column, ColumnOptionTask mainTask,
+			ColumnOptionTask optionTask) {
+		this.sheetBean = sheetBean;
+		this.column = column;
+
+		if (optionTask != null) {
+			columnTaskList.add(optionTask);
+		}
+		columnTaskList.add(mainTask);
+
+		allTaskList.addAll(columnTaskList);
+		allTaskList.addAll(sheetBean.getSheetOption());
+
+		initialize();
+	}
+
+	private void initialize() {
+		String type = null;
+		for (ColumnOptionTask task : columnTaskList) {
+			Optional<String> option = task.getValueType();
+			if (option.isPresent()) {
+				type = option.get();
+				break;
+			}
+		}
+		if (type == null) {
+			this.valueType = PoiExcelColumnValueType.CELL_VALUE;
+			return;
+		}
+
+		String suffix = null;
+		{
+			int n = type.indexOf('.');
+			if (n >= 0) {
+				suffix = type.substring(n + 1); // not trim
+				this.valueTypeSuffix = suffix.trim();
+				type = type.substring(0, n).trim();
+			}
+		}
+
+		try {
+			this.valueType = PoiExcelColumnValueType.valueOf(type.toUpperCase());
+		} catch (Exception e) {
+			throw new ConfigException(MessageFormat.format("illegal value_type={0}", type), e);
+		}
+
+		if (valueType == PoiExcelColumnValueType.CONSTANT) {
+			this.valueTypeSuffix = suffix; // not trim
+		}
+	}
+
+	public Column getColumn() {
+		return column;
+	}
+
+	public PoiExcelColumnValueType getValueType() {
+		return valueType;
+	}
+
+	public String getValueTypeSuffix() {
+		return valueTypeSuffix;
+	}
+
+	public void setColumnIndex(int columnIndex) {
+		this.columnIndex = columnIndex;
+	}
+
+	public int getColumnIndex() {
+		return columnIndex;
+	}
+
+	public Optional<String> getColumnNumber() {
+		for (ColumnOptionTask task : columnTaskList) {
+			Optional<String> option = task.getColumnNumber();
+			if (option.isPresent()) {
+				return option;
+			}
+		}
+		return Optional.absent();
+	}
+
+	protected abstract class CacheValue<T> {
+		private T value;
+
+		public CacheValue() {
+		}
+
+		public T get() {
+			if (value == null) {
+				T v = null;
+				for (ColumnCommonOptionTask task : allTaskList) {
+					Optional<T> option = getTaskValue(task);
+					if (option.isPresent()) {
+						v = option.get();
+						break;
+					}
+				}
+				if (v == null) {
+					v = getDefaultValue();
+				}
+				this.value = v;
+			}
+			return value;
+		}
+
+		protected abstract Optional<T> getTaskValue(ColumnCommonOptionTask task);
+
+		protected abstract T getDefaultValue();
+	}
+
+	public static final class ErrorStrategy {
+		private final Strategy strategy;
+		private final String value;
+
+		public static enum Strategy {
+			DEFAULT, EXCEPTION, CONSTANT, ERROR_CODE
+		}
+
+		public ErrorStrategy(Strategy strategy) {
+			this.strategy = strategy;
+			this.value = null;
+		}
+
+		public ErrorStrategy(String value) {
+			this.strategy = Strategy.CONSTANT;
+			this.value = value;
+		}
+
+		public Strategy getStrategy() {
+			return strategy;
+		}
+
+		public String getValue() {
+			return value;
+		}
+
+		@Override
+		public String toString() {
+			return String.format("ErrorStrategy(%s, %s)", strategy, value);
+		}
+	}
+
+	protected abstract class CacheErrorStrategy extends CacheValue<ErrorStrategy> {
+
+		public CacheErrorStrategy() {
+		}
+
+		@Override
+		protected Optional<ErrorStrategy> getTaskValue(ColumnCommonOptionTask task) {
+			Optional<String> option = getStringValue(task);
+			if (!option.isPresent()) {
+				return Optional.absent();
+			}
+			String value = option.get();
+			if ("null".equalsIgnoreCase(value)) {
+				value = Strategy.CONSTANT.name();
+			}
+
+			String suffix = null;
+			int n = value.indexOf('.');
+			if (n >= 0) {
+				suffix = value.substring(n + 1);
+				value = value.substring(0, n).trim();
+			}
+			try {
+				Strategy strategy = Strategy.valueOf(value.toUpperCase());
+				switch (strategy) {
+				case CONSTANT:
+					return Optional.of(new ErrorStrategy(suffix));
+				default:
+					return Optional.of(new ErrorStrategy(strategy));
+				}
+			} catch (Exception e) {
+				throw new ConfigException(MessageFormat.format("illegal on-error type={0}", value), e);
+			}
+		}
+
+		protected abstract Optional<String> getStringValue(ColumnCommonOptionTask task);
+
+		@Override
+		protected ErrorStrategy getDefaultValue() {
+			return new ErrorStrategy(Strategy.DEFAULT);
+		}
+	}
+
+	private CacheValue<List<String>> attributeName = new CacheValue<List<String>>() {
+
+		@Override
+		protected Optional<List<String>> getTaskValue(ColumnCommonOptionTask task) {
+			if (task instanceof ColumnOptionTask) {
+				return ((ColumnOptionTask) task).getAttributeName();
+			}
+			return Optional.absent();
+		}
+
+		@Override
+		protected List<String> getDefaultValue() {
+			return Collections.emptyList();
+		}
+	};
+
+	public List<String> getAttributeName() {
+		return attributeName.get();
+	}
+
+	private CacheValue<Boolean> searchMergedCell = new CacheValue<Boolean>() {
+
+		@Override
+		protected Optional<Boolean> getTaskValue(ColumnCommonOptionTask task) {
+			return task.getSearchMergedCell();
+		}
+
+		@Override
+		protected Boolean getDefaultValue() {
+			return true;
+		}
+	};
+
+	public boolean getSearchMergedCell() {
+		return searchMergedCell.get();
+	}
+
+	private CacheValue<List<FormulaReplaceTask>> formulaReplace = new CacheValue<List<FormulaReplaceTask>>() {
+
+		@Override
+		protected Optional<List<FormulaReplaceTask>> getTaskValue(ColumnCommonOptionTask task) {
+			return task.getFormulaReplace();
+		}
+
+		@Override
+		protected List<FormulaReplaceTask> getDefaultValue() {
+			return Collections.emptyList();
+		}
+	};
+
+	public List<FormulaReplaceTask> getFormulaReplace() {
+		return formulaReplace.get();
+	}
+
+	private CacheErrorStrategy evaluateErrorStrategy = new CacheErrorStrategy() {
+		@Override
+		protected Optional<String> getStringValue(ColumnCommonOptionTask task) {
+			return task.getOnEvaluateError();
+		}
+	};
+
+	public ErrorStrategy getEvaluateErrorStrategy() {
+		return evaluateErrorStrategy.get();
+	}
+
+	private CacheErrorStrategy cellErrorStrategy = new CacheErrorStrategy() {
+		@Override
+		protected Optional<String> getStringValue(ColumnCommonOptionTask task) {
+			return task.getOnCellError();
+		}
+	};
+
+	public ErrorStrategy getCellErrorStrategy() {
+		return cellErrorStrategy.get();
+	}
+
+	private CacheErrorStrategy convertErrorStrategy = new CacheErrorStrategy() {
+		@Override
+		protected Optional<String> getStringValue(ColumnCommonOptionTask task) {
+			return task.getOnConvertError();
+		}
+	};
+
+	public ErrorStrategy getConvertErrorStrategy() {
+		return convertErrorStrategy.get();
+	}
+}