embulk-parser-poi_excel 0.1.1 → 0.1.2

data/src/main/java/org/embulk/parser/poi_excel/{visitor → bean}/PoiExcelColumnIndex.java

@@ -1,4 +1,4 @@
-package org.embulk.parser.poi_excel.visitor;
+package org.embulk.parser.poi_excel.bean;
 
 import java.text.MessageFormat;
 import java.util.LinkedHashMap;
@@ -7,7 +7,6 @@ import java.util.Map;
 
 import org.apache.poi.ss.util.CellReference;
 import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
@@ -21,48 +20,66 @@ public class PoiExcelColumnIndex {
 
 	protected final Map<String, Integer> indexMap = new LinkedHashMap<>();
 
-	public void initializeColumnIndex(PluginTask task, List<ColumnOptionTask> columnOptions) {
+	public void initializeColumnIndex(PluginTask task, List<PoiExcelColumnBean> beanList) {
 		int index = -1;
 		indexMap.clear();
 
 		Schema schema = task.getColumns().toSchema();
 		for (Column column : schema.getColumns()) {
-			ColumnOptionTask option = columnOptions.get(column.getIndex());
-
-			String type = option.getValueType().trim();
-			int n = type.indexOf('.');
-			if (n >= 0) {
-				String suffix = type.substring(n + 1).trim();
-				option.setValueTypeSuffix(suffix);
-				type = type.substring(0, n).trim();
-			}
-
-			PoiExcelColumnValueType valueType;
-			try {
-				valueType = PoiExcelColumnValueType.valueOf(type.toUpperCase());
-			} catch (Exception e) {
-				throw new RuntimeException(MessageFormat.format("illegal value_type={0}", type), e);
-			}
-			option.setValueTypeEnum(valueType);
+			PoiExcelColumnBean bean = beanList.get(column.getIndex());
+			PoiExcelColumnValueType valueType = bean.getValueType();
 
 			if (valueType.useCell()) {
-				index = resolveColumnIndex(column, option, index, valueType);
+				index = resolveColumnIndex(column, bean, index, valueType);
 				if (index < 0) {
 					index = 0;
 				}
-				log.info("column.name={} <- cell column={}, value_type={}", column.getName(),
-						CellReference.convertNumToColString(index), valueType);
-				option.setColumnIndex(index);
+				bean.setColumnIndex(index);
 				indexMap.put(column.getName(), index);
+				if (log.isInfoEnabled()) {
+					String c = CellReference.convertNumToColString(index);
+					switch (valueType) {
+					default:
+						String suffix = bean.getValueTypeSuffix();
+						if (suffix != null) {
+							log.info("column.name={} <- cell_column={}, value_type={}, value=[{}]", column.getName(),
+									c, valueType, suffix);
+						} else {
+							log.info("column.name={} <- cell_column={}, value_type={}, value={}", column.getName(), c,
+									valueType, suffix);
+						}
+						break;
+					case CELL_VALUE:
+					case CELL_FORMULA:
+					case COLUMN_NUMBER:
+						log.info("column.name={} <- cell_column={}, value_type={}", column.getName(), c, valueType);
+						break;
+					}
+				}
 			} else {
-				log.info("column.name={} <- value_type={}", column.getName(), valueType);
+				if (log.isInfoEnabled()) {
+					switch (valueType) {
+					default:
+						String suffix = bean.getValueTypeSuffix();
+						if (suffix != null) {
+							log.info("column.name={} <- value_type={}, value=[{}]", column.getName(), valueType, suffix);
+						} else {
+							log.info("column.name={} <- value_type={}, value={}", column.getName(), valueType, suffix);
+						}
+						break;
+					case SHEET_NAME:
+					case ROW_NUMBER:
+						log.info("column.name={} <- value_type={}", column.getName(), valueType);
+						break;
+					}
+				}
 			}
 		}
 	}
 
-	protected int resolveColumnIndex(Column column, ColumnOptionTask option, int index,
+	protected int resolveColumnIndex(Column column, PoiExcelColumnBean bean, int index,
 			PoiExcelColumnValueType valueType) {
-		Optional<String> numberOption = option.getColumnNumber();
+		Optional<String> numberOption = bean.getColumnNumber();
 		if (numberOption.isPresent()) {
 			String columnNumber = numberOption.get();
 			if (columnNumber.length() >= 1) {

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelSheetBean.java

@@ -0,0 +1,102 @@
+package org.embulk.parser.poi_excel.bean;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.SheetOptionTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.Schema;
+
+import com.google.common.base.Optional;
+
+public class PoiExcelSheetBean {
+
+	protected final Sheet sheet;
+
+	private final List<SheetOptionTask> sheetTaskList = new ArrayList<>(2);
+
+	private final List<PoiExcelColumnBean> columnBeanList = new ArrayList<>();
+
+	public PoiExcelSheetBean(PluginTask task, Schema schema, Sheet sheet) {
+		this.sheet = sheet;
+
+		initializeSheetTask(task);
+		initializeColumnBean(task, schema);
+	}
+
+	private void initializeSheetTask(PluginTask task) {
+		String name = sheet.getSheetName();
+		Map<String, SheetOptionTask> map = task.getSheetOptions();
+		SheetOptionTask s = map.get(name);
+		if (s != null) {
+			sheetTaskList.add(s);
+		} else {
+			loop: for (Entry<String, SheetOptionTask> entry : map.entrySet()) {
+				String[] ss = entry.getKey().split("/");
+				for (String key : ss) {
+					if (key.trim().equalsIgnoreCase(name)) {
+						sheetTaskList.add(entry.getValue());
+						break loop;
+					}
+				}
+			}
+		}
+		sheetTaskList.add(task);
+	}
+
+	private void initializeColumnBean(PluginTask task, Schema schema) {
+		List<ColumnConfig> list = task.getColumns().getColumns();
+
+		Map<String, ColumnOptionTask> map = new HashMap<>();
+		List<SheetOptionTask> slist = getSheetOption();
+		if (slist.size() >= 2) {
+			SheetOptionTask s = slist.get(0);
+			for (ColumnConfig c : s.getColumns().getColumns()) {
+				String name = c.getName();
+				ColumnOptionTask t = c.getOption().loadConfig(ColumnOptionTask.class);
+				map.put(name, t);
+			}
+		}
+
+		for (Column column : schema.getColumns()) {
+			String name = column.getName();
+			ColumnConfig c = list.get(column.getIndex());
+			ColumnOptionTask t = c.getOption().loadConfig(ColumnOptionTask.class);
+			PoiExcelColumnBean bean = new PoiExcelColumnBean(this, column, t, map.get(name));
+			columnBeanList.add(bean);
+		}
+
+		new PoiExcelColumnIndex().initializeColumnIndex(task, columnBeanList);
+	}
+
+	public final List<SheetOptionTask> getSheetOption() {
+		return sheetTaskList;
+	}
+
+	public int getSkipHeaderLines() {
+		List<SheetOptionTask> list = getSheetOption();
+		for (SheetOptionTask sheetTask : list) {
+			Optional<Integer> value = sheetTask.getSkipHeaderLines();
+			if (value.isPresent()) {
+				return value.get();
+			}
+		}
+		return 0;
+	}
+
+	public final List<PoiExcelColumnBean> getColumnBeans() {
+		return columnBeanList;
+	}
+
+	public final PoiExcelColumnBean getColumnBean(Column column) {
+		List<PoiExcelColumnBean> list = getColumnBeans();
+		return list.get(column.getIndex());
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/AbstractPoiExcelCellAttributeVisitor.java

@@ -10,7 +10,7 @@ import java.util.TreeSet;
 
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Color;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
@@ -18,7 +18,6 @@ import org.embulk.spi.type.StringType;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Optional;
 
 public abstract class AbstractPoiExcelCellAttributeVisitor<A> {
 
@@ -30,24 +29,26 @@ public abstract class AbstractPoiExcelCellAttributeVisitor<A> {
 		this.pageBuilder = visitorValue.getPageBuilder();
 	}
 
-	public void visit(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
-		A source = getAttributeSource(column, option, cell);
+	public void visit(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
+		A source = getAttributeSource(bean, cell);
 		if (source == null) {
+			Column column = bean.getColumn();
 			pageBuilder.setNull(column);
 			return;
 		}
 
-		String suffix = option.getValueTypeSuffix();
+		String suffix = bean.getValueTypeSuffix();
 		if (suffix != null) {
-			visitKey(column, option, suffix, cell, source, visitor);
+			visitKey(bean, suffix, cell, source, visitor);
 		} else {
-			visitJson(column, option, cell, source, visitor);
+			visitJson(bean, cell, source, visitor);
 		}
 	}
 
-	protected abstract A getAttributeSource(Column column, ColumnOptionTask option, Cell cell);
+	protected abstract A getAttributeSource(PoiExcelColumnBean bean, Cell cell);
 
-	private void visitKey(Column column, ColumnOptionTask option, String key, Cell cell, A source, CellVisitor visitor) {
+	private void visitKey(PoiExcelColumnBean bean, String key, Cell cell, A source, CellVisitor visitor) {
+		Column column = bean.getColumn();
 		Object value = getAttributeValue(column, cell, source, key);
 		if (value == null) {
 			pageBuilder.setNull(column);
@@ -67,12 +68,13 @@ public abstract class AbstractPoiExcelCellAttributeVisitor<A> {
 		}
 	}
 
-	private void visitJson(Column column, ColumnOptionTask option, Cell cell, A source, CellVisitor visitor) {
+	private void visitJson(PoiExcelColumnBean bean, Cell cell, A source, CellVisitor visitor) {
+		Column column = bean.getColumn();
+
 		Map<String, Object> result;
 
-		Optional<List<String>> nameOption = option.getAttributeName();
-		if (nameOption.isPresent()) {
-			List<String> list = nameOption.get();
+		List<String> list = bean.getAttributeName();
+		if (!list.isEmpty()) {
 			result = getSpecifiedValues(column, cell, source, list);
 		} else {
 			result = getAllValues(column, cell, source);

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellCommentVisitor.java

@@ -8,7 +8,7 @@ import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.ClientAnchor;
 import org.apache.poi.ss.usermodel.Comment;
 import org.apache.poi.ss.usermodel.RichTextString;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.spi.Column;
 
 public class PoiExcelCellCommentVisitor extends AbstractPoiExcelCellAttributeVisitor<Comment> {
@@ -18,7 +18,7 @@ public class PoiExcelCellCommentVisitor extends AbstractPoiExcelCellAttributeVis
 	}
 
 	@Override
-	protected Comment getAttributeSource(Column column, ColumnOptionTask option, Cell cell) {
+	protected Comment getAttributeSource(PoiExcelColumnBean bean, Cell cell) {
 		return cell.getCellComment();
 	}
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellFontVisitor.java

@@ -9,7 +9,7 @@ import org.apache.poi.ss.usermodel.CellStyle;
 import org.apache.poi.ss.usermodel.Font;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFFont;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.spi.Column;
 
 public class PoiExcelCellFontVisitor extends AbstractPoiExcelCellAttributeVisitor<Font> {
@@ -19,7 +19,7 @@ public class PoiExcelCellFontVisitor extends AbstractPoiExcelCellAttributeVisito
 	}
 
 	@Override
-	protected Font getAttributeSource(Column column, ColumnOptionTask option, Cell cell) {
+	protected Font getAttributeSource(PoiExcelColumnBean bean, Cell cell) {
 		CellStyle style = cell.getCellStyle();
 		short index = style.getFontIndex();
 		Workbook book = visitorValue.getSheet().getWorkbook();

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellStyleVisitor.java

@@ -8,7 +8,7 @@ import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.CellStyle;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFCellStyle;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.spi.Column;
 import org.embulk.spi.type.StringType;
 
@@ -19,7 +19,7 @@ public class PoiExcelCellStyleVisitor extends AbstractPoiExcelCellAttributeVisit
 	}
 
 	@Override
-	protected CellStyle getAttributeSource(Column column, ColumnOptionTask option, Cell cell) {
+	protected CellStyle getAttributeSource(PoiExcelColumnBean bean, Cell cell) {
 		return cell.getCellStyle();
 	}
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellValueVisitor.java

@@ -6,23 +6,22 @@ import java.util.List;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.CellValue;
 import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.FormulaError;
 import org.apache.poi.ss.usermodel.FormulaEvaluator;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.ss.util.CellRangeAddress;
 import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.FormulaReplaceTask;
-import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.ErrorStrategy;
 import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
 import org.slf4j.Logger;
 
-import com.google.common.base.Optional;
-
 public class PoiExcelCellValueVisitor {
 	private final Logger log = Exec.getLogger(getClass());
 
@@ -34,9 +33,11 @@ public class PoiExcelCellValueVisitor {
 		this.pageBuilder = visitorValue.getPageBuilder();
 	}
 
-	public void visitCellValue(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+	public void visitCellValue(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
 		assert cell != null;
 
+		Column column = bean.getColumn();
+
 		int cellType = cell.getCellType();
 		switch (cellType) {
 		case Cell.CELL_TYPE_NUMERIC:
@@ -46,32 +47,33 @@ public class PoiExcelCellValueVisitor {
 			visitor.visitCellValueString(column, cell, cell.getStringCellValue());
 			return;
 		case Cell.CELL_TYPE_FORMULA:
-			PoiExcelColumnValueType valueType = option.getValueTypeEnum();
+			PoiExcelColumnValueType valueType = bean.getValueType();
 			if (valueType == PoiExcelColumnValueType.CELL_FORMULA) {
 				visitor.visitCellFormula(column, cell);
 			} else {
-				visitCellValueFormula(column, option, cell, visitor);
+				visitCellValueFormula(bean, cell, visitor);
 			}
 			return;
 		case Cell.CELL_TYPE_BLANK:
-			visitCellValueBlank(column, option, cell, visitor);
+			visitCellValueBlank(bean, cell, visitor);
 			return;
 		case Cell.CELL_TYPE_BOOLEAN:
 			visitor.visitCellValueBoolean(column, cell, cell.getBooleanCellValue());
 			return;
 		case Cell.CELL_TYPE_ERROR:
-			visitCellValueError(column, option, cell, cell.getErrorCellValue(), visitor);
+			visitCellValueError(bean, cell, cell.getErrorCellValue(), visitor);
 			return;
 		default:
 			throw new IllegalStateException(MessageFormat.format("unsupported POI cellType={0}", cellType));
 		}
 	}
 
-	protected void visitCellValueBlank(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+	protected void visitCellValueBlank(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
 		assert cell.getCellType() == Cell.CELL_TYPE_BLANK;
 
-		PluginTask task = visitorValue.getPluginTask();
-		boolean search = option.getSearchMergedCell().or(task.getSearchMergedCell());
+		Column column = bean.getColumn();
+
+		boolean search = bean.getSearchMergedCell();
 		if (!search) {
 			visitor.visitCellValueBlank(column, cell);
 			return;
@@ -96,7 +98,7 @@ public class PoiExcelCellValueVisitor {
 					return;
 				}
 
-				visitCellValue(column, option, firstCell, visitor);
+				visitCellValue(bean, firstCell, visitor);
 				return;
 			}
 		}
@@ -104,19 +106,16 @@ public class PoiExcelCellValueVisitor {
 		visitor.visitCellValueBlank(column, cell);
 	}
 
-	protected void visitCellValueFormula(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+	protected void visitCellValueFormula(PoiExcelColumnBean bean, Cell cell, CellVisitor visitor) {
 		assert cell.getCellType() == Cell.CELL_TYPE_FORMULA;
 
-		Optional<List<FormulaReplaceTask>> replaceOption = option.getFormulaReplace();
-		if (!replaceOption.isPresent()) {
-			PluginTask task = visitorValue.getPluginTask();
-			replaceOption = task.getFormulaReplace();
-		}
-		if (replaceOption.isPresent()) {
+		Column column = bean.getColumn();
+
+		List<FormulaReplaceTask> list = bean.getFormulaReplace();
+		if (!list.isEmpty()) {
 			String formula = cell.getCellFormula();
 			String old = formula;
 
-			List<FormulaReplaceTask> list = replaceOption.get();
 			for (FormulaReplaceTask replace : list) {
 				String regex = replace.getRegex();
 				String replacement = replace.getTo();
@@ -143,10 +142,17 @@ public class PoiExcelCellValueVisitor {
 			FormulaEvaluator evaluator = helper.createFormulaEvaluator();
 			cellValue = evaluator.evaluate(cell);
 		} catch (Exception e) {
-			PluginTask task = visitorValue.getPluginTask();
-			boolean setNull = option.getFormulaErrorNull().or(task.getFormulaErrorNull());
-			if (setNull) {
-				pageBuilder.setNull(column);
+			ErrorStrategy strategy = bean.getEvaluateErrorStrategy();
+			switch (strategy.getStrategy()) {
+			default:
+				break;
+			case CONSTANT:
+				String value = strategy.getValue();
+				if (value == null) {
+					pageBuilder.setNull(column);
+				} else {
+					visitor.visitCellValueString(column, cell, value);
+				}
 				return;
 			}
 
@@ -168,7 +174,7 @@ public class PoiExcelCellValueVisitor {
 			visitor.visitCellValueBoolean(column, cellValue, cellValue.getBooleanValue());
 			return;
 		case Cell.CELL_TYPE_ERROR:
-			visitCellValueError(column, option, cellValue, cellValue.getErrorValue(), visitor);
+			visitCellValueError(bean, cellValue, cellValue.getErrorValue(), visitor);
 			return;
 		case Cell.CELL_TYPE_FORMULA:
 		default:
@@ -176,13 +182,28 @@ public class PoiExcelCellValueVisitor {
 		}
 	}
 
-	protected void visitCellValueError(Column column, ColumnOptionTask option, Object cell, int errorCode,
-			CellVisitor visitor) {
-		PluginTask task = visitorValue.getPluginTask();
-		boolean setNull = option.getCellErrorNull().or(task.getCellErrorNull());
-		if (setNull) {
+	protected void visitCellValueError(PoiExcelColumnBean bean, Object cell, int errorCode, CellVisitor visitor) {
+		Column column = bean.getColumn();
+
+		ErrorStrategy strategy = bean.getCellErrorStrategy();
+		switch (strategy.getStrategy()) {
+		default:
 			pageBuilder.setNull(column);
 			return;
+		case CONSTANT:
+			String value = strategy.getValue();
+			if (value == null) {
+				pageBuilder.setNull(column);
+			} else {
+				visitor.visitCellValueString(column, cell, value);
+			}
+			return;
+		case ERROR_CODE:
+			break;
+		case EXCEPTION:
+			FormulaError error = FormulaError.forInt((byte) errorCode);
+			throw new RuntimeException(MessageFormat.format("encount cell error. error_code={0}({1})", errorCode,
+					error.getString()));
 		}
 
 		visitor.visitCellValueError(column, cell, errorCode);