embulk-parser-poi_excel 0.1.4 → 0.1.10

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelClientAnchorVisitor.java

@@ -40,7 +40,7 @@ public class PoiExcelClientAnchorVisitor extends AbstractPoiExcelCellAttributeVi
 		map.put("anchor_type", new AttributeSupplier<ClientAnchor>() {
 			@Override
 			public Object get(Column column, Cell cell, ClientAnchor anchor) {
-				return (long) anchor.getAnchorType();
+				return (long) anchor.getAnchorType().value;
 			}
 		});
 		map.put("col1", new AttributeSupplier<ClientAnchor>() {

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelColumnVisitor.java

@@ -3,16 +3,22 @@ package org.embulk.parser.poi_excel.visitor;
 import java.text.MessageFormat;
 
 import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellType;
 import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.util.CellReference;
 import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
+import org.embulk.parser.poi_excel.bean.util.PoiExcelCellAddress;
 import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
+import org.slf4j.Logger;
 
 public class PoiExcelColumnVisitor implements ColumnVisitor {
+	private final Logger log = Exec.getLogger(getClass());
 
 	protected final PoiExcelVisitorValue visitorValue;
 	protected final PageBuilder pageBuilder;
@@ -56,6 +62,9 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 	}
 
 	protected final void visitCell0(Column column, CellVisitor visitor) {
+		if (log.isTraceEnabled()) {
+			log.trace("{} start", column);
+		}
 		try {
 			visitCell(column, visitor);
 		} catch (Exception e) {
@@ -65,21 +74,52 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 			throw new RuntimeException(MessageFormat.format("error at {0} cell={1}!{2}. {3}", column, sheetName, ref,
 					e.getMessage()), e);
 		}
+		if (log.isTraceEnabled()) {
+			log.trace("{} end", column);
+		}
 	}
 
 	protected void visitCell(Column column, CellVisitor visitor) {
 		PoiExcelColumnBean bean = visitorValue.getColumnBean(column);
 		PoiExcelColumnValueType valueType = bean.getValueType();
+		PoiExcelCellAddress cellAddress = bean.getCellAddress();
 
 		switch (valueType) {
 		case SHEET_NAME:
-			visitor.visitSheetName(column);
+			if (cellAddress != null) {
+				Sheet sheet = cellAddress.getSheet(currentRow);
+				visitor.visitSheetName(column, sheet);
+			} else {
+				visitor.visitSheetName(column);
+			}
 			return;
 		case ROW_NUMBER:
-			visitor.visitRowNumber(column, currentRow.getRowNum() + 1);
+			int rowIndex;
+			if (cellAddress != null) {
+				Cell cell = cellAddress.getCell(currentRow);
+				if (cell == null) {
+					visitCellNull(column);
+					return;
+				}
+				rowIndex = cell.getRowIndex();
+			} else {
+				rowIndex = currentRow.getRowNum();
+			}
+			visitor.visitRowNumber(column, rowIndex + 1);
 			return;
 		case COLUMN_NUMBER:
-			visitor.visitColumnNumber(column, bean.getColumnIndex() + 1);
+			int columnIndex;
+			if (cellAddress != null) {
+				Cell cell = cellAddress.getCell(currentRow);
+				if (cell == null) {
+					visitCellNull(column);
+					return;
+				}
+				columnIndex = cell.getColumnIndex();
+			} else {
+				columnIndex = bean.getColumnIndex();
+			}
+			visitor.visitColumnNumber(column, columnIndex + 1);
 			return;
 		case CONSTANT:
 			visitCellConstant(column, bean.getValueTypeSuffix(), visitor);
@@ -89,7 +129,12 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 		}
 
 		assert valueType.useCell();
-		Cell cell = currentRow.getCell(bean.getColumnIndex());
+		Cell cell;
+		if (cellAddress != null) {
+			cell = cellAddress.getCell(currentRow);
+		} else {
+			cell = currentRow.getCell(bean.getColumnIndex());
+		}
 		if (cell == null) {
 			visitCellNull(column);
 			return;
@@ -108,6 +153,16 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 		case CELL_COMMENT:
 			visitCellComment(bean, cell, visitor);
 			return;
+		case CELL_TYPE:
+			visitCellType(bean, cell, cell.getCellTypeEnum(), visitor);
+			return;
+		case CELL_CACHED_TYPE:
+			if (cell.getCellTypeEnum() == CellType.FORMULA) {
+				visitCellType(bean, cell, cell.getCachedFormulaResultTypeEnum(), visitor);
+			} else {
+				visitCellType(bean, cell, cell.getCellTypeEnum(), visitor);
+			}
+			return;
 		default:
 			throw new UnsupportedOperationException(MessageFormat.format("unsupported value_type={0}", valueType));
 		}
@@ -144,4 +199,9 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 		PoiExcelCellCommentVisitor delegator = factory.getPoiExcelCellCommentVisitor();
 		delegator.visit(bean, cell, visitor);
 	}
+
+	private void visitCellType(PoiExcelColumnBean bean, Cell cell, CellType cellType, CellVisitor visitor) {
+		PoiExcelCellTypeVisitor delegator = factory.getPoiExcelCellTypeVisitor();
+		delegator.visit(bean, cell, cellType, visitor);
+	}
 }

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelVisitorFactory.java

@@ -159,6 +159,20 @@ public class PoiExcelVisitorFactory {
 		return new PoiExcelCellCommentVisitor(visitorValue);
 	}
 
+	// cell type
+	private PoiExcelCellTypeVisitor poiExcelCellTypeVisitor;
+
+	public final PoiExcelCellTypeVisitor getPoiExcelCellTypeVisitor() {
+		if (poiExcelCellTypeVisitor == null) {
+			poiExcelCellTypeVisitor = newPoiExcelCellTypeVisitor();
+		}
+		return poiExcelCellTypeVisitor;
+	}
+
+	protected PoiExcelCellTypeVisitor newPoiExcelCellTypeVisitor() {
+		return new PoiExcelCellTypeVisitor(visitorValue);
+	}
+
 	// ClientAnchor
 	private PoiExcelClientAnchorVisitor poiExcelClientAnchorVisitor;
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/BooleanCellVisitor.java

@@ -38,6 +38,11 @@ public class BooleanCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		int index = sheet.getWorkbook().getSheetIndex(sheet);
 		pageBuilder.setBoolean(column, index != 0);
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/CellVisitor.java

@@ -3,6 +3,7 @@ package org.embulk.parser.poi_excel.visitor.embulk;
 import java.text.MessageFormat;
 
 import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Sheet;
 import org.embulk.config.ConfigException;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.ErrorStrategy;
@@ -40,6 +41,8 @@ public abstract class CellVisitor {
 
 	public abstract void visitSheetName(Column column);
 
+	public abstract void visitSheetName(Column column, Sheet sheet);
+
 	public abstract void visitRowNumber(Column column, int index1);
 
 	public abstract void visitColumnNumber(Column column, int index1);

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/DoubleCellVisitor.java

@@ -45,6 +45,11 @@ public class DoubleCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		int index = sheet.getWorkbook().getSheetIndex(sheet);
 		pageBuilder.setDouble(column, index);
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/LongCellVisitor.java

@@ -45,6 +45,11 @@ public class LongCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		int index = sheet.getWorkbook().getSheetIndex(sheet);
 		pageBuilder.setLong(column, index);
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/StringCellVisitor.java

@@ -1,8 +1,11 @@
 package org.embulk.parser.poi_excel.visitor.embulk;
 
+import java.text.MessageFormat;
+
 import org.apache.poi.ss.usermodel.FormulaError;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.util.CellReference;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.visitor.PoiExcelVisitorValue;
 import org.embulk.spi.Column;
 
@@ -14,11 +17,31 @@ public class StringCellVisitor extends CellVisitor {
 
 	@Override
 	public void visitCellValueNumeric(Column column, Object source, double value) {
+		String s = toString(column, source, value);
+		pageBuilder.setString(column, s);
+	}
+
+	protected String toString(Column column, Object source, double value) {
+		String format = getNumericFormat(column);
+		if (!format.isEmpty()) {
+			try {
+				return String.format(format, value);
+			} catch (Exception e) {
+				throw new IllegalArgumentException(MessageFormat.format(
+						"illegal String.format for double. numeric_format=\"{0}\"", format), e);
+			}
+		}
+
 		String s = Double.toString(value);
 		if (s.endsWith(".0")) {
-			s = s.substring(0, s.length() - 2);
+			return s.substring(0, s.length() - 2);
 		}
-		pageBuilder.setString(column, s);
+		return s;
+	}
+
+	protected String getNumericFormat(Column column) {
+		PoiExcelColumnBean bean = visitorValue.getColumnBean(column);
+		return bean.getNumericFormat();
 	}
 
 	@Override
@@ -47,6 +70,11 @@ public class StringCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		pageBuilder.setString(column, sheet.getSheetName());
 	}
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/TimestampCellVisitor.java

@@ -61,6 +61,11 @@ public class TimestampCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		doConvertError(column, sheet.getSheetName(), new UnsupportedOperationException(
 				"unsupported conversion sheet_name to Embulk timestamp"));
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/util/MergedRegionMap.java

@@ -0,0 +1,51 @@
+package org.embulk.parser.poi_excel.visitor.util;
+
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.util.CellRangeAddress;
+
+public class MergedRegionMap {
+
+	private final Map<Sheet, Map<Integer, Map<Integer, CellRangeAddress>>> sheetMap = new ConcurrentHashMap<>();
+
+	public CellRangeAddress get(Sheet sheet, int rowIndex, int columnIndex) {
+		Map<Integer, Map<Integer, CellRangeAddress>> rowMap = sheetMap.get(sheet);
+		if (rowMap == null) {
+			synchronized (sheet) {
+				rowMap = createRowMap(sheet);
+				sheetMap.put(sheet, rowMap);
+			}
+		}
+
+		Map<Integer, CellRangeAddress> columnMap = rowMap.get(rowIndex);
+		if (columnMap == null) {
+			return null;
+		}
+		return columnMap.get(columnIndex);
+	}
+
+	protected Map<Integer, Map<Integer, CellRangeAddress>> createRowMap(Sheet sheet) {
+		Map<Integer, Map<Integer, CellRangeAddress>> rowMap = new TreeMap<>();
+
+		for (int i = sheet.getNumMergedRegions() - 1; i >= 0; i--) {
+			CellRangeAddress region = sheet.getMergedRegion(i);
+
+			for (int r = region.getFirstRow(); r <= region.getLastRow(); r++) {
+				Map<Integer, CellRangeAddress> columnMap = rowMap.get(r);
+				if (columnMap == null) {
+					columnMap = new TreeMap<>();
+					rowMap.put(r, columnMap);
+				}
+
+				for (int c = region.getFirstColumn(); c <= region.getLastColumn(); c++) {
+					columnMap.put(c, region);
+				}
+			}
+		}
+
+		return rowMap;
+	}
+}

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin.java

@@ -1,7 +1,7 @@
 package org.embulk.parser.poi_excel;
 
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 import java.net.URL;
 import java.text.ParseException;
@@ -10,7 +10,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.TimeZone;
 
-import org.embulk.config.ConfigSource;
 import org.embulk.parser.EmbulkPluginTester;
 import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
 import org.embulk.parser.EmbulkTestParserConfig;
@@ -74,6 +73,32 @@ public class TestPoiExcelParserPlugin {
 		assertThat(r.getAsTimestamp("timestamp"), is(timestamp));
 	}
 
+	@Theory
+	public void testNumricFormat(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test1"));
+			parser.set("skip_header_lines", 1);
+			parser.addColumn("value", "string").set("column_number", "C").set("numeric_format", "%.2f");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(7));
+			checkNumricFormat(result, 0, "123.40");
+			checkNumricFormat(result, 1, "456.70");
+			checkNumricFormat(result, 2, "123.00");
+		}
+	}
+
+	private void checkNumricFormat(List<OutputRecord> result, int index, String s) {
+		OutputRecord r = result.get(index);
+		// System.out.println(r);
+		assertThat(r.getAsString("value"), is(s));
+	}
+
 	@Theory
 	public void testRowNumber(String excelFile) throws ParseException {
 		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
@@ -114,83 +139,6 @@ public class TestPoiExcelParserPlugin {
 		assertThat(r.getAsString("col-s"), is("A"));
 	}
 
-	@Theory
-	public void testForumlaReplace(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "formula_replace");
-
-			ConfigSource replace0 = tester.newConfigSource();
-			replace0.set("regex", "test1");
-			replace0.set("to", "merged_cell");
-			ConfigSource replace1 = tester.newConfigSource();
-			replace1.set("regex", "B1");
-			replace1.set("to", "B${row}");
-			parser.set("formula_replace", Arrays.asList(replace0, replace1));
-
-			parser.addColumn("text", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(2));
-			assertThat(result.get(0).getAsString("text"), is("test3-a1"));
-			assertThat(result.get(1).getAsString("text"), is("test2-b2"));
-		}
-	}
-
-	@Theory
-	public void testSearchMergedCell_true(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "merged_cell");
-			parser.addColumn("a", "string");
-			parser.addColumn("b", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(4));
-			check6(result, 0, "test3-a1", "test3-a1");
-			check6(result, 1, "data", "0");
-			check6(result, 2, null, null);
-			check6(result, 3, null, null);
-		}
-	}
-
-	@Theory
-	public void testSearchMergedCell_false(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "merged_cell");
-			parser.set("search_merged_cell", false);
-			parser.addColumn("a", "string");
-			parser.addColumn("b", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(4));
-			check6(result, 0, "test3-a1", null);
-			check6(result, 1, "data", "0");
-			check6(result, 2, null, null);
-			check6(result, 3, null, null);
-		}
-	}
-
-	private void check6(List<OutputRecord> result, int index, String a, String b) {
-		OutputRecord r = result.get(index);
-		// System.out.println(r);
-		assertThat(r.getAsString("a"), is(a));
-		assertThat(r.getAsString("b"), is(b));
-	}
-
 	@Theory
 	public void test_sheets(String excelFile) throws ParseException {
 		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {