embulk-parser-poi_excel 0.1.5 → 0.1.12

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelVisitorFactory.java

@@ -159,6 +159,20 @@ public class PoiExcelVisitorFactory {
 		return new PoiExcelCellCommentVisitor(visitorValue);
 	}
 
+	// cell type
+	private PoiExcelCellTypeVisitor poiExcelCellTypeVisitor;
+
+	public final PoiExcelCellTypeVisitor getPoiExcelCellTypeVisitor() {
+		if (poiExcelCellTypeVisitor == null) {
+			poiExcelCellTypeVisitor = newPoiExcelCellTypeVisitor();
+		}
+		return poiExcelCellTypeVisitor;
+	}
+
+	protected PoiExcelCellTypeVisitor newPoiExcelCellTypeVisitor() {
+		return new PoiExcelCellTypeVisitor(visitorValue);
+	}
+
 	// ClientAnchor
 	private PoiExcelClientAnchorVisitor poiExcelClientAnchorVisitor;
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/BooleanCellVisitor.java

@@ -38,6 +38,11 @@ public class BooleanCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		int index = sheet.getWorkbook().getSheetIndex(sheet);
 		pageBuilder.setBoolean(column, index != 0);
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/CellVisitor.java

@@ -3,6 +3,7 @@ package org.embulk.parser.poi_excel.visitor.embulk;
 import java.text.MessageFormat;
 
 import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Sheet;
 import org.embulk.config.ConfigException;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean.ErrorStrategy;
@@ -40,6 +41,8 @@ public abstract class CellVisitor {
 
 	public abstract void visitSheetName(Column column);
 
+	public abstract void visitSheetName(Column column, Sheet sheet);
+
 	public abstract void visitRowNumber(Column column, int index1);
 
 	public abstract void visitColumnNumber(Column column, int index1);

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/DoubleCellVisitor.java

@@ -45,6 +45,11 @@ public class DoubleCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		int index = sheet.getWorkbook().getSheetIndex(sheet);
 		pageBuilder.setDouble(column, index);
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/LongCellVisitor.java

@@ -45,6 +45,11 @@ public class LongCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		int index = sheet.getWorkbook().getSheetIndex(sheet);
 		pageBuilder.setLong(column, index);
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/StringCellVisitor.java

@@ -1,8 +1,11 @@
 package org.embulk.parser.poi_excel.visitor.embulk;
 
+import java.text.MessageFormat;
+
 import org.apache.poi.ss.usermodel.FormulaError;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.util.CellReference;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.visitor.PoiExcelVisitorValue;
 import org.embulk.spi.Column;
 
@@ -14,11 +17,31 @@ public class StringCellVisitor extends CellVisitor {
 
 	@Override
 	public void visitCellValueNumeric(Column column, Object source, double value) {
+		String s = toString(column, source, value);
+		pageBuilder.setString(column, s);
+	}
+
+	protected String toString(Column column, Object source, double value) {
+		String format = getNumericFormat(column);
+		if (!format.isEmpty()) {
+			try {
+				return String.format(format, value);
+			} catch (Exception e) {
+				throw new IllegalArgumentException(MessageFormat.format(
+						"illegal String.format for double. numeric_format=\"{0}\"", format), e);
+			}
+		}
+
 		String s = Double.toString(value);
 		if (s.endsWith(".0")) {
-			s = s.substring(0, s.length() - 2);
+			return s.substring(0, s.length() - 2);
 		}
-		pageBuilder.setString(column, s);
+		return s;
+	}
+
+	protected String getNumericFormat(Column column) {
+		PoiExcelColumnBean bean = visitorValue.getColumnBean(column);
+		return bean.getNumericFormat();
 	}
 
 	@Override
@@ -47,6 +70,11 @@ public class StringCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		pageBuilder.setString(column, sheet.getSheetName());
 	}
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/TimestampCellVisitor.java

@@ -61,6 +61,11 @@ public class TimestampCellVisitor extends CellVisitor {
 	@Override
 	public void visitSheetName(Column column) {
 		Sheet sheet = visitorValue.getSheet();
+		visitSheetName(column, sheet);
+	}
+
+	@Override
+	public void visitSheetName(Column column, Sheet sheet) {
 		doConvertError(column, sheet.getSheetName(), new UnsupportedOperationException(
 				"unsupported conversion sheet_name to Embulk timestamp"));
 	}

data/src/main/java/org/embulk/parser/poi_excel/visitor/util/MergedRegionFinder.java

@@ -0,0 +1,9 @@
+package org.embulk.parser.poi_excel.visitor.util;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.util.CellRangeAddress;
+
+public interface MergedRegionFinder {
+
+	public CellRangeAddress get(Sheet sheet, int rowIndex, int columnIndex);
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/util/MergedRegionList.java

@@ -0,0 +1,20 @@
+package org.embulk.parser.poi_excel.visitor.util;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.util.CellRangeAddress;
+
+public class MergedRegionList implements MergedRegionFinder {
+
+	@Override
+	public CellRangeAddress get(Sheet sheet, int rowIndex, int columnIndex) {
+		int size = sheet.getNumMergedRegions();
+		for (int i = 0; i < size; i++) {
+			CellRangeAddress region = sheet.getMergedRegion(i);
+			if (region.isInRange(rowIndex, columnIndex)) {
+				return region;
+			}
+		}
+
+		return null;
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/util/MergedRegionMap.java

@@ -0,0 +1,55 @@
+package org.embulk.parser.poi_excel.visitor.util;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.util.CellRangeAddress;
+
+public abstract class MergedRegionMap implements MergedRegionFinder {
+
+	private final Map<Sheet, Map<Integer, Map<Integer, CellRangeAddress>>> sheetMap = new ConcurrentHashMap<>();
+
+	@Override
+	public CellRangeAddress get(Sheet sheet, int rowIndex, int columnIndex) {
+		Map<Integer, Map<Integer, CellRangeAddress>> rowMap = sheetMap.get(sheet);
+		if (rowMap == null) {
+			synchronized (sheet) {
+				rowMap = createRowMap(sheet);
+				sheetMap.put(sheet, rowMap);
+			}
+		}
+
+		Map<Integer, CellRangeAddress> columnMap = rowMap.get(rowIndex);
+		if (columnMap == null) {
+			return null;
+		}
+		return columnMap.get(columnIndex);
+	}
+
+	protected Map<Integer, Map<Integer, CellRangeAddress>> createRowMap(Sheet sheet) {
+		Map<Integer, Map<Integer, CellRangeAddress>> rowMap = newRowMap();
+
+		for (int i = sheet.getNumMergedRegions() - 1; i >= 0; i--) {
+			CellRangeAddress region = sheet.getMergedRegion(i);
+
+			for (int r = region.getFirstRow(); r <= region.getLastRow(); r++) {
+				Map<Integer, CellRangeAddress> columnMap = rowMap.get(r);
+				if (columnMap == null) {
+					columnMap = newColumnMap();
+					rowMap.put(r, columnMap);
+				}
+
+				for (int c = region.getFirstColumn(); c <= region.getLastColumn(); c++) {
+					columnMap.put(c, region);
+				}
+			}
+		}
+
+		return rowMap;
+	}
+
+	protected abstract Map<Integer, Map<Integer, CellRangeAddress>> newRowMap();
+
+	protected abstract Map<Integer, CellRangeAddress> newColumnMap();
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/util/MergedRegionNothing.java

@@ -0,0 +1,12 @@
+package org.embulk.parser.poi_excel.visitor.util;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.util.CellRangeAddress;
+
+public class MergedRegionNothing implements MergedRegionFinder {
+
+	@Override
+	public CellRangeAddress get(Sheet sheet, int rowIndex, int columnIndex) {
+		return null;
+	}
+}

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin.java

@@ -1,7 +1,7 @@
 package org.embulk.parser.poi_excel;
 
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 import java.net.URL;
 import java.text.ParseException;
@@ -10,7 +10,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.TimeZone;
 
-import org.embulk.config.ConfigSource;
 import org.embulk.parser.EmbulkPluginTester;
 import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
 import org.embulk.parser.EmbulkTestParserConfig;
@@ -74,6 +73,32 @@ public class TestPoiExcelParserPlugin {
 		assertThat(r.getAsTimestamp("timestamp"), is(timestamp));
 	}
 
+	@Theory
+	public void testNumricFormat(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test1"));
+			parser.set("skip_header_lines", 1);
+			parser.addColumn("value", "string").set("column_number", "C").set("numeric_format", "%.2f");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(7));
+			checkNumricFormat(result, 0, "123.40");
+			checkNumricFormat(result, 1, "456.70");
+			checkNumricFormat(result, 2, "123.00");
+		}
+	}
+
+	private void checkNumricFormat(List<OutputRecord> result, int index, String s) {
+		OutputRecord r = result.get(index);
+		// System.out.println(r);
+		assertThat(r.getAsString("value"), is(s));
+	}
+
 	@Theory
 	public void testRowNumber(String excelFile) throws ParseException {
 		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
@@ -114,83 +139,6 @@ public class TestPoiExcelParserPlugin {
 		assertThat(r.getAsString("col-s"), is("A"));
 	}
 
-	@Theory
-	public void testForumlaReplace(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "formula_replace");
-
-			ConfigSource replace0 = tester.newConfigSource();
-			replace0.set("regex", "test1");
-			replace0.set("to", "merged_cell");
-			ConfigSource replace1 = tester.newConfigSource();
-			replace1.set("regex", "B1");
-			replace1.set("to", "B${row}");
-			parser.set("formula_replace", Arrays.asList(replace0, replace1));
-
-			parser.addColumn("text", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(2));
-			assertThat(result.get(0).getAsString("text"), is("test3-a1"));
-			assertThat(result.get(1).getAsString("text"), is("test2-b2"));
-		}
-	}
-
-	@Theory
-	public void testSearchMergedCell_true(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "merged_cell");
-			parser.addColumn("a", "string");
-			parser.addColumn("b", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(4));
-			check6(result, 0, "test3-a1", "test3-a1");
-			check6(result, 1, "data", "0");
-			check6(result, 2, null, null);
-			check6(result, 3, null, null);
-		}
-	}
-
-	@Theory
-	public void testSearchMergedCell_false(String excelFile) throws ParseException {
-		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
-			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
-
-			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
-			parser.set("sheet", "merged_cell");
-			parser.set("search_merged_cell", false);
-			parser.addColumn("a", "string");
-			parser.addColumn("b", "string");
-
-			URL inFile = getClass().getResource(excelFile);
-			List<OutputRecord> result = tester.runParser(inFile, parser);
-
-			assertThat(result.size(), is(4));
-			check6(result, 0, "test3-a1", null);
-			check6(result, 1, "data", "0");
-			check6(result, 2, null, null);
-			check6(result, 3, null, null);
-		}
-	}
-
-	private void check6(List<OutputRecord> result, int index, String a, String b) {
-		OutputRecord r = result.get(index);
-		// System.out.println(r);
-		assertThat(r.getAsString("a"), is(a));
-		assertThat(r.getAsString("b"), is(b));
-	}
-
 	@Theory
 	public void test_sheets(String excelFile) throws ParseException {
 		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_cellAddress.java

@@ -0,0 +1,69 @@
+package org.embulk.parser.poi_excel;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import java.net.URL;
+import java.text.ParseException;
+import java.util.List;
+
+import org.embulk.parser.EmbulkPluginTester;
+import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
+import org.embulk.parser.EmbulkTestParserConfig;
+import org.junit.experimental.theories.DataPoints;
+import org.junit.experimental.theories.Theories;
+import org.junit.experimental.theories.Theory;
+import org.junit.runner.RunWith;
+
+@RunWith(Theories.class)
+public class TestPoiExcelParserPlugin_cellAddress {
+
+	@DataPoints
+	public static String[] FILES = { "test1.xls", "test2.xlsx" };
+
+	@Theory
+	public void testCellAddress(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheet", "test1");
+			parser.set("skip_header_lines", 1);
+			parser.addColumn("text", "string").set("column_number", "D");
+			parser.addColumn("fix_value", "string").set("cell_address", "B1").set("value", "cell_value");
+			parser.addColumn("fix_sheet", "string").set("cell_address", "B1").set("value", "sheet_name");
+			parser.addColumn("fix_row", "long").set("cell_address", "B1").set("value", "row_number");
+			parser.addColumn("fix_col", "long").set("cell_address", "B1").set("value", "column_number");
+			parser.addColumn("other_sheet_value", "string").set("cell_address", "style!B5").set("value", "cell_value");
+			parser.addColumn("other_sheet_name", "string").set("cell_address", "style!B5").set("value", "sheet_name");
+			parser.addColumn("other_sheet_row", "long").set("cell_address", "style!B5").set("value", "row_number");
+			parser.addColumn("other_sheet_col", "string").set("cell_address", "style!B5").set("value", "column_number");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(7));
+			check1(result, 0, "abc");
+			check1(result, 1, "def");
+			check1(result, 2, "456");
+			check1(result, 3, "abc");
+			check1(result, 4, "abc");
+			check1(result, 5, "true");
+			check1(result, 6, null);
+		}
+	}
+
+	private void check1(List<OutputRecord> result, int index, String text) {
+		OutputRecord record = result.get(index);
+		// System.out.println(record);
+		assertThat(record.getAsString("text"), is(text));
+		assertThat(record.getAsString("fix_value"), is("long"));
+		assertThat(record.getAsString("fix_sheet"), is("test1"));
+		assertThat(record.getAsLong("fix_row"), is(1L));
+		assertThat(record.getAsLong("fix_col"), is(2L));
+		assertThat(record.getAsString("other_sheet_value"), is("bottom"));
+		assertThat(record.getAsString("other_sheet_name"), is("style"));
+		assertThat(record.getAsLong("other_sheet_row"), is(5L));
+		assertThat(record.getAsString("other_sheet_col"), is("B"));
+	}
+}