embulk-parser-poi_excel 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a6920bea2e679cbcc53f02a2c70485ba3d4b350
-  data.tar.gz: a83e9e1a2cb8ef78da34d7ef5602b66b3f2e48c8
+  metadata.gz: 3ab71cb044d2238a08719c80b1092f50cce9b84e
+  data.tar.gz: c02f1e64148806369e8aca0e11a86daba48f134f
 SHA512:
-  metadata.gz: 33920df4d360e86b5609f3939c63762b2f1d9b87a16f1923cf59ab5f16060ce1a2070c860e66d46fd50631a6a08f7677c7f32cb3dfc33e2084f1aca6fab75fe7
-  data.tar.gz: 043b272a531e072b07033c43a014021c8cf9e1e65a7e12f7079341cff35adf6dc429e30c3da3415f994698fa02aa401fd527771469094db93583408c081c2c31
+  metadata.gz: 0966c93aba6475e4c77a1a987be10a0413025bb96f7925b9f41413802ad9217c1366b06b1378369e7ee9dcad8b70ca843532cac70d606c08aa0553a9195f108e
+  data.tar.gz: 06a1f63b2ba99e9f88410e72d1a29b5b3eb3b221394674a1568b9d4ed32a3292a95e93ed6639c2f26b6be513ee0f15d0199e96ee7b35ff4ba9082947f48908c5

data/README.md

@@ -44,6 +44,7 @@ if omit **column_number** when **value** is `cell_value`, specified next column.
 * **type**: Embulk column type. (string, required)
 * **value**: value type. see below. (string, defualt: `cell_value`)
 * **column_number**: Excel column number. see below. (string, default: next column)
+* **numeric_format**: format of numeric(double) to string such as `%4.2f`. (default: Java's Double.toString())
 * **attribute_name**: use with value `cell_style`, `cell_font`, etc. see below. (list of string)
 * **on_cell_error**: processing method of Cell error. see below. (string, default: `constant`)
 * **formula_handling** : processing method of formula. see below. (`evaluate` or `cashed_value`. default: `evaluate`)
@@ -58,6 +59,8 @@ if omit **column_number** when **value** is `cell_value`, specified next column.
 * `cell_style`: all cell style attributes. returned json string. see **attribute_name**. (**type** required `string`)
 * `cell_font`: all cell font attributes. returned json string. see **attribute_name**. (**type** required `string`)
 * `cell_comment`: all cell comment attributes. returned json string. see **attribute_name**. (**type** required `string`)
+* `cell_type`: cell type. returned Cell.getCellType() of POI.
+* `cell_cached_type`: cell cached formula result type. returned Cell.getCachedFormulaResultType() of POI when CellType==FORMULA, otherwise same as cell_type(returned Cell.getCellType()).
 * `sheet_name`: sheet name.
 * `row_number`: row number(1 origin).
 * `column_number`: column number(1 origin).

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.6"
+version = "0.1.7"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelColumnValueType.java

@@ -11,6 +11,10 @@ public enum PoiExcelColumnValueType {
 	CELL_FONT(true, false),
 	/** cell comment */
 	CELL_COMMENT(true, false),
+	/** cell type */
+	CELL_TYPE(true, false),
+	/** cell CachedFormulaResultType */
+	CELL_CACHED_TYPE(true, false),
 	/** sheet name */
 	SHEET_NAME(false, false),
 	/** row number (1 origin) */

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -103,6 +103,10 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 	}
 
 	public interface ColumnCommonOptionTask extends Task {
+		// format of numeric(double) to string
+		@Config("numeric_format")
+		@ConfigDefault("null")
+		public Optional<String> getNumericFormat();
 
 		// search merged cell if cellType=BLANK
 		@Config("search_merged_cell")

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelColumnBean.java

@@ -231,6 +231,23 @@ public class PoiExcelColumnBean {
 		return attributeName.get();
 	}
 
+	private CacheValue<String> numericFormat = new CacheValue<String>() {
+
+		@Override
+		protected Optional<String> getTaskValue(ColumnCommonOptionTask task) {
+			return task.getNumericFormat();
+		}
+
+		@Override
+		protected String getDefaultValue() {
+			return "";
+		}
+	};
+
+	public String getNumericFormat() {
+		return numericFormat.get();
+	}
+
 	private CacheValue<Boolean> searchMergedCell = new CacheValue<Boolean>() {
 
 		@Override

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelColumnIndex.java

@@ -51,6 +51,8 @@ public class PoiExcelColumnIndex {
 						break;
 					case CELL_VALUE:
 					case CELL_FORMULA:
+					case CELL_TYPE:
+					case CELL_CACHED_TYPE:
 					case COLUMN_NUMBER:
 						log.info("column.name={} <- cell_column={}, value_type={}", column.getName(), c, valueType);
 						break;

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellTypeVisitor.java

@@ -0,0 +1,38 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
+import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.StringType;
+
+public class PoiExcelCellTypeVisitor {
+	protected final PoiExcelVisitorValue visitorValue;
+	protected final PageBuilder pageBuilder;
+
+	public PoiExcelCellTypeVisitor(PoiExcelVisitorValue visitorValue) {
+		this.visitorValue = visitorValue;
+		this.pageBuilder = visitorValue.getPageBuilder();
+	}
+
+	private static final String[] CELL_TYPE_STRING = { "NUMERIC", "STRING", "FORMULA", "BLANK", "BOOLEAN", "ERROR" };
+
+	public void visit(PoiExcelColumnBean bean, Cell cell, int cellType, CellVisitor visitor) {
+		assert cell != null;
+
+		Column column = bean.getColumn();
+		if (column.getType() instanceof StringType) {
+			String type;
+			if (0 <= cellType && cellType < CELL_TYPE_STRING.length) {
+				type = CELL_TYPE_STRING[cellType];
+			} else {
+				type = Integer.toString(cellType);
+			}
+			visitor.visitCellValueString(column, cell, type);
+			return;
+		}
+
+		visitor.visitCellValueNumeric(column, cell, cellType);
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelColumnVisitor.java

@@ -117,6 +117,16 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 		case CELL_COMMENT:
 			visitCellComment(bean, cell, visitor);
 			return;
+		case CELL_TYPE:
+			visitCellType(bean, cell, cell.getCellType(), visitor);
+			return;
+		case CELL_CACHED_TYPE:
+			if (cell.getCellType() == Cell.CELL_TYPE_FORMULA) {
+				visitCellType(bean, cell, cell.getCachedFormulaResultType(), visitor);
+			} else {
+				visitCellType(bean, cell, cell.getCellType(), visitor);
+			}
+			return;
 		default:
 			throw new UnsupportedOperationException(MessageFormat.format("unsupported value_type={0}", valueType));
 		}
@@ -153,4 +163,9 @@ public class PoiExcelColumnVisitor implements ColumnVisitor {
 		PoiExcelCellCommentVisitor delegator = factory.getPoiExcelCellCommentVisitor();
 		delegator.visit(bean, cell, visitor);
 	}
+
+	private void visitCellType(PoiExcelColumnBean bean, Cell cell, int cellType, CellVisitor visitor) {
+		PoiExcelCellTypeVisitor delegator = factory.getPoiExcelCellTypeVisitor();
+		delegator.visit(bean, cell, cellType, visitor);
+	}
 }

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelVisitorFactory.java

@@ -159,6 +159,20 @@ public class PoiExcelVisitorFactory {
 		return new PoiExcelCellCommentVisitor(visitorValue);
 	}
 
+	// cell type
+	private PoiExcelCellTypeVisitor poiExcelCellTypeVisitor;
+
+	public final PoiExcelCellTypeVisitor getPoiExcelCellTypeVisitor() {
+		if (poiExcelCellTypeVisitor == null) {
+			poiExcelCellTypeVisitor = newPoiExcelCellTypeVisitor();
+		}
+		return poiExcelCellTypeVisitor;
+	}
+
+	protected PoiExcelCellTypeVisitor newPoiExcelCellTypeVisitor() {
+		return new PoiExcelCellTypeVisitor(visitorValue);
+	}
+
 	// ClientAnchor
 	private PoiExcelClientAnchorVisitor poiExcelClientAnchorVisitor;
 

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/StringCellVisitor.java

@@ -1,8 +1,11 @@
 package org.embulk.parser.poi_excel.visitor.embulk;
 
+import java.text.MessageFormat;
+
 import org.apache.poi.ss.usermodel.FormulaError;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.util.CellReference;
+import org.embulk.parser.poi_excel.bean.PoiExcelColumnBean;
 import org.embulk.parser.poi_excel.visitor.PoiExcelVisitorValue;
 import org.embulk.spi.Column;
 
@@ -14,11 +17,31 @@ public class StringCellVisitor extends CellVisitor {
 
 	@Override
 	public void visitCellValueNumeric(Column column, Object source, double value) {
+		String s = toString(column, source, value);
+		pageBuilder.setString(column, s);
+	}
+
+	protected String toString(Column column, Object source, double value) {
+		String format = getNumericFormat(column);
+		if (!format.isEmpty()) {
+			try {
+				return String.format(format, value);
+			} catch (Exception e) {
+				throw new IllegalArgumentException(MessageFormat.format(
+						"illegal String.format for double. numeric_format=\"{0}\"", format), e);
+			}
+		}
+
 		String s = Double.toString(value);
 		if (s.endsWith(".0")) {
-			s = s.substring(0, s.length() - 2);
+			return s.substring(0, s.length() - 2);
 		}
-		pageBuilder.setString(column, s);
+		return s;
+	}
+
+	protected String getNumericFormat(Column column) {
+		PoiExcelColumnBean bean = visitorValue.getColumnBean(column);
+		return bean.getNumericFormat();
 	}
 
 	@Override

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin.java

@@ -73,6 +73,32 @@ public class TestPoiExcelParserPlugin {
 		assertThat(r.getAsTimestamp("timestamp"), is(timestamp));
 	}
 
+	@Theory
+	public void testNumricFormat(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test1"));
+			parser.set("skip_header_lines", 1);
+			parser.addColumn("value", "string").set("column_number", "C").set("numeric_format", "%.2f");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(7));
+			checkNumricFormat(result, 0, "123.40");
+			checkNumricFormat(result, 1, "456.70");
+			checkNumricFormat(result, 2, "123.00");
+		}
+	}
+
+	private void checkNumricFormat(List<OutputRecord> result, int index, String s) {
+		OutputRecord r = result.get(index);
+		// System.out.println(r);
+		assertThat(r.getAsString("value"), is(s));
+	}
+
 	@Theory
 	public void testRowNumber(String excelFile) throws ParseException {
 		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_cellType.java

@@ -0,0 +1,78 @@
+package org.embulk.parser.poi_excel;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.net.URL;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.embulk.parser.EmbulkPluginTester;
+import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
+import org.embulk.parser.EmbulkTestParserConfig;
+import org.junit.experimental.theories.DataPoints;
+import org.junit.experimental.theories.Theories;
+import org.junit.experimental.theories.Theory;
+import org.junit.runner.RunWith;
+
+@RunWith(Theories.class)
+public class TestPoiExcelParserPlugin_cellType {
+
+	@DataPoints
+	public static String[] FILES = { "test1.xls", "test2.xlsx" };
+
+	@Theory
+	public void testCellType(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test1"));
+			parser.set("skip_header_lines", 3);
+			parser.addColumn("long", "long").set("column_number", "A").set("value", "cell_type");
+			parser.addColumn("string", "string").set("column_number", "A").set("value", "cell_type");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(5));
+			check1(result, 0, Cell.CELL_TYPE_NUMERIC, "NUMERIC");
+			check1(result, 1, Cell.CELL_TYPE_STRING, "STRING");
+			check1(result, 2, Cell.CELL_TYPE_FORMULA, "FORMULA");
+			check1(result, 3, Cell.CELL_TYPE_BOOLEAN, "BOOLEAN");
+			check1(result, 4, Cell.CELL_TYPE_FORMULA, "FORMULA");
+		}
+	}
+
+	@Theory
+	public void testCellCachedType(String excelFile) throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test1"));
+			parser.set("skip_header_lines", 3);
+			parser.addColumn("long", "long").set("column_number", "A").set("value", "cell_cached_type");
+			parser.addColumn("string", "string").set("column_number", "A").set("value", "cell_cached_type");
+
+			URL inFile = getClass().getResource(excelFile);
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(5));
+			check1(result, 0, Cell.CELL_TYPE_NUMERIC, "NUMERIC");
+			check1(result, 1, Cell.CELL_TYPE_STRING, "STRING");
+			check1(result, 2, Cell.CELL_TYPE_BOOLEAN, "BOOLEAN");
+			check1(result, 3, Cell.CELL_TYPE_BOOLEAN, "BOOLEAN");
+			check1(result, 4, Cell.CELL_TYPE_ERROR, "ERROR");
+		}
+	}
+
+	private void check1(List<OutputRecord> result, int index, long l, String s) throws ParseException {
+		OutputRecord r = result.get(index);
+		// System.out.println(r);
+		assertThat(r.getAsLong("long"), is(l));
+		assertThat(r.getAsString("string"), is(s));
+	}
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-poi_excel
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - hishidama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-28 00:00:00.000000000 Z
+date: 2017-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,6 +64,7 @@ files:
 - src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellCommentVisitor.java
 - src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellFontVisitor.java
 - src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellStyleVisitor.java
+- src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellTypeVisitor.java
 - src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellValueVisitor.java
 - src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelClientAnchorVisitor.java
 - src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelColorVisitor.java
@@ -85,6 +86,7 @@ files:
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_cellError.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_cellFont.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_cellStyle.java
+- src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_cellType.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_columnNumber.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_constant.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_convertError.java
@@ -93,7 +95,7 @@ files:
 - src/test/resources/org/embulk/parser/poi_excel/test1.xls
 - src/test/resources/org/embulk/parser/poi_excel/test2.xlsx
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-parser-poi_excel-0.1.6.jar
+- classpath/embulk-parser-poi_excel-0.1.7.jar
 - classpath/embulk-standards-0.7.5.jar
 - classpath/poi-3.13.jar
 - classpath/poi-ooxml-3.13.jar