embulk-parser-poi_excel 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d1be437da5c3e86a63ef4c925d3ba11218c5344
-  data.tar.gz: eb1b92328a4ac0c9619165bf046289538223e2ed
+  metadata.gz: 4fc1f7535c2d5dd180ad60a729c0daccab15144c
+  data.tar.gz: 2c09c8ffa12e314ff1af31dd0c2bab4c02d00057
 SHA512:
-  metadata.gz: de797f9fd1895bb79987ae4564d0a4288a7a4c4115b217194462d9f5912912b0c814214c6319a363923953feeeb62f70cc2e4261b2a27695b1a26e6b60324c5c
-  data.tar.gz: 8db340bcd1e3a46f54bd62513620c3d067432db0b5c28f71d9235f2dd3ae1aa5d0c37753ae1f1488f868e1915129c814d61d306dd58e9d58a602dcadd2ef9795
+  metadata.gz: 0236a3b74bee6392af244631640fd8fe317fa42728a690eecfe4ef11b149fa5c5f9237a334af0d757008aabfff7040c9787fef363b9a84a67c799eff5148e328
+  data.tar.gz: df386175cd7cbc2be51d70682f21d62581f527c040dfeaeac86def0ddff0b62297d58fcd3920cbd5ae4be636e49ae7ecd3f08de15e28c2930d7b4313140af6bb

data/README.md

@@ -13,6 +13,7 @@ This plugin uses Apache POI.
 * **sheets**: sheet name. (list of string, required)
 * **skip_header_lines**: skip rows. (integer, default: `0`)
 * **columns**: column definition. see below. (hash, required)
+* **sheet_options**: sheet option. see below. (hash, default: null)
 
 ### columns
 
@@ -115,6 +116,38 @@ Processing method of convert error. ex) Excel boolean to Embulk timestamp
 * `exception`: throw exception. (default)
 
 
+### sheet_options
+
+Options of indivisual sheet.
+
+```yaml
+  parser:
+    type: poi_excel
+    sheets: [Sheet1, Sheet2]
+    columns:
+    - {name: date, type: timestamp, column_number: A}
+    - {name: foo, type: string}
+    - {name: bar, type: long}
+    sheet_options:
+      Sheet1:
+        skip_header_lines: 1
+        columns:
+          foo: {column_number: B}
+          bar: {column_number: C}
+      Sheet2:
+        skip_header_lines: 0
+        columns:
+          foo: {column_number: D}
+          bar: {value: constant.0}
+```
+
+*sheet_options* is map of sheet name.  
+Map values are *skip_header_lines*, *colums*.
+
+*columns* is map of column name.  
+Map values are same *columns* in *parser*.
+
+
 ## Example
 
 ```yaml
@@ -148,5 +181,6 @@ $ embulk gem install embulk-parser-poi_excel
 ## Build
 
 ```
+$ ./gradlew test
 $ ./gradlew package
 ```

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -39,7 +39,7 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 
 	public static final String TYPE = "poi_excel";
 
-	public interface PluginTask extends Task, TimestampParser.Task, SheetOptionTask {
+	public interface PluginTask extends Task, TimestampParser.Task, SheetCommonOptionTask {
 		@Config("sheet")
 		@ConfigDefault("null")
 		public Optional<String> getSheet();
@@ -56,19 +56,26 @@ public class PoiExcelParserPlugin implements ParserPlugin {
 		@ConfigDefault("{}")
 		public Map<String, SheetOptionTask> getSheetOptions();
 
+		@Config("columns")
+		public SchemaConfig getColumns();
+
 		@Config("flush_count")
 		@ConfigDefault("100")
 		public int getFlushCount();
 	}
 
-	public interface SheetOptionTask extends Task, ColumnCommonOptionTask {
+	public interface SheetCommonOptionTask extends Task, ColumnCommonOptionTask {
 
 		@Config("skip_header_lines")
 		@ConfigDefault("null")
 		public Optional<Integer> getSkipHeaderLines();
+	}
+
+	public interface SheetOptionTask extends Task, SheetCommonOptionTask {
 
 		@Config("columns")
-		public SchemaConfig getColumns();
+		@ConfigDefault("null")
+		public Optional<Map<String, ColumnOptionTask>> getColumns();
 	}
 
 	public interface ColumnOptionTask extends Task, ColumnCommonOptionTask {

data/src/main/java/org/embulk/parser/poi_excel/bean/PoiExcelSheetBean.java

@@ -9,6 +9,7 @@ import java.util.Map.Entry;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.SheetCommonOptionTask;
 import org.embulk.parser.poi_excel.PoiExcelParserPlugin.SheetOptionTask;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnConfig;
@@ -20,7 +21,7 @@ public class PoiExcelSheetBean {
 
 	protected final Sheet sheet;
 
-	private final List<SheetOptionTask> sheetTaskList = new ArrayList<>(2);
+	private final List<SheetCommonOptionTask> sheetTaskList = new ArrayList<>(2);
 
 	private final List<PoiExcelColumnBean> columnBeanList = new ArrayList<>();
 
@@ -55,13 +56,14 @@ public class PoiExcelSheetBean {
 		List<ColumnConfig> list = task.getColumns().getColumns();
 
 		Map<String, ColumnOptionTask> map = new HashMap<>();
-		List<SheetOptionTask> slist = getSheetOption();
-		if (slist.size() >= 2) {
-			SheetOptionTask s = slist.get(0);
-			for (ColumnConfig c : s.getColumns().getColumns()) {
-				String name = c.getName();
-				ColumnOptionTask t = c.getOption().loadConfig(ColumnOptionTask.class);
-				map.put(name, t);
+		List<SheetCommonOptionTask> slist = getSheetOption();
+		for (int i = slist.size() - 1; i >= 0; i--) {
+			SheetCommonOptionTask s = slist.get(i);
+			if (s instanceof SheetOptionTask) {
+				Optional<Map<String, ColumnOptionTask>> option = ((SheetOptionTask) s).getColumns();
+				if (option.isPresent()) {
+					map.putAll(option.get());
+				}
 			}
 		}
 
@@ -76,13 +78,13 @@ public class PoiExcelSheetBean {
 		new PoiExcelColumnIndex().initializeColumnIndex(task, columnBeanList);
 	}
 
-	public final List<SheetOptionTask> getSheetOption() {
+	public final List<SheetCommonOptionTask> getSheetOption() {
 		return sheetTaskList;
 	}
 
 	public int getSkipHeaderLines() {
-		List<SheetOptionTask> list = getSheetOption();
-		for (SheetOptionTask sheetTask : list) {
+		List<SheetCommonOptionTask> list = getSheetOption();
+		for (SheetCommonOptionTask sheetTask : list) {
 			Optional<Integer> value = sheetTask.getSkipHeaderLines();
 			if (value.isPresent()) {
 				return value.get();

data/src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_sheets.java

@@ -0,0 +1,85 @@
+package org.embulk.parser.poi_excel;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.net.URL;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.embulk.parser.EmbulkPluginTester;
+import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
+import org.embulk.parser.EmbulkTestParserConfig;
+import org.junit.Test;
+
+public class TestPoiExcelParserPlugin_sheets {
+
+	@Test
+	public void testSheets() throws ParseException {
+		try (EmbulkPluginTester tester = new EmbulkPluginTester()) {
+			tester.addParserPlugin(PoiExcelParserPlugin.TYPE, PoiExcelParserPlugin.class);
+
+			EmbulkTestParserConfig parser = tester.newParserConfig(PoiExcelParserPlugin.TYPE);
+			parser.set("sheets", Arrays.asList("test1", "formula_replace", "style"));
+			parser.addColumn("text", "string");
+			parser.addColumn("number", "long");
+
+			Map<String, Object> sheetOptions = new HashMap<>();
+			{
+				Map<String, Object> sheet = new HashMap<>();
+				sheet.put("skip_header_lines", "5");
+				Map<String, Object> columns = new HashMap<>();
+				columns.put("text", newMap("column_number", "D"));
+				columns.put("number", newMap("column_number", "B"));
+				sheet.put("columns", columns);
+				sheetOptions.put("test1", sheet);
+			}
+			{
+				Map<String, Object> sheet = new HashMap<>();
+				Map<String, Object> columns = new HashMap<>();
+				columns.put("number", newMap("value", "constant.0"));
+				sheet.put("columns", columns);
+				sheetOptions.put("formula_replace", sheet);
+			}
+			{
+				Map<String, Object> sheet = new HashMap<>();
+				sheet.put("skip_header_lines", "2");
+				Map<String, Object> columns = new HashMap<>();
+				columns.put("text", newMap("column_number", "B"));
+				columns.put("number", newMap("value", "constant.-1"));
+				sheet.put("columns", columns);
+				sheetOptions.put("style", sheet);
+			}
+			parser.set("sheet_options", sheetOptions);
+
+			URL inFile = getClass().getResource("test1.xls");
+			List<OutputRecord> result = tester.runParser(inFile, parser);
+
+			assertThat(result.size(), is(8));
+			check1(result, 0, "abc", 123L);
+			check1(result, 1, "true", 1L);
+			check1(result, 2, null, null);
+			check1(result, 3, "boolean", 0L);
+			check1(result, 4, "test2-b1", 0L);
+			check1(result, 5, "left", -1L);
+			check1(result, 6, "right", -1L);
+			check1(result, 7, "bottom", -1L);
+		}
+	}
+
+	private Map<String, Object> newMap(String key, Object value) {
+		Map<String, Object> map = new HashMap<>();
+		map.put(key, value);
+		return map;
+	}
+
+	private void check1(List<OutputRecord> result, int index, String text, Long number) {
+		OutputRecord record = result.get(index);
+		// System.out.println(record);
+		assertThat(record.getAsString("text"), is(text));
+		assertThat(record.getAsLong("number"), is(number));
+	}
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-poi_excel
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - hishidama
@@ -88,9 +88,10 @@ files:
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_columnNumber.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_constant.java
 - src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_convertError.java
+- src/test/java/org/embulk/parser/poi_excel/TestPoiExcelParserPlugin_sheets.java
 - src/test/resources/org/embulk/parser/poi_excel/test1.xls
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-parser-poi_excel-0.1.2.jar
+- classpath/embulk-parser-poi_excel-0.1.3.jar
 - classpath/embulk-standards-0.7.5.jar
 - classpath/poi-3.13.jar
 - classpath/poi-ooxml-3.13.jar