embulk-parser-poi_excel 0.1.0

data/src/main/java/org/embulk/parser/poi_excel/visitor/embulk/TimestampCellVisitor.java

@@ -0,0 +1,73 @@
+package org.embulk.parser.poi_excel.visitor.embulk;
+
+import java.util.Date;
+import java.util.TimeZone;
+
+import org.apache.poi.ss.usermodel.DateUtil;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
+import org.embulk.parser.poi_excel.visitor.PoiExcelVisitorValue;
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.Timestamps;
+
+public class TimestampCellVisitor extends CellVisitor {
+
+	public TimestampCellVisitor(PoiExcelVisitorValue visitorValue) {
+		super(visitorValue);
+	}
+
+	@Override
+	public void visitCellValueNumeric(Column column, Object source, double value) {
+		TimestampParser parser = getTimestampParser(column);
+		TimeZone tz = parser.getDefaultTimeZone().toTimeZone();
+		Date date = DateUtil.getJavaDate(value, tz);
+		pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(date.getTime()));
+	}
+
+	@Override
+	public void visitCellValueString(Column column, Object source, String value) {
+		TimestampParser parser = getTimestampParser(column);
+		pageBuilder.setTimestamp(column, parser.parse(value));
+	}
+
+	@Override
+	public void visitCellValueBoolean(Column column, Object source, boolean value) {
+		throw new UnsupportedOperationException("unsupported conversion Excel boolean to Embulk timestamp.");
+	}
+
+	@Override
+	public void visitCellValueError(Column column, Object source, int code) {
+		pageBuilder.setNull(column);
+	}
+
+	@Override
+	public void visitValueLong(Column column, Object source, long value) {
+		pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(value));
+	}
+
+	@Override
+	public void visitSheetName(Column column) {
+		throw new UnsupportedOperationException("unsupported conversion sheet_name to Embulk timestamp.");
+	}
+
+	@Override
+	public void visitRowNumber(Column column, int index1) {
+		throw new UnsupportedOperationException("unsupported conversion row_number to Embulk timestamp.");
+	}
+
+	@Override
+	public void visitColumnNumber(Column column, int index1) {
+		throw new UnsupportedOperationException("unsupported conversion column_number to Embulk timestamp.");
+	}
+
+	private TimestampParser[] timestampParsers;
+
+	protected final TimestampParser getTimestampParser(Column column) {
+		if (timestampParsers == null) {
+			PluginTask task = visitorValue.getPluginTask();
+			timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getColumns());
+		}
+		return timestampParsers[column.getIndex()];
+	}
+}

data/src/test/java/org/embulk/parser/EmbulkPluginTester.java

@@ -0,0 +1,176 @@
+package org.embulk.parser;
+
+import java.io.Closeable;
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.EmbulkEmbed;
+import org.embulk.EmbulkEmbed.Bootstrap;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.parser.EmbulkTestOutputPlugin.OutputRecord;
+import org.embulk.plugin.InjectedPluginSource;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ParserPlugin;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+
+// @see https://github.com/embulk/embulk-input-jdbc/blob/master/embulk-input-mysql/src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java
+public class EmbulkPluginTester implements Closeable {
+
+	protected static class PluginDefinition {
+		public final Class<?> iface;
+		public final String name;
+		public final Class<?> impl;
+
+		public PluginDefinition(Class<?> iface, String name, Class<?> impl) {
+			this.iface = iface;
+			this.name = name;
+			this.impl = impl;
+		}
+	}
+
+	private final List<PluginDefinition> plugins = new ArrayList<>();
+
+	private EmbulkEmbed embulk;
+
+	private ConfigLoader configLoader;
+
+	private EmbulkTestFileInputPlugin embulkTestFileInputPlugin = new EmbulkTestFileInputPlugin();
+
+	private EmbulkTestOutputPlugin embulkTestOutputPlugin = new EmbulkTestOutputPlugin();
+
+	public EmbulkPluginTester() {
+	}
+
+	public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl) {
+		addPlugin(iface, name, impl);
+	}
+
+	public void addPlugin(Class<?> iface, String name, Class<?> impl) {
+		plugins.add(new PluginDefinition(iface, name, impl));
+	}
+
+	public void addParserPlugin(String name, Class<? extends ParserPlugin> impl) {
+		addPlugin(ParserPlugin.class, name, impl);
+	}
+
+	protected EmbulkEmbed getEmbulkEmbed() {
+		if (embulk == null) {
+			Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+			bootstrap.addModules(new Module() {
+				@Override
+				public void configure(Binder binder) {
+					EmbulkPluginTester.this.configurePlugin(binder);
+
+					for (PluginDefinition plugin : plugins) {
+						InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
+					}
+				}
+			});
+			embulk = bootstrap.initializeCloseable();
+		}
+		return embulk;
+	}
+
+	protected void configurePlugin(Binder binder) {
+		// input plugins
+		InjectedPluginSource.registerPluginTo(binder, InputPlugin.class, EmbulkTestFileInputPlugin.TYPE,
+				EmbulkTestFileInputPlugin.class);
+		binder.bind(EmbulkTestFileInputPlugin.class).toProvider(new Provider<EmbulkTestFileInputPlugin>() {
+
+			@Override
+			public EmbulkTestFileInputPlugin get() {
+				return embulkTestFileInputPlugin;
+			}
+		});
+
+		// output plugins
+		InjectedPluginSource.registerPluginTo(binder, OutputPlugin.class, EmbulkTestOutputPlugin.TYPE,
+				EmbulkTestOutputPlugin.class);
+		binder.bind(EmbulkTestOutputPlugin.class).toProvider(new Provider<EmbulkTestOutputPlugin>() {
+
+			@Override
+			public EmbulkTestOutputPlugin get() {
+				return embulkTestOutputPlugin;
+			}
+		});
+	}
+
+	public ConfigLoader getConfigLoader() {
+		if (configLoader == null) {
+			configLoader = getEmbulkEmbed().newConfigLoader();
+		}
+		return configLoader;
+	}
+
+	public ConfigSource newConfigSource() {
+		return getConfigLoader().newConfigSource();
+	}
+
+	public EmbulkTestParserConfig newParserConfig(String type) {
+		EmbulkTestParserConfig parser = new EmbulkTestParserConfig();
+		parser.setType(type);
+		return parser;
+	}
+
+	public List<OutputRecord> runParser(URL inFile, EmbulkTestParserConfig parser) {
+		File file;
+		try {
+			file = new File(inFile.toURI());
+		} catch (URISyntaxException e) {
+			throw new RuntimeException(e);
+		}
+		return runParser(file, parser);
+	}
+
+	public List<OutputRecord> runParser(File inFile, EmbulkTestParserConfig parser) {
+		ConfigSource in = newConfigSource();
+		in.set("type", "file");
+		in.set("path_prefix", inFile.getAbsolutePath());
+		in.set("parser", parser);
+		return runInput(in);
+	}
+
+	public List<OutputRecord> runParser(List<String> list, EmbulkTestParserConfig parser) {
+		ConfigSource in = newConfigSource();
+		in.set("type", EmbulkTestFileInputPlugin.TYPE);
+		in.set("parser", parser);
+
+		embulkTestFileInputPlugin.setText(list);
+		return runInput(in);
+	}
+
+	public List<OutputRecord> runInput(ConfigSource in) {
+		ConfigSource out = newConfigSource();
+		out.set("type", EmbulkTestOutputPlugin.TYPE);
+
+		embulkTestOutputPlugin.clearResult();
+		run(in, out);
+		return embulkTestOutputPlugin.getResult();
+	}
+
+	public void run(ConfigSource in, ConfigSource out) {
+		ConfigSource config = newConfigSource();
+		config.set("in", in);
+		config.set("out", out);
+		run(config);
+	}
+
+	public void run(ConfigSource config) {
+		getEmbulkEmbed().run(config);
+	}
+
+	@Override
+	public void close() {
+		if (embulk != null) {
+			embulk.destroy();
+		}
+	}
+}

data/src/test/java/org/embulk/parser/EmbulkTestFileInputPlugin.java

@@ -0,0 +1,83 @@
+package org.embulk.parser;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+
+public class EmbulkTestFileInputPlugin implements FileInputPlugin {
+
+	public static final String TYPE = "EmbulkTestFileInputPlugin";
+
+	public interface PluginTask extends Task {
+	}
+
+	private List<String> list;
+
+	public void setText(List<String> list) {
+		this.list = list;
+	}
+
+	@Override
+	public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
+		PluginTask task = config.loadConfig(PluginTask.class);
+
+		int taskCount = 1;
+		return resume(task.dump(), taskCount, control);
+	}
+
+	@Override
+	public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) {
+		control.run(taskSource, taskCount);
+		return Exec.newConfigDiff();
+	}
+
+	@Override
+	public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
+	}
+
+	@Override
+	public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
+		return new TransactionalFileInput() {
+			private boolean eof = false;
+			private int index = 0;
+
+			@Override
+			public Buffer poll() {
+				if (index < list.size()) {
+					String s = list.get(index++) + "\n";
+					return Buffer.copyOf(s.getBytes(StandardCharsets.UTF_8));
+				}
+
+				eof = true;
+				return null;
+			}
+
+			@Override
+			public boolean nextFile() {
+				return !eof;
+			}
+
+			@Override
+			public void close() {
+			}
+
+			@Override
+			public void abort() {
+			}
+
+			@Override
+			public TaskReport commit() {
+				return Exec.newTaskReport();
+			}
+		};
+	}
+}

data/src/test/java/org/embulk/parser/EmbulkTestOutputPlugin.java

@@ -0,0 +1,193 @@
+package org.embulk.parser;
+
+import java.text.MessageFormat;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+
+public class EmbulkTestOutputPlugin implements OutputPlugin {
+
+	public static final String TYPE = "EmbulkTestOutputPlugin";
+
+	public interface PluginTask extends Task, TimestampFormatter.Task {
+	}
+
+	public static class OutputRecord {
+		private Map<String, Object> map = new LinkedHashMap<>();
+
+		public void set(String name, Object value) {
+			map.put(name, value);
+		}
+
+		public String getAsString(String name) {
+			try {
+				return (String) map.get(name);
+			} catch (Exception e) {
+				throw new RuntimeException(MessageFormat.format("name={0}", name), e);
+			}
+		}
+
+		public Long getAsLong(String name) {
+			try {
+				return (Long) map.get(name);
+			} catch (Exception e) {
+				throw new RuntimeException(MessageFormat.format("name={0}", name), e);
+			}
+		}
+
+		public Double getAsDouble(String name) {
+			try {
+				return (Double) map.get(name);
+			} catch (Exception e) {
+				throw new RuntimeException(MessageFormat.format("name={0}", name), e);
+			}
+		}
+
+		public Boolean getAsBoolean(String name) {
+			try {
+				return (Boolean) map.get(name);
+			} catch (Exception e) {
+				throw new RuntimeException(MessageFormat.format("name={0}", name), e);
+			}
+		}
+
+		public Timestamp getAsTimestamp(String name) {
+			try {
+				return (Timestamp) map.get(name);
+			} catch (Exception e) {
+				throw new RuntimeException(MessageFormat.format("name={0}", name), e);
+			}
+		}
+
+		@Override
+		public String toString() {
+			return map.toString();
+		}
+	}
+
+	private final List<OutputRecord> result = new CopyOnWriteArrayList<>();
+
+	@Override
+	public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
+		final PluginTask task = config.loadConfig(PluginTask.class);
+		return resume(task.dump(), schema, taskCount, control);
+	}
+
+	@Override
+	public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, OutputPlugin.Control control) {
+		control.run(taskSource);
+		return Exec.newConfigDiff();
+	}
+
+	@Override
+	public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports) {
+	}
+
+	@Override
+	public TransactionalPageOutput open(TaskSource taskSource, final Schema schema, int taskIndex) {
+		return new TransactionalPageOutput() {
+			private final PageReader reader = new PageReader(schema);
+
+			@Override
+			public void add(Page page) {
+				reader.setPage(page);
+				while (reader.nextRecord()) {
+					final OutputRecord record = new OutputRecord();
+					for (Column column : schema.getColumns()) {
+						column.visit(new ColumnVisitor() {
+
+							@Override
+							public void timestampColumn(Column column) {
+								if (reader.isNull(column)) {
+									record.set(column.getName(), null);
+									return;
+								}
+								record.set(column.getName(), reader.getTimestamp(column));
+							}
+
+							@Override
+							public void stringColumn(Column column) {
+								if (reader.isNull(column)) {
+									record.set(column.getName(), null);
+									return;
+								}
+								record.set(column.getName(), reader.getString(column));
+							}
+
+							@Override
+							public void longColumn(Column column) {
+								if (reader.isNull(column)) {
+									record.set(column.getName(), null);
+									return;
+								}
+								record.set(column.getName(), reader.getLong(column));
+							}
+
+							@Override
+							public void doubleColumn(Column column) {
+								if (reader.isNull(column)) {
+									record.set(column.getName(), null);
+									return;
+								}
+								record.set(column.getName(), reader.getDouble(column));
+							}
+
+							@Override
+							public void booleanColumn(Column column) {
+								if (reader.isNull(column)) {
+									record.set(column.getName(), null);
+									return;
+								}
+								record.set(column.getName(), reader.getBoolean(column));
+							}
+						});
+					}
+					result.add(record);
+				}
+			}
+
+			@Override
+			public void finish() {
+			}
+
+			@Override
+			public void close() {
+				reader.close();
+			}
+
+			@Override
+			public void abort() {
+			}
+
+			@Override
+			public TaskReport commit() {
+				return Exec.newTaskReport();
+			}
+		};
+	}
+
+	public void clearResult() {
+		result.clear();
+	}
+
+	public List<OutputRecord> getResult() {
+		return result;
+	}
+}

data/src/test/java/org/embulk/parser/EmbulkTestParserConfig.java

@@ -0,0 +1,51 @@
+package org.embulk.parser;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+@SuppressWarnings("serial")
+public class EmbulkTestParserConfig extends HashMap<String, Object> {
+
+	public void setType(String type) {
+		set("type", type);
+	}
+
+	public void set(String key, Object value) {
+		if (value == null) {
+			super.remove(key);
+		} else {
+			super.put(key, value);
+		}
+	}
+
+	public List<EmbulkTestColumn> getColumns() {
+		@SuppressWarnings("unchecked")
+		List<EmbulkTestColumn> columns = (List<EmbulkTestColumn>) super.get("columns");
+		if (columns == null) {
+			columns = new ArrayList<>();
+			super.put("columns", columns);
+		}
+		return columns;
+	}
+
+	public EmbulkTestColumn addColumn(String name, String type) {
+		EmbulkTestColumn column = new EmbulkTestColumn();
+		column.set("name", name);
+		column.set("type", type);
+		getColumns().add(column);
+		return column;
+	}
+
+	public static class EmbulkTestColumn extends HashMap<String, Object> {
+
+		public EmbulkTestColumn set(String key, Object value) {
+			if (value == null) {
+				super.remove(key);
+			} else {
+				super.put(key, value);
+			}
+			return this;
+		}
+	}
+}