embulk-parser-poi_excel 0.1.0

data/src/main/java/org/embulk/parser/poi_excel/PoiExcelParserPlugin.java

@@ -0,0 +1,199 @@
+package org.embulk.parser.poi_excel;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.poi.EncryptedDocumentException;
+import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.parser.poi_excel.visitor.PoiExcelColumnVisitor;
+import org.embulk.parser.poi_excel.visitor.PoiExcelVisitorFactory;
+import org.embulk.parser.poi_excel.visitor.PoiExcelVisitorValue;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.FileInputInputStream;
+
+import com.google.common.base.Optional;
+import com.ibm.icu.text.MessageFormat;
+
+public class PoiExcelParserPlugin implements ParserPlugin {
+
+	public static final String TYPE = "poi_excel";
+
+	public interface PluginTask extends Task, TimestampParser.Task {
+		@Config("sheet")
+		@ConfigDefault("\"Sheet1\"")
+		public String getSheet();
+
+		@Config("skip_header_lines")
+		@ConfigDefault("0")
+		public int getSkipHeaderLines();
+
+		// search merged cell if cellType=BLANK
+		@Config("search_merged_cell")
+		@ConfigDefault("true")
+		public boolean getSearchMergedCell();
+
+		@Config("formula_replace")
+		@ConfigDefault("null")
+		public Optional<List<FormulaReplaceTask>> getFormulaReplace();
+
+		// true: set null if formula error
+		// false: throw exception if formula error
+		@Config("formula_error_null")
+		@ConfigDefault("false")
+		public boolean getFormulaErrorNull();
+
+		// true: set null if cell value error
+		// false: set error code if cell value error
+		@Config("cell_error_null")
+		@ConfigDefault("true")
+		public boolean getCellErrorNull();
+
+		@Config("flush_count")
+		@ConfigDefault("100")
+		public int getFlushCount();
+
+		@Config("columns")
+		public SchemaConfig getColumns();
+	}
+
+	public interface ColumnOptionTask extends Task {
+
+		/**
+		 * @see PoiExcelColumnValueType
+		 * @return value_type
+		 */
+		@Config("value")
+		@ConfigDefault("\"cell_value\"")
+		public String getValueType();
+
+		public void setValueTypeEnum(PoiExcelColumnValueType valueType);
+
+		public PoiExcelColumnValueType getValueTypeEnum();
+
+		public void setValueTypeSuffix(String suffix);
+
+		public String getValueTypeSuffix();
+
+		// A,B,... or number(1 origin)
+		@Config("column_number")
+		@ConfigDefault("null")
+		public Optional<String> getColumnNumber();
+
+		public void setColumnIndex(int index);
+
+		public int getColumnIndex();
+
+		@Config("search_merged_cell")
+		@ConfigDefault("null")
+		public Optional<Boolean> getSearchMergedCell();
+
+		@Config("formula_replace")
+		@ConfigDefault("null")
+		public Optional<List<FormulaReplaceTask>> getFormulaReplace();
+
+		@Config("formula_error_null")
+		@ConfigDefault("null")
+		public Optional<Boolean> getFormulaErrorNull();
+
+		@Config("cell_error_null")
+		@ConfigDefault("null")
+		public Optional<Boolean> getCellErrorNull();
+
+		// use when value_type=cell_style, cell_font, ...
+		@Config("attribute_name")
+		@ConfigDefault("null")
+		public Optional<List<String>> getAttributeName();
+	}
+
+	public interface FormulaReplaceTask extends Task {
+
+		@Config("regex")
+		public String getRegex();
+
+		// replace string
+		// use variable: "${row}"
+		@Config("to")
+		public String getTo();
+	}
+
+	@Override
+	public void transaction(ConfigSource config, ParserPlugin.Control control) {
+		PluginTask task = config.loadConfig(PluginTask.class);
+
+		Schema schema = task.getColumns().toSchema();
+
+		control.run(task.dump(), schema);
+	}
+
+	@Override
+	public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output) {
+		PluginTask task = taskSource.loadTask(PluginTask.class);
+
+		try (FileInputInputStream is = new FileInputInputStream(input)) {
+			while (is.nextFile()) {
+				Workbook workbook;
+				try {
+					workbook = WorkbookFactory.create(is);
+				} catch (IOException | EncryptedDocumentException | InvalidFormatException e) {
+					throw new RuntimeException(e);
+				}
+
+				String sheetName = task.getSheet();
+				Sheet sheet = workbook.getSheet(sheetName);
+				if (sheet == null) {
+					throw new RuntimeException(MessageFormat.format("not found sheet={0}", sheetName));
+				}
+
+				run(task, schema, sheet, output);
+			}
+		}
+	}
+
+	protected void run(PluginTask task, Schema schema, Sheet sheet, PageOutput output) {
+		int skipHeaderLines = task.getSkipHeaderLines();
+		final int flushCount = task.getFlushCount();
+
+		try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+			PoiExcelVisitorFactory factory = newPoiExcelVisitorFactory(task, sheet, pageBuilder);
+			PoiExcelColumnVisitor visitor = factory.getPoiExcelColumnVisitor();
+
+			int count = 0;
+			for (final Row row : sheet) {
+				if (row.getRowNum() < skipHeaderLines) {
+					continue;
+				}
+
+				visitor.setRow(row);
+				schema.visitColumns(visitor);
+				pageBuilder.addRecord();
+
+				if (++count >= flushCount) {
+					pageBuilder.flush();
+					count = 0;
+				}
+			}
+			pageBuilder.finish();
+		}
+	}
+
+	protected PoiExcelVisitorFactory newPoiExcelVisitorFactory(PluginTask task, Sheet sheet, PageBuilder pageBuilder) {
+		PoiExcelVisitorValue visitorValue = new PoiExcelVisitorValue(task, sheet, pageBuilder);
+		return new PoiExcelVisitorFactory(visitorValue);
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/AbstractPoiExcelCellAttributeVisitor.java

@@ -0,0 +1,133 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.text.MessageFormat;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Color;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.StringType;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Optional;
+
+public abstract class AbstractPoiExcelCellAttributeVisitor<A> {
+
+	protected final PoiExcelVisitorValue visitorValue;
+	protected final PageBuilder pageBuilder;
+
+	public AbstractPoiExcelCellAttributeVisitor(PoiExcelVisitorValue visitorValue) {
+		this.visitorValue = visitorValue;
+		this.pageBuilder = visitorValue.getPageBuilder();
+	}
+
+	public void visit(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+		A source = getAttributeSource(column, option, cell);
+		if (source == null) {
+			pageBuilder.setNull(column);
+			return;
+		}
+
+		String suffix = option.getValueTypeSuffix();
+		if (suffix != null) {
+			visitKey(column, option, suffix, cell, source, visitor);
+		} else {
+			visitJson(column, option, cell, source, visitor);
+		}
+	}
+
+	protected abstract A getAttributeSource(Column column, ColumnOptionTask option, Cell cell);
+
+	private void visitKey(Column column, ColumnOptionTask option, String key, Cell cell, A source, CellVisitor visitor) {
+		Object value = getAttributeValue(column, option, cell, source, key);
+		if (value == null) {
+			pageBuilder.setNull(column);
+		} else if (value instanceof String) {
+			visitor.visitCellValueString(column, source, (String) value);
+		} else if (value instanceof Long) {
+			visitor.visitValueLong(column, source, (Long) value);
+		} else if (value instanceof Boolean) {
+			visitor.visitCellValueBoolean(column, source, (Boolean) value);
+		} else if (value instanceof Double) {
+			visitor.visitCellValueNumeric(column, source, (Double) value);
+		} else {
+			throw new IllegalStateException(MessageFormat.format("unsupported conversion. type={0}, value={1}", value
+					.getClass().getName(), value));
+		}
+	}
+
+	private void visitJson(Column column, ColumnOptionTask option, Cell cell, A source, CellVisitor visitor) {
+		Map<String, Object> result;
+
+		Optional<List<String>> nameOption = option.getAttributeName();
+		if (nameOption.isPresent()) {
+			result = new LinkedHashMap<>();
+
+			List<String> list = nameOption.get();
+			for (String key : list) {
+				Object value = getAttributeValue(column, option, cell, source, key);
+				result.put(key, value);
+			}
+		} else {
+			result = new TreeMap<>();
+
+			Collection<String> keys = getAttributeSupplierMap().keySet();
+			for (String key : keys) {
+				if (acceptKey(key)) {
+					Object value = getAttributeValue(column, option, cell, source, key);
+					result.put(key, value);
+				}
+			}
+		}
+
+		String json;
+		try {
+			ObjectMapper mapper = new ObjectMapper();
+			json = mapper.writeValueAsString(result);
+		} catch (JsonProcessingException e) {
+			throw new RuntimeException(e);
+		}
+
+		visitor.visitCellValueString(column, cell, json);
+	}
+
+	protected boolean acceptKey(String key) {
+		return true;
+	}
+
+	private Object getAttributeValue(Column column, ColumnOptionTask option, Cell cell, A source, String key) {
+		Map<String, AttributeSupplier<A>> map = getAttributeSupplierMap();
+		AttributeSupplier<A> supplier = map.get(key.toLowerCase());
+		if (supplier == null) {
+			throw new UnsupportedOperationException(MessageFormat.format(
+					"unsupported attribute name={0}, choose in {1}", key, new TreeSet<>(map.keySet())));
+		}
+		Object value = supplier.get(column, cell, source);
+
+		if (value instanceof Color) {
+			int rgb = PoiExcelColorVisitor.getRGB((Color) value);
+			if (column.getType() instanceof StringType) {
+				value = String.format("%06x", rgb);
+			} else {
+				value = (long) rgb;
+			}
+		}
+		return value;
+	}
+
+	// @FunctionalInterface
+	protected static interface AttributeSupplier<A> {
+		public Object get(Column column, Cell cell, A source);
+	}
+
+	protected abstract Map<String, AttributeSupplier<A>> getAttributeSupplierMap();
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellCommentVisitor.java

@@ -0,0 +1,68 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Comment;
+import org.apache.poi.ss.usermodel.RichTextString;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.spi.Column;
+
+public class PoiExcelCellCommentVisitor extends AbstractPoiExcelCellAttributeVisitor<Comment> {
+
+	public PoiExcelCellCommentVisitor(PoiExcelVisitorValue visitorValue) {
+		super(visitorValue);
+	}
+
+	@Override
+	protected Comment getAttributeSource(Column column, ColumnOptionTask option, Cell cell) {
+		return cell.getCellComment();
+	}
+
+	@Override
+	protected Map<String, AttributeSupplier<Comment>> getAttributeSupplierMap() {
+		return SUPPLIER_MAP;
+	}
+
+	protected static final Map<String, AttributeSupplier<Comment>> SUPPLIER_MAP;
+	static {
+		Map<String, AttributeSupplier<Comment>> map = new HashMap<>(32);
+		map.put("author", new AttributeSupplier<Comment>() {
+			@Override
+			public Object get(Column column, Cell cell, Comment comment) {
+				return comment.getAuthor();
+			}
+		});
+		map.put("column", new AttributeSupplier<Comment>() {
+			@Override
+			public Object get(Column column, Cell cell, Comment comment) {
+				return (long) comment.getColumn();
+			}
+		});
+		map.put("row", new AttributeSupplier<Comment>() {
+			@Override
+			public Object get(Column column, Cell cell, Comment comment) {
+				return (long) comment.getRow();
+			}
+		});
+		map.put("is_visible", new AttributeSupplier<Comment>() {
+			@Override
+			public Object get(Column column, Cell cell, Comment comment) {
+				return comment.isVisible();
+			}
+		});
+		map.put("string", new AttributeSupplier<Comment>() {
+			@Override
+			public Object get(Column column, Cell cell, Comment comment) {
+				RichTextString rich = comment.getString();
+				return rich.getString();
+			}
+		});
+
+		// TODO getClientAnchor
+
+		SUPPLIER_MAP = Collections.unmodifiableMap(map);
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellFontVisitor.java

@@ -0,0 +1,117 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.Font;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFFont;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.spi.Column;
+
+public class PoiExcelCellFontVisitor extends AbstractPoiExcelCellAttributeVisitor<Font> {
+
+	public PoiExcelCellFontVisitor(PoiExcelVisitorValue visitorValue) {
+		super(visitorValue);
+	}
+
+	@Override
+	protected Font getAttributeSource(Column column, ColumnOptionTask option, Cell cell) {
+		CellStyle style = cell.getCellStyle();
+		short index = style.getFontIndex();
+		Workbook book = visitorValue.getSheet().getWorkbook();
+		return book.getFontAt(index);
+	}
+
+	@Override
+	protected Map<String, AttributeSupplier<Font>> getAttributeSupplierMap() {
+		return SUPPLIER_MAP;
+	}
+
+	protected static final Map<String, AttributeSupplier<Font>> SUPPLIER_MAP;
+	static {
+		Map<String, AttributeSupplier<Font>> map = new HashMap<>(32);
+		map.put("font_name", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return font.getFontName();
+			}
+		});
+		map.put("font_height", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getFontHeight();
+			}
+		});
+		map.put("font_height_in_points", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getFontHeightInPoints();
+			}
+		});
+		map.put("italic", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return font.getItalic();
+			}
+		});
+		map.put("strikeout", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return font.getStrikeout();
+			}
+		});
+		map.put("color", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				if (font instanceof XSSFFont) {
+					return ((XSSFFont) font).getXSSFColor();
+				} else {
+					Workbook book = cell.getSheet().getWorkbook();
+					short color = font.getColor();
+					return PoiExcelColorVisitor.getHssfColor(book, color);
+				}
+			}
+		});
+		map.put("type_offset", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getTypeOffset();
+			}
+		});
+		map.put("underline", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getUnderline();
+			}
+		});
+		map.put("char_set", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getCharSet();
+			}
+		});
+		map.put("index", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getIndex();
+			}
+		});
+		map.put("boldweight", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return (long) font.getBoldweight();
+			}
+		});
+		map.put("bold", new AttributeSupplier<Font>() {
+			@Override
+			public Object get(Column column, Cell cell, Font font) {
+				return font.getBold();
+			}
+		});
+		SUPPLIER_MAP = Collections.unmodifiableMap(map);
+	}
+}