embulk-parser-poi_excel 0.1.0

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellStyleVisitor.java

@@ -0,0 +1,205 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellStyle;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.type.StringType;
+
+public class PoiExcelCellStyleVisitor extends AbstractPoiExcelCellAttributeVisitor<CellStyle> {
+
+	public PoiExcelCellStyleVisitor(PoiExcelVisitorValue visitorValue) {
+		super(visitorValue);
+	}
+
+	@Override
+	protected CellStyle getAttributeSource(Column column, ColumnOptionTask option, Cell cell) {
+		return cell.getCellStyle();
+	}
+
+	protected boolean acceptKey(String key) {
+		if (key.equals("border")) {
+			return false;
+		}
+		return true;
+	}
+
+	@Override
+	protected Map<String, AttributeSupplier<CellStyle>> getAttributeSupplierMap() {
+		return SUPPLIER_MAP;
+	}
+
+	protected static final Map<String, AttributeSupplier<CellStyle>> SUPPLIER_MAP;
+
+	static {
+		Map<String, AttributeSupplier<CellStyle>> map = new HashMap<>(32);
+		map.put("alignment", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getAlignment();
+			}
+		});
+		map.put("border", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				int n0 = style.getBorderTop();
+				int n1 = style.getBorderBottom();
+				int n2 = style.getBorderLeft();
+				int n3 = style.getBorderRight();
+				if (column.getType() instanceof StringType) {
+					return String.format("%02x%02x%02x%02x", n0, n1, n2, n3);
+				}
+				return (long) ((n0 << 24) | (n1 << 16) | (n2 << 8) | n3);
+			}
+		});
+		map.put("border_bottom", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getBorderBottom();
+			}
+		});
+		map.put("border_left", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getBorderLeft();
+			}
+		});
+		map.put("border_right", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getBorderRight();
+			}
+		});
+		map.put("border_top", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getBorderTop();
+			}
+		});
+		map.put("border_bottom_color", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				if (style instanceof XSSFCellStyle) {
+					return ((XSSFCellStyle) style).getBottomBorderXSSFColor();
+				} else {
+					Workbook book = cell.getSheet().getWorkbook();
+					short color = style.getBottomBorderColor();
+					return PoiExcelColorVisitor.getHssfColor(book, color);
+				}
+			}
+		});
+		map.put("border_left_color", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				if (style instanceof XSSFCellStyle) {
+					return ((XSSFCellStyle) style).getLeftBorderXSSFColor();
+				} else {
+					Workbook book = cell.getSheet().getWorkbook();
+					short color = style.getLeftBorderColor();
+					return PoiExcelColorVisitor.getHssfColor(book, color);
+				}
+			}
+		});
+		map.put("border_right_color", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				if (style instanceof XSSFCellStyle) {
+					return ((XSSFCellStyle) style).getRightBorderXSSFColor();
+				} else {
+					Workbook book = cell.getSheet().getWorkbook();
+					short color = style.getRightBorderColor();
+					return PoiExcelColorVisitor.getHssfColor(book, color);
+				}
+			}
+		});
+		map.put("border_top_color", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				if (style instanceof XSSFCellStyle) {
+					return ((XSSFCellStyle) style).getTopBorderXSSFColor();
+				} else {
+					Workbook book = cell.getSheet().getWorkbook();
+					short color = style.getTopBorderColor();
+					return PoiExcelColorVisitor.getHssfColor(book, color);
+				}
+			}
+		});
+		map.put("data_format", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				if (column.getType() instanceof StringType) {
+					return style.getDataFormatString();
+				} else {
+					return (long) style.getDataFormat();
+				}
+			}
+		});
+		map.put("fill_background_color", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return style.getFillBackgroundColorColor();
+			}
+		});
+		map.put("fill_foreground_color", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return style.getFillForegroundColorColor();
+			}
+		});
+		map.put("fill_pattern", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getFillPattern();
+			}
+		});
+		map.put("font_index", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getFontIndex();
+			}
+		});
+		map.put("hidden", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return style.getHidden();
+			}
+		});
+		map.put("indention", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getIndention();
+			}
+		});
+		map.put("locked", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return style.getLocked();
+			}
+		});
+		map.put("rotation", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getRotation();
+			}
+		});
+		map.put("vertical_alignment", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return (long) style.getVerticalAlignment();
+			}
+		});
+		map.put("wrap_text", new AttributeSupplier<CellStyle>() {
+			@Override
+			public Object get(Column column, Cell cell, CellStyle style) {
+				return style.getWrapText();
+			}
+		});
+		SUPPLIER_MAP = Collections.unmodifiableMap(map);
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelCellVisitor.java

@@ -0,0 +1,194 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.text.MessageFormat;
+import java.util.List;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellValue;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.FormulaEvaluator;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.ss.util.CellRangeAddress;
+import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.FormulaReplaceTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
+import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.slf4j.Logger;
+
+import com.google.common.base.Optional;
+
+public class PoiExcelCellVisitor {
+	private final Logger log = Exec.getLogger(getClass());
+
+	protected final PoiExcelVisitorValue visitorValue;
+	protected final PageBuilder pageBuilder;
+
+	public PoiExcelCellVisitor(PoiExcelVisitorValue visitorValue) {
+		this.visitorValue = visitorValue;
+		this.pageBuilder = visitorValue.getPageBuilder();
+	}
+
+	public void visitCellValue(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+		assert cell != null;
+
+		int cellType = cell.getCellType();
+		switch (cellType) {
+		case Cell.CELL_TYPE_NUMERIC:
+			visitor.visitCellValueNumeric(column, cell, cell.getNumericCellValue());
+			return;
+		case Cell.CELL_TYPE_STRING:
+			visitor.visitCellValueString(column, cell, cell.getStringCellValue());
+			return;
+		case Cell.CELL_TYPE_FORMULA:
+			PoiExcelColumnValueType valueType = option.getValueTypeEnum();
+			if (valueType == PoiExcelColumnValueType.CELL_FORMULA) {
+				visitor.visitCellFormula(column, cell);
+			} else {
+				visitCellValueFormula(column, option, cell, visitor);
+			}
+			return;
+		case Cell.CELL_TYPE_BLANK:
+			visitCellValueBlank(column, option, cell, visitor);
+			return;
+		case Cell.CELL_TYPE_BOOLEAN:
+			visitor.visitCellValueBoolean(column, cell, cell.getBooleanCellValue());
+			return;
+		case Cell.CELL_TYPE_ERROR:
+			visitCellValueError(column, option, cell, cell.getErrorCellValue(), visitor);
+			return;
+		default:
+			throw new IllegalStateException(MessageFormat.format("unsupported POI cellType={0}", cellType));
+		}
+	}
+
+	protected void visitCellValueBlank(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+		assert cell.getCellType() == Cell.CELL_TYPE_BLANK;
+
+		PluginTask task = visitorValue.getPluginTask();
+		boolean search = option.getSearchMergedCell().or(task.getSearchMergedCell());
+		if (!search) {
+			visitor.visitCellValueBlank(column, cell);
+			return;
+		}
+
+		int r = cell.getRowIndex();
+		int c = cell.getColumnIndex();
+
+		Sheet sheet = cell.getSheet();
+		int size = sheet.getNumMergedRegions();
+		for (int i = 0; i < size; i++) {
+			CellRangeAddress range = visitorValue.getSheet().getMergedRegion(i);
+			if (range.isInRange(r, c)) {
+				Row firstRow = sheet.getRow(range.getFirstRow());
+				if (firstRow == null) {
+					visitCellNull(column);
+					return;
+				}
+				Cell firstCell = firstRow.getCell(range.getFirstColumn());
+				if (firstCell == null) {
+					visitCellNull(column);
+					return;
+				}
+
+				visitCellValue(column, option, firstCell, visitor);
+				return;
+			}
+		}
+
+		visitor.visitCellValueBlank(column, cell);
+	}
+
+	protected void visitCellValueFormula(Column column, ColumnOptionTask option, Cell cell, CellVisitor visitor) {
+		assert cell.getCellType() == Cell.CELL_TYPE_FORMULA;
+
+		Optional<List<FormulaReplaceTask>> replaceOption = option.getFormulaReplace();
+		if (!replaceOption.isPresent()) {
+			PluginTask task = visitorValue.getPluginTask();
+			replaceOption = task.getFormulaReplace();
+		}
+		if (replaceOption.isPresent()) {
+			String formula = cell.getCellFormula();
+			String old = formula;
+
+			List<FormulaReplaceTask> list = replaceOption.get();
+			for (FormulaReplaceTask replace : list) {
+				String regex = replace.getRegex();
+				String replacement = replace.getTo();
+
+				replacement = replacement.replace("${row}", Integer.toString(cell.getRowIndex() + 1));
+
+				formula = formula.replaceAll(regex, replacement);
+			}
+
+			if (!formula.equals(old)) {
+				log.debug("formula replaced. old=\"{}\", new=\"{}\"", old, formula);
+				try {
+					cell.setCellFormula(formula);
+				} catch (Exception e) {
+					throw new RuntimeException(MessageFormat.format("setCellFormula error. formula={0}", formula), e);
+				}
+			}
+		}
+
+		CellValue cellValue;
+		try {
+			Workbook book = cell.getSheet().getWorkbook();
+			CreationHelper helper = book.getCreationHelper();
+			FormulaEvaluator evaluator = helper.createFormulaEvaluator();
+			cellValue = evaluator.evaluate(cell);
+		} catch (Exception e) {
+			PluginTask task = visitorValue.getPluginTask();
+			boolean setNull = option.getFormulaErrorNull().or(task.getFormulaErrorNull());
+			if (setNull) {
+				pageBuilder.setNull(column);
+				return;
+			}
+
+			throw new RuntimeException(MessageFormat.format("evaluate error. formula={0}", cell.getCellFormula()), e);
+		}
+
+		int cellType = cellValue.getCellType();
+		switch (cellType) {
+		case Cell.CELL_TYPE_NUMERIC:
+			visitor.visitCellValueNumeric(column, cellValue, cellValue.getNumberValue());
+			return;
+		case Cell.CELL_TYPE_STRING:
+			visitor.visitCellValueString(column, cellValue, cellValue.getStringValue());
+			return;
+		case Cell.CELL_TYPE_BLANK:
+			visitor.visitCellValueBlank(column, cellValue);
+			return;
+		case Cell.CELL_TYPE_BOOLEAN:
+			visitor.visitCellValueBoolean(column, cellValue, cellValue.getBooleanValue());
+			return;
+		case Cell.CELL_TYPE_ERROR:
+			visitCellValueError(column, option, cellValue, cellValue.getErrorValue(), visitor);
+			return;
+		case Cell.CELL_TYPE_FORMULA:
+		default:
+			throw new IllegalStateException(MessageFormat.format("unsupported POI cellType={0}", cellType));
+		}
+	}
+
+	protected void visitCellValueError(Column column, ColumnOptionTask option, Object cell, int errorCode,
+			CellVisitor visitor) {
+		PluginTask task = visitorValue.getPluginTask();
+		boolean setNull = option.getCellErrorNull().or(task.getCellErrorNull());
+		if (setNull) {
+			pageBuilder.setNull(column);
+			return;
+		}
+
+		visitor.visitCellValueError(column, cell, errorCode);
+	}
+
+	protected void visitCellNull(Column column) {
+		pageBuilder.setNull(column);
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelColorVisitor.java

@@ -0,0 +1,81 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.text.MessageFormat;
+
+import org.apache.poi.hssf.usermodel.HSSFPalette;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.hssf.util.HSSFColor;
+import org.apache.poi.ss.usermodel.Color;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFColor;
+import org.embulk.parser.poi_excel.visitor.embulk.CellVisitor;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.StringType;
+
+public class PoiExcelColorVisitor {
+
+	protected final PoiExcelVisitorValue visitorValue;
+
+	public PoiExcelColorVisitor(PoiExcelVisitorValue visitorValue) {
+		this.visitorValue = visitorValue;
+	}
+
+	public void visitCellColor(Column column, short colorIndex, CellVisitor visitor) {
+		Color color = getHssfColor(colorIndex);
+		visitCellColor(column, color, visitor);
+	}
+
+	public void visitCellColor(Column column, Color color, CellVisitor visitor) {
+		int rgb = getRGB(color);
+		if (rgb < 0) {
+			PageBuilder pageBuilder = visitorValue.getPageBuilder();
+			pageBuilder.setNull(column);
+			return;
+		}
+
+		if (column.getType() instanceof StringType) {
+			String s = String.format("%06x", rgb);
+			visitor.visitCellValueString(column, color, s);
+		} else {
+			visitor.visitValueLong(column, color, rgb);
+		}
+	}
+
+	public Color getHssfColor(short colorIndex) {
+		HSSFWorkbook book = (HSSFWorkbook) visitorValue.getSheet().getWorkbook();
+		return getHssfColor(book, colorIndex);
+	}
+
+	public static Color getHssfColor(Workbook workbook, short colorIndex) {
+		HSSFWorkbook book = (HSSFWorkbook) workbook;
+		HSSFPalette palette = book.getCustomPalette();
+		HSSFColor color = palette.getColor(colorIndex);
+		return color;
+	}
+
+	public static int getRGB(Color color) {
+		if (color == null) {
+			return -1;
+		}
+
+		int[] rgb = new int[3];
+		if (color instanceof HSSFColor) {
+			HSSFColor hssf = (HSSFColor) color;
+			short[] s = hssf.getTriplet();
+			rgb[0] = s[0] & 0xff;
+			rgb[1] = s[1] & 0xff;
+			rgb[2] = s[2] & 0xff;
+		} else if (color instanceof XSSFColor) {
+			XSSFColor xssf = (XSSFColor) color;
+			byte[] b = xssf.getRGB();
+			rgb[0] = b[0] & 0xff;
+			rgb[1] = b[1] & 0xff;
+			rgb[2] = b[2] & 0xff;
+		} else {
+			throw new IllegalStateException(MessageFormat.format("unsupported POI color={0}", color));
+		}
+
+		return (rgb[0] << 16) | (rgb[1] << 8) | rgb[2];
+	}
+}

data/src/main/java/org/embulk/parser/poi_excel/visitor/PoiExcelColumnIndex.java

@@ -0,0 +1,174 @@
+package org.embulk.parser.poi_excel.visitor;
+
+import java.text.MessageFormat;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.poi.ss.util.CellReference;
+import org.embulk.parser.poi_excel.PoiExcelColumnValueType;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.ColumnOptionTask;
+import org.embulk.parser.poi_excel.PoiExcelParserPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.slf4j.Logger;
+
+import com.google.common.base.Optional;
+
+public class PoiExcelColumnIndex {
+	private final Logger log = Exec.getLogger(getClass());
+
+	protected final Map<String, Integer> indexMap = new LinkedHashMap<>();
+
+	public void initializeColumnIndex(PluginTask task, List<ColumnOptionTask> columnOptions) {
+		int index = -1;
+		indexMap.clear();
+
+		Schema schema = task.getColumns().toSchema();
+		for (Column column : schema.getColumns()) {
+			ColumnOptionTask option = columnOptions.get(column.getIndex());
+
+			String type = option.getValueType().trim();
+			int n = type.indexOf('.');
+			if (n >= 0) {
+				String suffix = type.substring(n + 1).trim();
+				option.setValueTypeSuffix(suffix);
+				type = type.substring(0, n).trim();
+			}
+
+			PoiExcelColumnValueType valueType;
+			try {
+				valueType = PoiExcelColumnValueType.valueOf(type.toUpperCase());
+			} catch (Exception e) {
+				throw new RuntimeException(MessageFormat.format("illegal value_type={0}", type), e);
+			}
+			option.setValueTypeEnum(valueType);
+
+			if (valueType.useCell()) {
+				index = resolveColumnIndex(column, option, index, valueType);
+				if (index < 0) {
+					index = 0;
+				}
+				log.info("column.name={} <- cell column={}, value_type={}", column.getName(),
+						CellReference.convertNumToColString(index), valueType);
+				option.setColumnIndex(index);
+				indexMap.put(column.getName(), index);
+			}
+		}
+	}
+
+	protected int resolveColumnIndex(Column column, ColumnOptionTask option, int index,
+			PoiExcelColumnValueType valueType) {
+		Optional<String> numberOption = option.getColumnNumber();
+		if (numberOption.isPresent()) {
+			String columnNumber = numberOption.get();
+			if (columnNumber.length() >= 1) {
+				char c = columnNumber.charAt(0);
+				String arg = columnNumber.substring(1).trim();
+				switch (c) {
+				case '=':
+					return resolveSameColumnIndex(column, index, columnNumber, arg);
+				case '+':
+					return resolveNextColumnIndex(column, index, columnNumber, arg);
+				case '-':
+					return resolvePreviousColumnIndex(column, index, columnNumber, arg);
+				default:
+					break;
+				}
+			}
+			return convertColumnIndex(column, columnNumber);
+		} else {
+			if (valueType.nextIndex()) {
+				index++;
+			}
+			return index;
+		}
+	}
+
+	protected int resolveSameColumnIndex(Column column, int index, String columnNumber, String arg) {
+		if (arg.isEmpty()) {
+			return index;
+		}
+
+		Integer value = indexMap.get(arg);
+		if (value == null) {
+			throw new RuntimeException(MessageFormat.format("not found column name={0} before {1}", arg, column));
+		}
+		return value;
+	}
+
+	protected int resolveNextColumnIndex(Column column, int index, String columnNumber, String arg) {
+		if (index < 0) {
+			index = 0;
+		}
+		int add = 1;
+		if (!arg.isEmpty()) {
+			try {
+				add = Integer.parseInt(arg);
+			} catch (Exception e) {
+				Integer value = indexMap.get(arg);
+				if (value == null) {
+					throw new RuntimeException(
+							MessageFormat.format("not found column name={0} before {1}", arg, column));
+				}
+				index = value;
+				add = 1;
+			}
+		}
+
+		index += add;
+		checkIndex(column, index);
+		return index;
+	}
+
+	protected int resolvePreviousColumnIndex(Column column, int index, String columnNumber, String arg) {
+		if (index < 0) {
+			index = 0;
+		}
+		int sub = 1;
+		if (!arg.isEmpty()) {
+			try {
+				sub = Integer.parseInt(arg);
+			} catch (Exception e) {
+				Integer value = indexMap.get(arg);
+				if (value == null) {
+					throw new RuntimeException(
+							MessageFormat.format("not found column name={0} before {1}", arg, column));
+				}
+				index = value;
+				sub = 1;
+			}
+		}
+
+		index -= sub;
+		checkIndex(column, index);
+		return index;
+	}
+
+	protected void checkIndex(Column column, int index) {
+		if (index < 0) {
+			throw new RuntimeException(MessageFormat.format("column_number out of range at {0}", column));
+		}
+	}
+
+	protected int convertColumnIndex(Column column, String columnNumber) {
+		int index;
+		try {
+			char c = columnNumber.charAt(0);
+			if ('0' <= c && c <= '9') {
+				index = Integer.parseInt(columnNumber) - 1;
+			} else {
+				index = CellReference.convertColStringToIndex(columnNumber);
+			}
+		} catch (Exception e) {
+			throw new RuntimeException(MessageFormat.format("illegal column_number=\"{0}\" at {1}", columnNumber,
+					column), e);
+		}
+		if (index < 0) {
+			throw new RuntimeException(MessageFormat.format("illegal column_number=\"{0}\" at {1}", columnNumber,
+					column));
+		}
+		return index;
+	}
+}