embulk-output-oracle 0.2.1 → 0.2.2

data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java

@@ -12,16 +12,18 @@ public class OracleOutputConnector
 {
     private final String url;
     private final Properties properties;
+    private final boolean direct;
 
-    public OracleOutputConnector(String url, Properties properties)
+    public OracleOutputConnector(String url, Properties properties, boolean direct)
     {
         try {
-        	Class.forName("oracle.jdbc.OracleDriver");
+            Class.forName("oracle.jdbc.OracleDriver");
         } catch (Exception ex) {
             throw new RuntimeException(ex);
         }
         this.url = url;
         this.properties = properties;
+        this.direct = direct;
     }
 
     @Override
@@ -29,12 +31,12 @@ public class OracleOutputConnector
     {
         Connection c = DriverManager.getConnection(url, properties);
         if (c == null) {
-        	// driver.connect returns null when url is "jdbc:mysql://...".
-        	throw new SQLException("Invalid url : " + url);
+            // driver.connect returns null when url is "jdbc:mysql://...".
+            throw new SQLException("Invalid url : " + url);
         }
 
         try {
-            OracleOutputConnection con = new OracleOutputConnection(c, autoCommit);
+            OracleOutputConnection con = new OracleOutputConnection(c, autoCommit, direct);
             c = null;
             return con;
 

data/src/main/java/org/embulk/output/oracle/TimestampFormat.java

@@ -0,0 +1,37 @@
+package org.embulk.output.oracle;
+
+import java.sql.Timestamp;
+import java.text.FieldPosition;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+
+public class TimestampFormat extends SimpleDateFormat
+{
+
+    private final int scale;
+
+    public TimestampFormat(String pattern, int scale)
+    {
+        super(pattern);
+
+        this.scale = scale;
+    }
+
+    @Override
+    public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos)
+    {
+        StringBuffer buffer = super.format(date, toAppendTo, pos);
+        if (scale > 0) {
+            buffer.append('.');
+            String nanos = Integer.toString(((Timestamp)date).getNanos());
+            int zeros = Math.min(scale, 9 - nanos.length());
+            for (int i = 0; i < zeros; i++) {
+                buffer.append('0');
+            }
+            buffer.append(nanos.substring(0, scale - zeros));
+        }
+        return buffer;
+    }
+
+}

data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java

@@ -0,0 +1,26 @@
+package org.embulk.output.oracle.oci;
+
+public class ColumnDefinition
+{
+    public static int SQLT_CHR = 1;
+    public static int SQLT_INT = 3;
+
+    public final String columnName;
+    public final int columnType;
+    public final int columnSize;
+    public final String columnDateFormat;
+
+
+    public ColumnDefinition(String columnName, int columnType, int columnSize, String columnDateFormat)
+    {
+        this.columnName = columnName;
+        this.columnType = columnType;
+        this.columnSize = columnSize;
+        this.columnDateFormat = columnDateFormat;
+    }
+
+    public ColumnDefinition(String columnName, int columnType, int columnSize)
+    {
+        this(columnName, columnType, columnSize, null);
+    }
+}

data/src/main/java/org/embulk/output/oracle/oci/OCI.java

@@ -0,0 +1,139 @@
+package org.embulk.output.oracle.oci;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import com.google.common.io.Files;
+
+
+public class OCI
+{
+    private static final Logger logger = Exec.getLogger(OCI.class);
+
+    private static final String PLUGIN_NAME = "embulk-output-oracle";
+
+    static {
+        try {
+            loadLibrary();
+        } catch (URISyntaxException | IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public native byte[] createContext();
+
+    public native byte[] getLasetMessage(byte[] context);
+
+    public native boolean open(byte[] context, String dbName, String userName, String password);
+
+    public native boolean prepareLoad(byte[] context, TableDefinition tableDefinition);
+
+    public native boolean loadBuffer(byte[] context, byte[] buffer, int rowCount);
+
+    public native boolean commit(byte[] context);
+
+    public native boolean rollback(byte[] context);
+
+    public native void close(byte[] context);
+
+    private static void loadLibrary() throws URISyntaxException, IOException
+    {
+        loadLibrary(getPluginRoot());
+    }
+
+    private static File getPluginRoot() throws MalformedURLException, URISyntaxException
+    {
+        URL url = OCI.class.getResource("/" + OCI.class.getName().replace('.', '/') + ".class");
+        if (url.toString().startsWith("jar:")) {
+            url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+        }
+
+        File folder = new File(url.toURI()).getParentFile();
+        for (;; folder = folder.getParentFile()) {
+            if (folder == null) {
+                String message = String.format("OCI : %s folder not found.", PLUGIN_NAME);
+                throw new RuntimeException(message);
+            }
+
+            if (folder.getName().startsWith(PLUGIN_NAME)) {
+                return folder;
+            }
+        }
+    }
+
+    private static void loadLibrary(File folder) throws IOException
+    {
+        File lib = new File(new File(folder, "lib"), "embulk");
+
+        String osName = System.getProperty("os.name");
+        String osArch = System.getProperty("os.arch");
+
+        String libraryName = System.mapLibraryName(PLUGIN_NAME);
+        File libFolder = null;
+
+        if (osName.startsWith("Windows")) {
+            if (osArch.endsWith("64")) {
+                libFolder = new File(lib, "win_x64");
+            } else if (osArch.equals("x86")) {
+                libFolder = new File(lib, "win_x86");
+            }
+        } else if (osName.equals("Linux")) {
+            if (osArch.endsWith("64")) {
+                libFolder = new File(lib, "linux_x64");
+            } else if (osArch.equals("x86")) {
+                libFolder = new File(lib, "linux_x86");
+            }
+        }
+
+        if (libFolder == null) {
+            logger.error(String.format("OCI : library \"%s\" for %s %s doesn't exist in lib folder.", libraryName, osName, osArch));
+            return;
+        }
+
+        File libFile = new File(libFolder, libraryName);
+        if (!libFile.exists()) {
+            logger.error(String.format("OCI : library \"%s\" doesn't exist.", libFile.getAbsolutePath()));
+            return;
+        }
+
+        logger.info(String.format("OCI : load library \"%s\".", libFile.getAbsolutePath()));
+
+        File tempFolder = new File(lib, "temp");
+        tempFolder.mkdirs();
+
+        long currentTime = System.currentTimeMillis();
+        File[] files = tempFolder.listFiles();
+        if (files != null) {
+            for (File file : files) {
+                // delete old temporary files.
+                // if another plugin is using a file, it cannot be deleted.
+                // don't delete a recent file because another plugin may intend to use it.
+                if (file.isFile() && file.getName().startsWith(PLUGIN_NAME) && file.lastModified() < currentTime - 60000) {
+                    file.delete();
+                }
+            }
+        }
+
+        String extension = libraryName.replaceAll("^[^\\.]*", "");
+        for (int i = 0; i < 10; i++) {
+            File tempLibFile = new File(tempFolder, PLUGIN_NAME + "-" + currentTime + "-" + i + extension);
+            if (tempLibFile.createNewFile()) {
+                // copy and load the library because different plugins cannot load the same library.
+                logger.info(String.format("OCI : create temporary library \"%s\".", tempLibFile.getAbsolutePath()));
+                Files.copy(libFile, tempLibFile);
+                System.load(tempLibFile.getAbsolutePath());
+                tempLibFile.deleteOnExit();
+                // but may not be deleted because loaded as a library.
+                return;
+            }
+        }
+
+        logger.error("OCI : cannot create temporary library.");
+    }
+}

data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java

@@ -0,0 +1,64 @@
+package org.embulk.output.oracle.oci;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+public class OCIManager
+{
+    private static class OCIWrapperAndCounter {
+        public OCIWrapper oci;
+        public int counter;
+    }
+
+
+    private final Logger logger = Exec.getLogger(getClass());
+
+    private Map<Object, OCIWrapperAndCounter> ociAndCounters = new HashMap<Object, OCIWrapperAndCounter>();
+
+
+    public OCIWrapper open(Object key, String dbName, String userName, String password, TableDefinition tableDefinition) throws SQLException
+    {
+        synchronized(ociAndCounters) {
+            OCIWrapperAndCounter ociAndCounter;
+            if (ociAndCounters.containsKey(key)) {
+                ociAndCounter = ociAndCounters.get(key);
+            } else {
+                logger.info(String.format("OCI : open for %s.", key));
+                ociAndCounter = new OCIWrapperAndCounter();
+                ociAndCounter.oci = new OCIWrapper();
+                ociAndCounter.oci.open(dbName, userName, password);
+                ociAndCounter.oci.prepareLoad(tableDefinition);
+                ociAndCounters.put(key, ociAndCounter);
+            }
+            ociAndCounter.counter++;
+            return ociAndCounter.oci;
+        }
+    }
+
+    public OCIWrapper get(Object key)
+    {
+        synchronized(ociAndCounters) {
+            return ociAndCounters.get(key).oci;
+        }
+    }
+
+    public void close(Object key) throws SQLException
+    {
+        synchronized(ociAndCounters) {
+            OCIWrapperAndCounter ociAndCounter = ociAndCounters.get(key);
+            if (ociAndCounter != null) {
+                ociAndCounter.counter--;
+                if (ociAndCounter.counter == 0) {
+                    logger.info(String.format("OCI : close for %s.", key));
+                    ociAndCounter.oci.close();
+                    ociAndCounters.remove(key);
+                }
+            }
+        }
+    }
+
+}

data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java

@@ -0,0 +1,96 @@
+package org.embulk.output.oracle.oci;
+
+import java.nio.charset.Charset;
+import java.sql.SQLException;
+
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+
+public class OCIWrapper implements AutoCloseable
+{
+    private final Logger logger = Exec.getLogger(getClass());
+
+    private final OCI oci = new OCI();
+    // used for messages
+    private final Charset defaultCharset;
+    private byte[] context;
+    private boolean errorOccured;
+    private boolean committedOrRollbacked;
+
+
+    public OCIWrapper()
+    {
+        // enable to change default encoding for test
+        defaultCharset = Charset.forName(System.getProperty("file.encoding"));
+        context = oci.createContext();
+    }
+
+    public void open(String dbName, String userName, String password) throws SQLException
+    {
+        if (!oci.open(context, dbName, userName, password)) {
+            throwException();
+        }
+    }
+
+    public void prepareLoad(TableDefinition tableDefinition) throws SQLException
+    {
+        if (!oci.prepareLoad(context, tableDefinition)) {
+            throwException();
+        }
+    }
+
+    public void loadBuffer(byte[] buffer, int rowCount) throws SQLException
+    {
+        if (!oci.loadBuffer(context, buffer, rowCount)) {
+            throwException();
+        }
+    }
+
+    public void commit() throws SQLException
+    {
+        committedOrRollbacked = true;
+        logger.info("OCI : start to commit.");
+        if (!oci.commit(context)) {
+            throwException();
+        }
+    }
+
+    public void rollback() throws SQLException
+    {
+        committedOrRollbacked = true;
+        logger.info("OCI : start to rollback.");
+        if (!oci.rollback(context)) {
+            throwException();
+        }
+    }
+
+    private void throwException() throws SQLException
+    {
+        errorOccured = true;
+        String message = new String(oci.getLasetMessage(context), defaultCharset);
+        logger.error(message);
+        throw new SQLException(message);
+    }
+
+
+    @Override
+    public void close() throws SQLException
+    {
+        if (context != null) {
+            try {
+                if (!committedOrRollbacked) {
+                    if (errorOccured) {
+                        rollback();
+                    } else {
+                        commit();
+                    }
+                }
+            } finally {
+                oci.close(context);
+                context = null;
+            }
+        }
+    }
+
+}

data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java

@@ -0,0 +1,99 @@
+package org.embulk.output.oracle.oci;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+public class RowBuffer
+{
+    private final TableDefinition table;
+    private final int rowCount;
+    private final byte[] buffer;
+    private int currentRow = 0;
+    private int currentColumn = 0;
+    private int currentPosition = 0;
+    private final Charset charset;
+
+
+    public RowBuffer(TableDefinition table, int rowCount, Charset charset)
+    {
+        this.table = table;
+        this.rowCount = rowCount;
+        this.charset = charset;
+
+        int rowSize = 0;
+        for (ColumnDefinition column : table.columns) {
+            rowSize += column.columnSize;
+        }
+
+        buffer = new byte[rowSize * rowCount];
+    }
+
+    public void addValue(int value)
+    {
+        if (isFull()) {
+            throw new IllegalStateException();
+        }
+
+        buffer[currentPosition] = (byte)value;
+        buffer[currentPosition + 1] = (byte)(value >> 8);
+        buffer[currentPosition + 2] = (byte)(value >> 16);
+        buffer[currentPosition + 3] = (byte)(value >> 24);
+
+        next();
+    }
+
+    public void addValue(String value)
+    {
+        if (isFull()) {
+            throw new IllegalStateException();
+        }
+
+        ByteBuffer bytes = charset.encode(value);
+        int length = bytes.remaining();
+        // TODO:warning or error if truncated
+        bytes.get(buffer, currentPosition, length);
+        if (length < table.columns[currentColumn].columnSize) {
+            buffer[currentPosition + length] = 0;
+        }
+
+        next();
+    }
+
+    private void next()
+    {
+        currentPosition += table.columns[currentColumn].columnSize;
+        currentColumn++;
+        if (currentColumn == table.columns.length) {
+            currentColumn = 0;
+            currentRow++;
+        }
+    }
+
+    public byte[] getBuffer()
+    {
+        return buffer;
+    }
+
+    public int getCurrentColumn()
+    {
+        return currentColumn;
+    }
+
+    public int getRowCount()
+    {
+        return currentRow;
+    }
+
+    public boolean isFull()
+    {
+        return currentRow >= rowCount;
+    }
+
+    public void clear()
+    {
+        currentPosition = 0;
+        currentRow = 0;
+        currentColumn = 0;
+    }
+
+}