embulk-output-oracle 0.6.3 → 0.6.4

data/src/main/java/org/embulk/output/oracle/oci/BulkOCI.java

@@ -0,0 +1,15 @@
+package org.embulk.output.oracle.oci;
+
+import jnr.ffi.Pointer;
+import jnr.ffi.types.u_int16_t;
+import jnr.ffi.types.u_int32_t;
+
+public interface BulkOCI {
+
+    short embulk_output_oracle_OCIDirPathColArrayEntriesSet(Pointer dpca,
+            Pointer errhp,
+            @u_int16_t short columnCount,
+            @u_int32_t int rowCount,
+            Pointer data,
+            Pointer sizes);
+}

data/src/main/java/org/embulk/output/oracle/oci/OCI.java

@@ -32,6 +32,7 @@ public interface OCI
     static int OCI_ATTR_SCHEMA_NAME = 9;
     static int OCI_ATTR_CHARSET_ID = 31;
     static int OCI_ATTR_DATEFORMAT = 75;
+    static int OCI_ATTR_BUF_SIZE = 77;
     static int OCI_ATTR_NUM_ROWS = 81;
     static int OCI_ATTR_NUM_COLS = 102;
     static int OCI_ATTR_LIST_COLUMNS = 103;

data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java

@@ -20,7 +20,8 @@ public class OCIManager
     private Map<Object, OCIWrapperAndCounter> ociAndCounters = new HashMap<Object, OCIWrapperAndCounter>();
 
 
-    public OCIWrapper open(Object key, String dbName, String userName, String password, TableDefinition tableDefinition) throws SQLException
+    public OCIWrapper open(Object key, String dbName, String userName, String password, TableDefinition tableDefinition, int bufferSize)
+            throws SQLException
     {
         synchronized(ociAndCounters) {
             OCIWrapperAndCounter ociAndCounter;
@@ -31,7 +32,7 @@ public class OCIManager
                 ociAndCounter = new OCIWrapperAndCounter();
                 ociAndCounter.oci = new OCIWrapper();
                 ociAndCounter.oci.open(dbName, userName, password);
-                ociAndCounter.oci.prepareLoad(tableDefinition);
+                ociAndCounter.oci.prepareLoad(tableDefinition, bufferSize);
                 ociAndCounters.put(key, ociAndCounter);
             }
             ociAndCounter.counter++;

data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java

@@ -1,13 +1,17 @@
 package org.embulk.output.oracle.oci;
 
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.sql.SQLException;
 
 import jnr.ffi.LibraryLoader;
+import jnr.ffi.Platform;
 import jnr.ffi.Pointer;
 import jnr.ffi.Runtime;
-import jnr.ffi.provider.BoundedMemoryIO;
 import jnr.ffi.provider.jffi.ArrayMemoryIO;
 import jnr.ffi.provider.jffi.ByteBufferMemoryIO;
 
@@ -17,7 +21,10 @@ import org.slf4j.Logger;
 
 public class OCIWrapper
 {
+    private static final String PLUGIN_NAME = "embulk-output-oracle";
+
     private static OCI oci;
+    private static BulkOCI bulkOci;
 
     private final Logger logger = Exec.getLogger(getClass());
 
@@ -31,6 +38,8 @@ public class OCIWrapper
 
     private TableDefinition tableDefinition;
     private int maxRowCount;
+    private long totalRows;
+    private int loadCount;
 
     private boolean errorOccured;
     private boolean committedOrRollbacked;
@@ -43,14 +52,17 @@ public class OCIWrapper
 
         synchronized (OCIWrapper.class) {
             if (oci == null) {
-                oci = loadLibrary();
+                oci = loadOCILibrary();
+            }
+            if (bulkOci == null) {
+                bulkOci = loadBulkOCILibrary(oci);
             }
         }
     }
 
-    private OCI loadLibrary()
+    private OCI loadOCILibrary()
     {
-        logger.info("Loading OCI library.");
+        logger.info("OCI : Loading OCI library.");
 
         // "oci" for Windows, "clntsh" for Linux
         StringBuilder libraryNames = new StringBuilder();
@@ -69,12 +81,57 @@ public class OCIWrapper
         throw new UnsatisfiedLinkError("Cannot find library: " + libraryNames);
     }
 
+    private BulkOCI loadBulkOCILibrary(OCI oci)
+    {
+        String libraryName = "embulk-output-oracle-oci";
+        logger.info("OCI : Loading " + libraryName + " library.");
+
+        Platform platform = Platform.getNativePlatform();
+
+        File folder = getPluginRoot();
+        folder = new File(new File(new File(new File(folder ,"lib"), "embulk"), "native"), platform.getName());
+
+        File file = new File(folder, System.mapLibraryName(libraryName));
+        if (!file.exists()) {
+            logger.info("OCI : Library '" + file.getAbsolutePath() + "' doesn't exist, so Java implementation is used instead.");
+            return new PrimitiveBulkOCI(oci);
+        }
+
+        logger.info("OCI : Library '" + file.getAbsolutePath() + "' is found.");
+        return LibraryLoader.create(BulkOCI.class).search(folder.getAbsolutePath()).failImmediately().load(libraryName);
+    }
+
+    private File getPluginRoot()
+    {
+        try {
+            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
+            if (url.toString().startsWith("jar:")) {
+                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+            }
+
+            File folder = new File(url.toURI()).getParentFile();
+            for (;; folder = folder.getParentFile()) {
+                if (folder == null) {
+                    String message = String.format("OCI : %s folder not found.", PLUGIN_NAME);
+                    throw new RuntimeException(message);
+                }
+
+                if (folder.getName().startsWith(PLUGIN_NAME)) {
+                    return folder;
+                }
+            }
+        } catch (MalformedURLException | URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public void open(String dbName, String userName, String password) throws SQLException
     {
         Pointer envHandlePointer = createPointerPointer();
+        // OCI_THREADED is not needed because synchronized in Java side.
         check("OCIEnvCreate", oci.OCIEnvCreate(
                 envHandlePointer,
-                OCI.OCI_THREADED | OCI.OCI_OBJECT,
+                /*OCI.OCI_THREADED |*/ OCI.OCI_OBJECT,
                 null,
                 null,
                 null,
@@ -126,10 +183,27 @@ public class OCIWrapper
         dpHandle = dpHandlePointer.getPointer(0);
     }
 
-    public void prepareLoad(TableDefinition tableDefinition) throws SQLException
+    public void prepareLoad(TableDefinition tableDefinition, int bufferSize) throws SQLException
     {
         this.tableDefinition = tableDefinition;
 
+        check("OCIAttrSet(OCI_ATTR_BUF_SIZE)", oci.OCIAttrSet(
+                dpHandle,
+                OCI.OCI_HTYPE_DIRPATH_CTX,
+                createPointer(bufferSize),
+                4,
+                OCI.OCI_ATTR_BUF_SIZE,
+                errHandle));
+
+        int numRows = Math.max(bufferSize / tableDefinition.getRowSize(), 16);
+        check("OCIAttrSet(OCI_ATTR_NUM_ROWS)", oci.OCIAttrSet(
+                dpHandle,
+                OCI.OCI_HTYPE_DIRPATH_CTX,
+                createPointer(numRows),
+                4,
+                OCI.OCI_ATTR_NUM_ROWS,
+                errHandle));
+
         if (tableDefinition.getSchemaName() != null) {
             Pointer schemaName = createPointer(tableDefinition.getSchemaName());
             check("OCIAttrSet(OCI_ATTR_NAME)", oci.OCIAttrSet(
@@ -269,7 +343,6 @@ public class OCIWrapper
         dpstrHandle = dpstrHandlePointer.getPointer(0);
 
         Pointer maxRowCountPointer = createPointer(0);
-
         check("OCIAttrGet(OCI_ATTR_NUM_ROWS)", oci.OCIAttrGet(
                 dpcaHandle,
                 OCI.OCI_HTYPE_DIRPATH_COLUMN_ARRAY,
@@ -278,42 +351,31 @@ public class OCIWrapper
                 OCI.OCI_ATTR_NUM_ROWS,
                 errHandle));
         maxRowCount = maxRowCountPointer.getInt(0);
+        logger.info(String.format("OCI : DirectPathColumnArray.numRows = %,d", maxRowCount));
     }
 
-    public void loadBuffer(RowBuffer rowBuffer) throws SQLException
-    {
-        Pointer pointer = new ByteBufferMemoryIO(Runtime.getSystemRuntime(), rowBuffer.getBuffer());
-        short[] sizes = rowBuffer.getSizes();
+    public int getMaxRowCount() {
+        return maxRowCount;
+    }
 
-        int i = 0;
-        int position = 0;
-        int rowCount = 0;
-        for (int row = 0; row < rowBuffer.getRowCount(); row++) {
-            for (short col = 0; col < tableDefinition.getColumnCount(); col++) {
-                short size = sizes[i++];
+    public void loadBuffer(ByteBuffer buffer, ByteBuffer sizes, int rowCount) throws SQLException
+    {
+        logger.info(String.format("Loading %,d rows", rowCount));
+        long startTime = System.currentTimeMillis();
 
-                check("OCIDirPathColArrayEntrySet", oci.OCIDirPathColArrayEntrySet(
-                        dpcaHandle,
-                        errHandle,
-                        rowCount,
-                        col,
-                        new BoundedMemoryIO(pointer, position, size),
-                        size,
-                        OCI.OCI_DIRPATH_COL_COMPLETE));
-
-                position += size;
-            }
+        check("OCIDirPathColArrayEntriesSet", bulkOci.embulk_output_oracle_OCIDirPathColArrayEntriesSet(
+                dpcaHandle,
+                errHandle,
+                (short)tableDefinition.getColumnCount(),
+                rowCount,
+                new ByteBufferMemoryIO(Runtime.getSystemRuntime(), buffer),
+                new ByteBufferMemoryIO(Runtime.getSystemRuntime(), sizes)));
 
-            rowCount++;
-            if (rowCount == maxRowCount) {
-                loadRows(rowCount);
-                rowCount = 0;
-            }
-        }
+        loadRows(rowCount);
 
-        if (rowCount > 0) {
-            loadRows(rowCount);
-        }
+        totalRows += rowCount;
+        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+        logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
     }
 
     private void loadRows(int rowCount) throws SQLException
@@ -334,6 +396,7 @@ public class OCIWrapper
                 check("OCIDirPathColArrayToStream", result);
             }
 
+            loadCount++;
             check("OCIDirPathLoadStream", oci.OCIDirPathLoadStream(
                     dpHandle,
                     dpstrHandle,
@@ -358,6 +421,9 @@ public class OCIWrapper
     public void commit() throws SQLException
     {
         committedOrRollbacked = true;
+        if (loadCount > 0) {
+            logger.info(String.format("OCI : OCIDirPathLoadStream : %,d rows x %,d times.", totalRows / loadCount, loadCount));
+        }
         logger.info("OCI : start to commit.");
 
         try {

data/src/main/java/org/embulk/output/oracle/oci/PrimitiveBulkOCI.java

@@ -0,0 +1,47 @@
+package org.embulk.output.oracle.oci;
+
+import jnr.ffi.Pointer;
+import jnr.ffi.types.u_int16_t;
+import jnr.ffi.types.u_int32_t;
+
+public class PrimitiveBulkOCI implements BulkOCI {
+
+    private final OCI oci;
+
+
+    public PrimitiveBulkOCI(OCI oci)
+    {
+        this.oci = oci;
+    }
+
+    public short embulk_output_oracle_OCIDirPathColArrayEntriesSet(
+            Pointer dpca,
+            Pointer errhp,
+            @u_int16_t short columnCount,
+            @u_int32_t int rowCount,
+            Pointer data,
+            Pointer sizes) {
+
+        int index = 0;
+        long offset = 0;
+        for (int row = 0; row < rowCount; row++) {
+            for (short column = 0; column < columnCount; column++) {
+                short size = sizes.getShort(index++ * 2);
+                short result = oci.OCIDirPathColArrayEntrySet(
+                        dpca,
+                        errhp,
+                        row,
+                        column,
+                        data.slice(offset),
+                        size,
+                        OCI.OCI_DIRPATH_COL_COMPLETE);
+                if (result != OCI.OCI_SUCCESS) {
+                    return result;
+                }
+                offset += size;
+            }
+        }
+
+        return OCI.OCI_SUCCESS;
+    }
+}

data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java

@@ -2,6 +2,7 @@ package org.embulk.output.oracle.oci;
 
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.nio.charset.Charset;
 import java.sql.SQLException;
 
@@ -12,48 +13,46 @@ import org.embulk.output.oracle.oci.TableDefinition;
 
 public class RowBuffer
 {
+    // this value was calculated by tests
+    private static final double OPTIMAL_LOAD_TIME = 10;
+    private static final int MIN_ROW_COUNT_TO_LOAD = 100;
+
+    private final OCIWrapper oci;
     private final TableDefinition table;
-    private final int rowCount;
+    private final int maxRowCount;
 
-    private int currentRow = 0;
+    private int rowCount = 0;
     private int currentColumn = 0;
 
-    private final short[] sizes;
+    private int rowCountToLoad;
+    private int totalLoadedRowCount = 0;
+    private int loadCount = 0;
+    private double totalLoadTime = 0;
+
+    private final ByteBuffer sizes;
+    private final ByteBuffer defaultSizes;
     private final ByteBuffer buffer;
     private final ByteBuffer defaultBuffer;
 
-    public RowBuffer(TableDefinition table, int rowCount)
+    public RowBuffer(OCIWrapper oci, TableDefinition table)
     {
+        this.oci = oci;
         this.table = table;
-        this.rowCount = rowCount;
-
-        int rowSize = 0;
-        for (int i = 0; i < table.getColumnCount(); i++) {
-            rowSize += table.getColumn(i).getDataSize();
-        }
+        maxRowCount = oci.getMaxRowCount();
+        rowCountToLoad = maxRowCount;
 
+        ByteOrder byteOrder = Runtime.getSystemRuntime().byteOrder();
         // should be direct because used by native library
-        buffer = ByteBuffer.allocateDirect(rowSize * rowCount).order(Runtime.getSystemRuntime().byteOrder());
+        buffer = ByteBuffer.allocateDirect(table.getRowSize() * maxRowCount).order(byteOrder);
         // position is not updated
-        defaultBuffer = buffer.duplicate();
-
-        sizes = new short[table.getColumnCount() * rowCount];
-    }
-
-    public ByteBuffer getBuffer() {
-        return defaultBuffer;
-    }
+        defaultBuffer = buffer.duplicate().order(byteOrder);
 
-    public short[] getSizes() {
-        return sizes;
+        sizes = ByteBuffer.allocateDirect(table.getColumnCount() * maxRowCount * 2).order(byteOrder);
+        defaultSizes = sizes.duplicate().order(byteOrder);
     }
 
-    public void addValue(int value)
+    public void addValue(int value) throws SQLException
     {
-        if (isFull()) {
-            throw new IllegalStateException();
-        }
-
         buffer.putInt(value);
 
         next((short)4);
@@ -61,10 +60,6 @@ public class RowBuffer
 
     public void addValue(String value) throws SQLException
     {
-        if (isFull()) {
-            throw new IllegalStateException();
-        }
-
         ColumnDefinition column = table.getColumn(currentColumn);
         Charset charset = column.getCharset().getJavaCharset();
 
@@ -88,14 +83,18 @@ public class RowBuffer
         addValue(value.toPlainString());
     }
 
-    private void next(short size)
+    private void next(short size) throws SQLException
     {
-        sizes[currentRow * table.getColumnCount() + currentColumn] = size;
+        sizes.putShort(size);
 
         currentColumn++;
         if (currentColumn == table.getColumnCount()) {
             currentColumn = 0;
-            currentRow++;
+            rowCount++;
+
+            if (rowCount >= rowCountToLoad) {
+                flush();
+            }
         }
     }
 
@@ -104,21 +103,26 @@ public class RowBuffer
         return currentColumn;
     }
 
-    public int getRowCount()
-    {
-        return currentRow;
-    }
-
-    public boolean isFull()
+    public void flush() throws SQLException
     {
-        return currentRow >= rowCount;
-    }
+        if (rowCount > 0) {
+            synchronized (oci) {
+                long time = System.currentTimeMillis();
+                oci.loadBuffer(defaultBuffer, defaultSizes, rowCount);
+                totalLoadTime += System.currentTimeMillis() - time;
+                totalLoadedRowCount += rowCount;
+                loadCount += 1;
+            }
+
+            rowCount = 0;
+            currentColumn = 0;
+            buffer.clear();
+            sizes.clear();
 
-    public void clear()
-    {
-        currentRow = 0;
-        currentColumn = 0;
-        buffer.clear();
+            if (loadCount >= 4) {
+                rowCountToLoad = Math.min(Math.max((int)(totalLoadedRowCount / totalLoadTime * OPTIMAL_LOAD_TIME), MIN_ROW_COUNT_TO_LOAD), maxRowCount);
+            }
+        }
     }
 
 }