embulk-output-oracle 0.4.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f19ed247b3f0e55ae2a970607323439e04e5c83e
-  data.tar.gz: 13955cf610304e6e53f1230cc99160b8d1c89439
+  metadata.gz: 7f22f18309d604502073834ef9bcac39ba1c7ed2
+  data.tar.gz: f4e47f7c177a3f8d99c0a6ebbc8eb7e3d5882417
 SHA512:
-  metadata.gz: 57685288f87ed884e5f35358ed03f1c46f4cc1d030607842c8243fba31ed3913233f0c4d809b6d3793007daee8e7785b5f787c42ae40cf5c17e9bbd7373c669e
-  data.tar.gz: 9ca1b2156ce837ad874f134b8fce9c2e809792498763655e02627236961e9363fd8ebfae5fa55a2f993ef3c24d82151e92351597687268269ebe964958258946
+  metadata.gz: f8791a70edae2c6af65682b2e96fb6186563509f7eff0d1c952d6e19b641e663e1f64624f4f57d441d4dc3d8efd8838d0006bc75c537b386c51fae3252e2ebdc
+  data.tar.gz: 713c24da25c5b10dcb47589ec5aa690ea6f64643961f3bf3273eed14bed835c23dec84e8c352bc3fcce04555c4ec00432334b850143fd53ec4f4861708760ecb

data/README.md

@@ -61,10 +61,6 @@ It requires Oracle JDBC driver too, but the version 12 driver doesn't work (the
 It requires both Oracle JDBC driver and Oracle Instant Client (version 12.1.0.2.0).
 You must set the library loading path to the OCI library.
 
-If you use "oci", platform dependent library written in cpp is required.
-Windows(x64) library and Linux(x64) are bundled, but others are not bundled.
-You should build by yourself and set the library loading path to it.
-
 ### Example
 
 ```yaml
@@ -107,52 +103,6 @@ out:
 $ ./gradlew gem
 ```
 
-#### Build environment for native library
-
-For Windows (x64)
-
-(1) Install JDK.
-
-(2) Install Microsoft Visual Studio (only 2010 is tested).
-
-(3) Install Oracle Instant Client SDK 11.1.0.6.0 for Microsoft Windows (x64).
-
-(4) Set environment variables.
-
-* JAVA_HOME
-* OCI\_SDK_PATH ("sdk" directory of Oracle Instant Client)
-
-(5) Open src/main/cpp/win/embulk-output-oracle.sln by Visual Studio and build.
-
-For Windows command line, the following are needed in addition to (1) - (4).
-
-(6) Set environment variables.
-
-* MSVC_PATH (ex. C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC)
-* MSSDK_PATH (ex. C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A)
-
-(7) Execute src/main/cpp/win/build.bat .
-
-
-For Linux (x64) (only Ubuntu Server 14.04 is tested)
-
-(1) Install JDK.
-
-(2) Install gcc and g++ .
-
-(3) Install Oracle Instant Client Basic and SDK 11.1.0.6.0 for Linux (x64).
-
-(4) Create symbolic links of OCI libraries.
-
-    ln -s libocci.so.11.1 libocci.so
-    ln -s libclntsh.so.11.1 libclntsh.so
-
-(5) Set environment variables.
-
-* JAVA_HOME
-* OCI_PATH (the directory of Oracle Instant Client Basic and the parent of the "sdk" directory)
-
-(6) Execute src/main/cpp/linux/build.sh .
 
 ***
 <img src="https://www.yourkit.com/images/yklogo.png" alt="YourKit"/> is used to improve performance of embulk-output-oracle.

data/build.gradle

@@ -3,4 +3,5 @@
 dependencies {
     compile project(':embulk-output-jdbc')
     testCompile 'org.embulk:embulk-standards:0.8.0'
+    testCompile files('../embulk-output-jdbc/build/classes/test/')
 }

data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java

@@ -8,18 +8,20 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Calendar;
+import java.util.List;
+
 import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.oracle.oci.ColumnDefinition;
+import org.embulk.output.oracle.oci.OCI;
 import org.embulk.output.oracle.oci.OCIManager;
 import org.embulk.output.oracle.oci.OCIWrapper;
 import org.embulk.output.oracle.oci.RowBuffer;
 import org.embulk.output.oracle.oci.TableDefinition;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.Exec;
+import org.embulk.spi.time.Timestamp;
 import org.slf4j.Logger;
 
 public class DirectBatchInsert implements BatchInsert
@@ -87,9 +89,9 @@ public class DirectBatchInsert implements BatchInsert
                 case Types.LONGVARCHAR:
                 case Types.CLOB:
                     columns.add(new ColumnDefinition(insertColumn.getName(),
-                            ColumnDefinition.SQLT_CHR,
+                            OCI.SQLT_CHR,
                             insertColumn.getDataLength(),
-                            charset.getId()));
+                            charset));
                     break;
 
                 case Types.NCHAR:
@@ -97,9 +99,9 @@ public class DirectBatchInsert implements BatchInsert
                 case Types.LONGNVARCHAR:
                 case Types.NCLOB:
                     columns.add(new ColumnDefinition(insertColumn.getName(),
-                            ColumnDefinition.SQLT_CHR,
+                            OCI.SQLT_CHR,
                             insertColumn.getDataLength(),
-                            nationalCharset.getId()));
+                            nationalCharset));
                     break;
 
                 case Types.DECIMAL:
@@ -110,9 +112,9 @@ public class DirectBatchInsert implements BatchInsert
                         size += 1;
                     }
                     columns.add(new ColumnDefinition(insertColumn.getName(),
-                            ColumnDefinition.SQLT_CHR,
+                            OCI.SQLT_CHR,
                             size,
-                            charset.getId()));
+                            charset));
                     break;
 
                 case Types.DATE:
@@ -130,9 +132,9 @@ public class DirectBatchInsert implements BatchInsert
                     }
                     formats[i] = javaFormat;
                     columns.add(new ColumnDefinition(insertColumn.getName(),
-                            ColumnDefinition.SQLT_CHR,
+                            OCI.SQLT_CHR,
                             javaFormat.format(dummy).length(),
-                            charset.getId(),
+                            charset,
                             oracleFormat));
                     break;
 
@@ -144,14 +146,14 @@ public class DirectBatchInsert implements BatchInsert
 
         rowSize = 0;
         for (ColumnDefinition column : columns) {
-            rowSize += column.columnSize;
+            rowSize += column.getDataSize();
         }
 
         TableDefinition tableDefinition = new TableDefinition("\"" + loadTable + "\"", columns);
         ociKey = Arrays.asList(database, user, loadTable);
         ociManager.open(ociKey, database, user, password, tableDefinition);
 
-        buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
+        buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8));
     }
 
     @Override
@@ -186,7 +188,7 @@ public class DirectBatchInsert implements BatchInsert
 
                 OCIWrapper oci = ociManager.get(ociKey);
                 synchronized (oci) {
-                    oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
+                    oci.loadBuffer(buffer);
                 }
 
                 totalRows += buffer.getRowCount();
@@ -269,7 +271,7 @@ public class DirectBatchInsert implements BatchInsert
     @Override
     public void setNString(String v) throws IOException, SQLException
     {
-        buffer.addValue(v, nationalCharset.getJavaCharset());
+        buffer.addValue(v);
     }
 
     @Override

data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java

@@ -137,7 +137,7 @@ public class OracleOutputConnection
     public synchronized OracleCharset getOracleCharset() throws SQLException
     {
         if (charset == null) {
-        	charset = getOracleCharset("NLS_CHARACTERSET", "UTF8");
+            charset = getOracleCharset("NLS_CHARACTERSET", "UTF8");
         }
         return charset;
     }
@@ -145,7 +145,7 @@ public class OracleOutputConnection
     public synchronized OracleCharset getOracleNationalCharset() throws SQLException
     {
         if (nationalCharset == null) {
-        	nationalCharset = getOracleCharset("NLS_NCHAR_CHARACTERSET", "AL16UTF16");
+            nationalCharset = getOracleCharset("NLS_NCHAR_CHARACTERSET", "AL16UTF16");
         }
         return nationalCharset;
     }
@@ -154,7 +154,7 @@ public class OracleOutputConnection
     {
         String charsetName = defaultCharsetName;
         try (PreparedStatement statement = connection.prepareStatement("SELECT VALUE FROM NLS_DATABASE_PARAMETERS WHERE PARAMETER=?")) {
-        	statement.setString(1, parameterName);
+            statement.setString(1, parameterName);
             try (ResultSet resultSet = statement.executeQuery()) {
                 if (resultSet.next()) {
                     String nlsCharacterSet = resultSet.getString(1);
@@ -180,14 +180,14 @@ public class OracleOutputConnection
     }
 
 
-    private static final String[] STANDARD_SIZE_TYPE_NAMES = {
+    private static final String[] SIZE_TYPE_NAMES = {
         "VARCHAR2", "NVARCHAR2",
     };
 
     @Override
     protected ColumnDeclareType getColumnDeclareType(String convertedTypeName, JdbcColumn col)
     {
-        if (Arrays.asList(STANDARD_SIZE_TYPE_NAMES).contains(convertedTypeName)) {
+        if (Arrays.asList(SIZE_TYPE_NAMES).contains(convertedTypeName)) {
             return ColumnDeclareType.SIZE;
         }
         return super.getColumnDeclareType(convertedTypeName, col);

data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java

@@ -1,28 +1,52 @@
 package org.embulk.output.oracle.oci;
 
+import org.embulk.output.oracle.OracleCharset;
+
 public class ColumnDefinition
 {
-    public static int SQLT_CHR = 1;
-    public static int SQLT_INT = 3;
-
-    public final String columnName;
-    public final int columnType;
-    public final int columnSize;
-    public final short charsetId;
-    public final String columnDateFormat;
+    private final String columnName;
+    private final int dataType;
+    private final int dataSize;
+    private final OracleCharset charset;
+    private final String dateFormat;
 
 
-    public ColumnDefinition(String columnName, int columnType, int columnSize, short charsetId, String columnDateFormat)
+    public ColumnDefinition(String columnName, int dataType, int dataSize, OracleCharset charset, String dateFormat)
     {
         this.columnName = columnName;
-        this.columnType = columnType;
-        this.columnSize = columnSize;
-        this.charsetId = charsetId;
-        this.columnDateFormat = columnDateFormat;
+        this.dataType = dataType;
+        this.dataSize = dataSize;
+        this.charset = charset;
+        this.dateFormat = dateFormat;
+    }
+
+    public ColumnDefinition(String columnName, int columnType, int columnSize, OracleCharset charset)
+    {
+        this(columnName, columnType, columnSize, charset, null);
+    }
+
+    public String getColumnName()
+    {
+        return columnName;
+    }
+
+    public int getDataType()
+    {
+        return dataType;
+    }
+
+    public int getDataSize()
+    {
+        return dataSize;
+    }
+
+    public OracleCharset getCharset()
+    {
+        return charset;
     }
 
-    public ColumnDefinition(String columnName, int columnType, int columnSize, short charsetId)
+    public String getDateFormat()
     {
-        this(columnName, columnType, columnSize, charsetId, null);
+        return dateFormat;
     }
 }

data/src/main/java/org/embulk/output/oracle/oci/OCI.java

@@ -1,139 +1,131 @@
 package org.embulk.output.oracle.oci;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
+import jnr.ffi.Pointer;
+import jnr.ffi.types.size_t;
+import jnr.ffi.types.u_int16_t;
+import jnr.ffi.types.u_int32_t;
+import jnr.ffi.types.u_int8_t;
 
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
 
-import com.google.common.io.Files;
-
-
-public class OCI
+public interface OCI
 {
-    private static final Logger logger = Exec.getLogger(OCI.class);
-
-    private static final String PLUGIN_NAME = "embulk-output-oracle";
-
-    static {
-        try {
-            loadLibrary();
-        } catch (URISyntaxException | IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public native byte[] createContext();
-
-    public native byte[] getLasetMessage(byte[] context);
-
-    public native boolean open(byte[] context, String dbName, String userName, String password);
-
-    public native boolean prepareLoad(byte[] context, TableDefinition tableDefinition);
-
-    public native boolean loadBuffer(byte[] context, byte[] buffer, int rowCount);
-
-    public native boolean commit(byte[] context);
-
-    public native boolean rollback(byte[] context);
-
-    public native void close(byte[] context);
-
-    private static void loadLibrary() throws URISyntaxException, IOException
-    {
-        loadLibrary(getPluginRoot());
-    }
-
-    private static File getPluginRoot() throws MalformedURLException, URISyntaxException
-    {
-        URL url = OCI.class.getResource("/" + OCI.class.getName().replace('.', '/') + ".class");
-        if (url.toString().startsWith("jar:")) {
-            url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
-        }
-
-        File folder = new File(url.toURI()).getParentFile();
-        for (;; folder = folder.getParentFile()) {
-            if (folder == null) {
-                String message = String.format("OCI : %s folder not found.", PLUGIN_NAME);
-                throw new RuntimeException(message);
-            }
-
-            if (folder.getName().startsWith(PLUGIN_NAME)) {
-                return folder;
-            }
-        }
-    }
-
-    private static void loadLibrary(File folder) throws IOException
-    {
-        File lib = new File(new File(folder, "lib"), "embulk");
-
-        String osName = System.getProperty("os.name");
-        String osArch = System.getProperty("os.arch");
-
-        String libraryName = System.mapLibraryName(PLUGIN_NAME);
-        File libFolder = null;
-
-        if (osName.startsWith("Windows")) {
-            if (osArch.endsWith("64")) {
-                libFolder = new File(lib, "win_x64");
-            } else if (osArch.equals("x86")) {
-                libFolder = new File(lib, "win_x86");
-            }
-        } else if (osName.equals("Linux")) {
-            if (osArch.endsWith("64")) {
-                libFolder = new File(lib, "linux_x64");
-            } else if (osArch.equals("x86")) {
-                libFolder = new File(lib, "linux_x86");
-            }
-        }
-
-        if (libFolder == null) {
-            logger.error(String.format("OCI : library \"%s\" for %s %s doesn't exist in lib folder.", libraryName, osName, osArch));
-            return;
-        }
-
-        File libFile = new File(libFolder, libraryName);
-        if (!libFile.exists()) {
-            logger.error(String.format("OCI : library \"%s\" doesn't exist.", libFile.getAbsolutePath()));
-            return;
-        }
-
-        logger.info(String.format("OCI : load library \"%s\".", libFile.getAbsolutePath()));
-
-        File tempFolder = new File(lib, "temp");
-        tempFolder.mkdirs();
-
-        long currentTime = System.currentTimeMillis();
-        File[] files = tempFolder.listFiles();
-        if (files != null) {
-            for (File file : files) {
-                // delete old temporary files.
-                // if another plugin is using a file, it cannot be deleted.
-                // don't delete a recent file because another plugin may intend to use it.
-                if (file.isFile() && file.getName().startsWith(PLUGIN_NAME) && file.lastModified() < currentTime - 60000) {
-                    file.delete();
-                }
-            }
-        }
-
-        String extension = libraryName.replaceAll("^[^\\.]*", "");
-        for (int i = 0; i < 10; i++) {
-            File tempLibFile = new File(tempFolder, PLUGIN_NAME + "-" + currentTime + "-" + i + extension);
-            if (tempLibFile.createNewFile()) {
-                // copy and load the library because different plugins cannot load the same library.
-                logger.info(String.format("OCI : create temporary library \"%s\".", tempLibFile.getAbsolutePath()));
-                Files.copy(libFile, tempLibFile);
-                System.load(tempLibFile.getAbsolutePath());
-                tempLibFile.deleteOnExit();
-                // but may not be deleted because loaded as a library.
-                return;
-            }
-        }
-
-        logger.error("OCI : cannot create temporary library.");
-    }
+    static short OCI_SUCCESS = 0;
+    static short OCI_ERROR = -1;
+    static short OCI_INVALID_HANDLE = -2;
+    static short OCI_NO_DATA = 100;
+    static short OCI_CONTINUE = -24200;
+
+
+    static int OCI_THREADED = 1;
+    static int OCI_OBJECT = 2;
+
+    static int OCI_HTYPE_ENV = 1;
+    static int OCI_HTYPE_ERROR = 2;
+    static int OCI_HTYPE_SVCCTX = 3;
+    static int OCI_HTYPE_DIRPATH_CTX = 14;
+    static int OCI_HTYPE_DIRPATH_COLUMN_ARRAY = 15;
+    static int OCI_HTYPE_DIRPATH_STREAM = 16;
+
+    static int OCI_ATTR_DATA_SIZE = 1;
+    static int OCI_ATTR_DATA_TYPE = 2;
+    static int OCI_ATTR_NAME = 4;
+    static int OCI_ATTR_ROW_COUNT = 9;
+    static int OCI_ATTR_CHARSET_ID = 31;
+    static int OCI_ATTR_DATEFORMAT = 75;
+    static int OCI_ATTR_NUM_ROWS = 81;
+    static int OCI_ATTR_NUM_COLS = 102;
+    static int OCI_ATTR_LIST_COLUMNS = 103;
+
+    static int OCI_DTYPE_PARAM = 53;
+
+    static byte OCI_DIRPATH_COL_COMPLETE = 0;
+
+    static int SQLT_CHR = 1;
+    static int SQLT_INT = 3;
+
+    short OCIErrorGet(Pointer hndlp,
+            @u_int32_t int  recordno,
+            String sqlstate,
+            Pointer errcodep,
+            Pointer bufp,
+            @u_int32_t int bufsiz,
+            @u_int32_t int type);
+
+    short OCIEnvCreate(Pointer envp,
+            @u_int32_t int mode,
+            Pointer ctxp,
+            Pointer malocfp,
+            Pointer ralocfp,
+            Pointer mfreefp,
+            @size_t long xtramemSz,
+            Pointer usrmempp);
+
+    short OCIHandleAlloc(Pointer parenth,
+            Pointer hndlpp,
+            @u_int32_t int type,
+            @size_t long xtramemSz,
+            Pointer usrmempp);
+
+    short OCIHandleFree(Pointer hndlpp,
+            @u_int32_t int type);
+
+    short OCILogon(Pointer envhp,
+            Pointer errhp,
+            Pointer svchp,
+            String username,
+            @u_int32_t int usernameLen,
+            String password,
+            @u_int32_t int passwordLen,
+            String dbname,
+            @u_int32_t int dbnameLen);
+
+    short OCILogoff(Pointer svchp,
+            Pointer errhp);
+
+    short OCIAttrSet(Pointer trgthndlp,
+            @u_int32_t int trghndltyp,
+            Pointer attributep,
+            @u_int32_t int size,
+            @u_int32_t int attrtype,
+            Pointer errhp);
+
+    short OCIAttrGet(Pointer trgthndlp,
+            @u_int32_t int trghndltyp,
+            Pointer attributep,
+            Pointer sizep,
+            @u_int32_t int attrtype,
+            Pointer errhp);
+
+    short OCIParamGet(Pointer hndlp,
+            @u_int32_t int htype,
+            Pointer errhp,
+            Pointer parmdpp,
+            @u_int32_t int pos);
+
+    short OCIDescriptorFree(Pointer descp, @u_int32_t int type);
+
+    short OCIDirPathPrepare(Pointer dpctx, Pointer svchp, Pointer errhp);
+
+    short OCIDirPathColArrayEntrySet(Pointer dpca,
+            Pointer errhp,
+            @u_int32_t int rownum,
+            @u_int16_t short colIdx,
+            Pointer cvalp,
+            @u_int32_t int size,
+            @u_int8_t byte cflg);
+
+    short OCIDirPathStreamReset(Pointer dpstr, Pointer errhp);
+
+    short OCIDirPathColArrayToStream(Pointer dpca,
+            Pointer dpctx,
+            Pointer dpstr,
+            Pointer errhp,
+            @u_int32_t int rowcnt,
+            @u_int32_t int rowoff);
+
+    short OCIDirPathLoadStream(Pointer dpctx, Pointer dpstr, Pointer errhp);
+
+    short OCIDirPathFinish(Pointer dpctx, Pointer errhp);
+
+    short OCIDirPathAbort(Pointer dpctx, Pointer errhp);
 }