embulk-output-oracle 0.4.0 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +160 -160
  3. data/build.gradle +6 -6
  4. data/classpath/{embulk-output-jdbc-0.4.0.jar → embulk-output-jdbc-0.4.1.jar} +0 -0
  5. data/classpath/{embulk-output-oracle-0.4.0.jar → embulk-output-oracle-0.4.1.jar} +0 -0
  6. data/lib/embulk/output/oracle.rb +3 -3
  7. data/src/main/cpp/common/dir-path-load.cpp +424 -424
  8. data/src/main/cpp/common/dir-path-load.h +36 -36
  9. data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
  10. data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
  11. data/src/main/cpp/linux/build.sh +21 -21
  12. data/src/main/cpp/win/build.bat +31 -31
  13. data/src/main/cpp/win/dllmain.cpp +25 -25
  14. data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
  15. data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
  16. data/src/main/java/org/embulk/output/OracleOutputPlugin.java +153 -153
  17. data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
  18. data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
  19. data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
  20. data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -165
  21. data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
  22. data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
  23. data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
  24. data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
  25. data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
  26. data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
  27. data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
  28. data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
  29. data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
  30. data/src/test/cpp/linux/build.sh +19 -19
  31. data/src/test/cpp/win/build.bat +28 -28
  32. data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
  33. data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
  34. data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
  35. data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
  36. data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
  37. data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +124 -120
  38. data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +105 -100
  39. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +38 -14
  40. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +540 -484
  41. data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
  42. data/src/test/resources/data/test1/test1.csv +3 -3
  43. data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -0
  44. data/src/test/resources/yml/test-insert-direct-empty.yml +25 -0
  45. data/src/test/resources/yml/{test-insert-oci-split.yml → test-insert-direct-oci-method-split.yml} +28 -28
  46. data/src/test/resources/yml/{test-insert-oci.yml → test-insert-direct-oci-method.yml} +28 -28
  47. data/src/test/resources/yml/test-insert-direct.yml +25 -26
  48. data/src/test/resources/yml/test-insert-empty.yml +27 -0
  49. data/src/test/resources/yml/test-insert.yml +27 -25
  50. data/src/test/resources/yml/test-replace-empty.yml +31 -0
  51. data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
  52. data/src/test/resources/yml/test-replace-long-name.yml +31 -31
  53. data/src/test/resources/yml/test-replace.yml +31 -31
  54. data/src/test/resources/yml/test-string-timestamp.yml +28 -28
  55. data/src/test/resources/yml/test-url.yml +24 -24
  56. metadata +10 -6
@@ -1,37 +1,37 @@
1
- package org.embulk.output.oracle;
2
-
3
- import java.sql.Timestamp;
4
- import java.text.FieldPosition;
5
- import java.text.SimpleDateFormat;
6
- import java.util.Date;
7
-
8
-
9
- public class TimestampFormat extends SimpleDateFormat
10
- {
11
-
12
- private final int scale;
13
-
14
- public TimestampFormat(String pattern, int scale)
15
- {
16
- super(pattern);
17
-
18
- this.scale = scale;
19
- }
20
-
21
- @Override
22
- public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos)
23
- {
24
- StringBuffer buffer = super.format(date, toAppendTo, pos);
25
- if (scale > 0) {
26
- buffer.append('.');
27
- String nanos = Integer.toString(((Timestamp)date).getNanos());
28
- int zeros = Math.min(scale, 9 - nanos.length());
29
- for (int i = 0; i < zeros; i++) {
30
- buffer.append('0');
31
- }
32
- buffer.append(nanos.substring(0, scale - zeros));
33
- }
34
- return buffer;
35
- }
36
-
37
- }
1
+ package org.embulk.output.oracle;
2
+
3
+ import java.sql.Timestamp;
4
+ import java.text.FieldPosition;
5
+ import java.text.SimpleDateFormat;
6
+ import java.util.Date;
7
+
8
+
9
+ public class TimestampFormat extends SimpleDateFormat
10
+ {
11
+
12
+ private final int scale;
13
+
14
+ public TimestampFormat(String pattern, int scale)
15
+ {
16
+ super(pattern);
17
+
18
+ this.scale = scale;
19
+ }
20
+
21
+ @Override
22
+ public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos)
23
+ {
24
+ StringBuffer buffer = super.format(date, toAppendTo, pos);
25
+ if (scale > 0) {
26
+ buffer.append('.');
27
+ String nanos = Integer.toString(((Timestamp)date).getNanos());
28
+ int zeros = Math.min(scale, 9 - nanos.length());
29
+ for (int i = 0; i < zeros; i++) {
30
+ buffer.append('0');
31
+ }
32
+ buffer.append(nanos.substring(0, scale - zeros));
33
+ }
34
+ return buffer;
35
+ }
36
+
37
+ }
@@ -1,26 +1,26 @@
1
- package org.embulk.output.oracle.oci;
2
-
3
- public class ColumnDefinition
4
- {
5
- public static int SQLT_CHR = 1;
6
- public static int SQLT_INT = 3;
7
-
8
- public final String columnName;
9
- public final int columnType;
10
- public final int columnSize;
11
- public final String columnDateFormat;
12
-
13
-
14
- public ColumnDefinition(String columnName, int columnType, int columnSize, String columnDateFormat)
15
- {
16
- this.columnName = columnName;
17
- this.columnType = columnType;
18
- this.columnSize = columnSize;
19
- this.columnDateFormat = columnDateFormat;
20
- }
21
-
22
- public ColumnDefinition(String columnName, int columnType, int columnSize)
23
- {
24
- this(columnName, columnType, columnSize, null);
25
- }
26
- }
1
+ package org.embulk.output.oracle.oci;
2
+
3
+ public class ColumnDefinition
4
+ {
5
+ public static int SQLT_CHR = 1;
6
+ public static int SQLT_INT = 3;
7
+
8
+ public final String columnName;
9
+ public final int columnType;
10
+ public final int columnSize;
11
+ public final String columnDateFormat;
12
+
13
+
14
+ public ColumnDefinition(String columnName, int columnType, int columnSize, String columnDateFormat)
15
+ {
16
+ this.columnName = columnName;
17
+ this.columnType = columnType;
18
+ this.columnSize = columnSize;
19
+ this.columnDateFormat = columnDateFormat;
20
+ }
21
+
22
+ public ColumnDefinition(String columnName, int columnType, int columnSize)
23
+ {
24
+ this(columnName, columnType, columnSize, null);
25
+ }
26
+ }
@@ -1,139 +1,139 @@
1
- package org.embulk.output.oracle.oci;
2
-
3
- import java.io.File;
4
- import java.io.IOException;
5
- import java.net.MalformedURLException;
6
- import java.net.URISyntaxException;
7
- import java.net.URL;
8
-
9
- import org.embulk.spi.Exec;
10
- import org.slf4j.Logger;
11
-
12
- import com.google.common.io.Files;
13
-
14
-
15
- public class OCI
16
- {
17
- private static final Logger logger = Exec.getLogger(OCI.class);
18
-
19
- private static final String PLUGIN_NAME = "embulk-output-oracle";
20
-
21
- static {
22
- try {
23
- loadLibrary();
24
- } catch (URISyntaxException | IOException e) {
25
- throw new RuntimeException(e);
26
- }
27
- }
28
-
29
- public native byte[] createContext();
30
-
31
- public native byte[] getLasetMessage(byte[] context);
32
-
33
- public native boolean open(byte[] context, String dbName, String userName, String password);
34
-
35
- public native boolean prepareLoad(byte[] context, TableDefinition tableDefinition);
36
-
37
- public native boolean loadBuffer(byte[] context, byte[] buffer, int rowCount);
38
-
39
- public native boolean commit(byte[] context);
40
-
41
- public native boolean rollback(byte[] context);
42
-
43
- public native void close(byte[] context);
44
-
45
- private static void loadLibrary() throws URISyntaxException, IOException
46
- {
47
- loadLibrary(getPluginRoot());
48
- }
49
-
50
- private static File getPluginRoot() throws MalformedURLException, URISyntaxException
51
- {
52
- URL url = OCI.class.getResource("/" + OCI.class.getName().replace('.', '/') + ".class");
53
- if (url.toString().startsWith("jar:")) {
54
- url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
55
- }
56
-
57
- File folder = new File(url.toURI()).getParentFile();
58
- for (;; folder = folder.getParentFile()) {
59
- if (folder == null) {
60
- String message = String.format("OCI : %s folder not found.", PLUGIN_NAME);
61
- throw new RuntimeException(message);
62
- }
63
-
64
- if (folder.getName().startsWith(PLUGIN_NAME)) {
65
- return folder;
66
- }
67
- }
68
- }
69
-
70
- private static void loadLibrary(File folder) throws IOException
71
- {
72
- File lib = new File(new File(folder, "lib"), "embulk");
73
-
74
- String osName = System.getProperty("os.name");
75
- String osArch = System.getProperty("os.arch");
76
-
77
- String libraryName = System.mapLibraryName(PLUGIN_NAME);
78
- File libFolder = null;
79
-
80
- if (osName.startsWith("Windows")) {
81
- if (osArch.endsWith("64")) {
82
- libFolder = new File(lib, "win_x64");
83
- } else if (osArch.equals("x86")) {
84
- libFolder = new File(lib, "win_x86");
85
- }
86
- } else if (osName.equals("Linux")) {
87
- if (osArch.endsWith("64")) {
88
- libFolder = new File(lib, "linux_x64");
89
- } else if (osArch.equals("x86")) {
90
- libFolder = new File(lib, "linux_x86");
91
- }
92
- }
93
-
94
- if (libFolder == null) {
95
- logger.error(String.format("OCI : library \"%s\" for %s %s doesn't exist in lib folder.", libraryName, osName, osArch));
96
- return;
97
- }
98
-
99
- File libFile = new File(libFolder, libraryName);
100
- if (!libFile.exists()) {
101
- logger.error(String.format("OCI : library \"%s\" doesn't exist.", libFile.getAbsolutePath()));
102
- return;
103
- }
104
-
105
- logger.info(String.format("OCI : load library \"%s\".", libFile.getAbsolutePath()));
106
-
107
- File tempFolder = new File(lib, "temp");
108
- tempFolder.mkdirs();
109
-
110
- long currentTime = System.currentTimeMillis();
111
- File[] files = tempFolder.listFiles();
112
- if (files != null) {
113
- for (File file : files) {
114
- // delete old temporary files.
115
- // if another plugin is using a file, it cannot be deleted.
116
- // don't delete a recent file because another plugin may intend to use it.
117
- if (file.isFile() && file.getName().startsWith(PLUGIN_NAME) && file.lastModified() < currentTime - 60000) {
118
- file.delete();
119
- }
120
- }
121
- }
122
-
123
- String extension = libraryName.replaceAll("^[^\\.]*", "");
124
- for (int i = 0; i < 10; i++) {
125
- File tempLibFile = new File(tempFolder, PLUGIN_NAME + "-" + currentTime + "-" + i + extension);
126
- if (tempLibFile.createNewFile()) {
127
- // copy and load the library because different plugins cannot load the same library.
128
- logger.info(String.format("OCI : create temporary library \"%s\".", tempLibFile.getAbsolutePath()));
129
- Files.copy(libFile, tempLibFile);
130
- System.load(tempLibFile.getAbsolutePath());
131
- tempLibFile.deleteOnExit();
132
- // but may not be deleted because loaded as a library.
133
- return;
134
- }
135
- }
136
-
137
- logger.error("OCI : cannot create temporary library.");
138
- }
139
- }
1
+ package org.embulk.output.oracle.oci;
2
+
3
+ import java.io.File;
4
+ import java.io.IOException;
5
+ import java.net.MalformedURLException;
6
+ import java.net.URISyntaxException;
7
+ import java.net.URL;
8
+
9
+ import org.embulk.spi.Exec;
10
+ import org.slf4j.Logger;
11
+
12
+ import com.google.common.io.Files;
13
+
14
+
15
+ public class OCI
16
+ {
17
+ private static final Logger logger = Exec.getLogger(OCI.class);
18
+
19
+ private static final String PLUGIN_NAME = "embulk-output-oracle";
20
+
21
+ static {
22
+ try {
23
+ loadLibrary();
24
+ } catch (URISyntaxException | IOException e) {
25
+ throw new RuntimeException(e);
26
+ }
27
+ }
28
+
29
+ public native byte[] createContext();
30
+
31
+ public native byte[] getLasetMessage(byte[] context);
32
+
33
+ public native boolean open(byte[] context, String dbName, String userName, String password);
34
+
35
+ public native boolean prepareLoad(byte[] context, TableDefinition tableDefinition);
36
+
37
+ public native boolean loadBuffer(byte[] context, byte[] buffer, int rowCount);
38
+
39
+ public native boolean commit(byte[] context);
40
+
41
+ public native boolean rollback(byte[] context);
42
+
43
+ public native void close(byte[] context);
44
+
45
+ private static void loadLibrary() throws URISyntaxException, IOException
46
+ {
47
+ loadLibrary(getPluginRoot());
48
+ }
49
+
50
+ private static File getPluginRoot() throws MalformedURLException, URISyntaxException
51
+ {
52
+ URL url = OCI.class.getResource("/" + OCI.class.getName().replace('.', '/') + ".class");
53
+ if (url.toString().startsWith("jar:")) {
54
+ url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
55
+ }
56
+
57
+ File folder = new File(url.toURI()).getParentFile();
58
+ for (;; folder = folder.getParentFile()) {
59
+ if (folder == null) {
60
+ String message = String.format("OCI : %s folder not found.", PLUGIN_NAME);
61
+ throw new RuntimeException(message);
62
+ }
63
+
64
+ if (folder.getName().startsWith(PLUGIN_NAME)) {
65
+ return folder;
66
+ }
67
+ }
68
+ }
69
+
70
+ private static void loadLibrary(File folder) throws IOException
71
+ {
72
+ File lib = new File(new File(folder, "lib"), "embulk");
73
+
74
+ String osName = System.getProperty("os.name");
75
+ String osArch = System.getProperty("os.arch");
76
+
77
+ String libraryName = System.mapLibraryName(PLUGIN_NAME);
78
+ File libFolder = null;
79
+
80
+ if (osName.startsWith("Windows")) {
81
+ if (osArch.endsWith("64")) {
82
+ libFolder = new File(lib, "win_x64");
83
+ } else if (osArch.equals("x86")) {
84
+ libFolder = new File(lib, "win_x86");
85
+ }
86
+ } else if (osName.equals("Linux")) {
87
+ if (osArch.endsWith("64")) {
88
+ libFolder = new File(lib, "linux_x64");
89
+ } else if (osArch.equals("x86")) {
90
+ libFolder = new File(lib, "linux_x86");
91
+ }
92
+ }
93
+
94
+ if (libFolder == null) {
95
+ logger.error(String.format("OCI : library \"%s\" for %s %s doesn't exist in lib folder.", libraryName, osName, osArch));
96
+ return;
97
+ }
98
+
99
+ File libFile = new File(libFolder, libraryName);
100
+ if (!libFile.exists()) {
101
+ logger.error(String.format("OCI : library \"%s\" doesn't exist.", libFile.getAbsolutePath()));
102
+ return;
103
+ }
104
+
105
+ logger.info(String.format("OCI : load library \"%s\".", libFile.getAbsolutePath()));
106
+
107
+ File tempFolder = new File(lib, "temp");
108
+ tempFolder.mkdirs();
109
+
110
+ long currentTime = System.currentTimeMillis();
111
+ File[] files = tempFolder.listFiles();
112
+ if (files != null) {
113
+ for (File file : files) {
114
+ // delete old temporary files.
115
+ // if another plugin is using a file, it cannot be deleted.
116
+ // don't delete a recent file because another plugin may intend to use it.
117
+ if (file.isFile() && file.getName().startsWith(PLUGIN_NAME) && file.lastModified() < currentTime - 60000) {
118
+ file.delete();
119
+ }
120
+ }
121
+ }
122
+
123
+ String extension = libraryName.replaceAll("^[^\\.]*", "");
124
+ for (int i = 0; i < 10; i++) {
125
+ File tempLibFile = new File(tempFolder, PLUGIN_NAME + "-" + currentTime + "-" + i + extension);
126
+ if (tempLibFile.createNewFile()) {
127
+ // copy and load the library because different plugins cannot load the same library.
128
+ logger.info(String.format("OCI : create temporary library \"%s\".", tempLibFile.getAbsolutePath()));
129
+ Files.copy(libFile, tempLibFile);
130
+ System.load(tempLibFile.getAbsolutePath());
131
+ tempLibFile.deleteOnExit();
132
+ // but may not be deleted because loaded as a library.
133
+ return;
134
+ }
135
+ }
136
+
137
+ logger.error("OCI : cannot create temporary library.");
138
+ }
139
+ }
@@ -1,64 +1,64 @@
1
- package org.embulk.output.oracle.oci;
2
-
3
- import java.sql.SQLException;
4
- import java.util.HashMap;
5
- import java.util.Map;
6
-
7
- import org.embulk.spi.Exec;
8
- import org.slf4j.Logger;
9
-
10
- public class OCIManager
11
- {
12
- private static class OCIWrapperAndCounter {
13
- public OCIWrapper oci;
14
- public int counter;
15
- }
16
-
17
-
18
- private final Logger logger = Exec.getLogger(getClass());
19
-
20
- private Map<Object, OCIWrapperAndCounter> ociAndCounters = new HashMap<Object, OCIWrapperAndCounter>();
21
-
22
-
23
- public OCIWrapper open(Object key, String dbName, String userName, String password, TableDefinition tableDefinition) throws SQLException
24
- {
25
- synchronized(ociAndCounters) {
26
- OCIWrapperAndCounter ociAndCounter;
27
- if (ociAndCounters.containsKey(key)) {
28
- ociAndCounter = ociAndCounters.get(key);
29
- } else {
30
- logger.info(String.format("OCI : open for %s.", key));
31
- ociAndCounter = new OCIWrapperAndCounter();
32
- ociAndCounter.oci = new OCIWrapper();
33
- ociAndCounter.oci.open(dbName, userName, password);
34
- ociAndCounter.oci.prepareLoad(tableDefinition);
35
- ociAndCounters.put(key, ociAndCounter);
36
- }
37
- ociAndCounter.counter++;
38
- return ociAndCounter.oci;
39
- }
40
- }
41
-
42
- public OCIWrapper get(Object key)
43
- {
44
- synchronized(ociAndCounters) {
45
- return ociAndCounters.get(key).oci;
46
- }
47
- }
48
-
49
- public void close(Object key) throws SQLException
50
- {
51
- synchronized(ociAndCounters) {
52
- OCIWrapperAndCounter ociAndCounter = ociAndCounters.get(key);
53
- if (ociAndCounter != null) {
54
- ociAndCounter.counter--;
55
- if (ociAndCounter.counter == 0) {
56
- logger.info(String.format("OCI : close for %s.", key));
57
- ociAndCounter.oci.close();
58
- ociAndCounters.remove(key);
59
- }
60
- }
61
- }
62
- }
63
-
64
- }
1
+ package org.embulk.output.oracle.oci;
2
+
3
+ import java.sql.SQLException;
4
+ import java.util.HashMap;
5
+ import java.util.Map;
6
+
7
+ import org.embulk.spi.Exec;
8
+ import org.slf4j.Logger;
9
+
10
+ public class OCIManager
11
+ {
12
+ private static class OCIWrapperAndCounter {
13
+ public OCIWrapper oci;
14
+ public int counter;
15
+ }
16
+
17
+
18
+ private final Logger logger = Exec.getLogger(getClass());
19
+
20
+ private Map<Object, OCIWrapperAndCounter> ociAndCounters = new HashMap<Object, OCIWrapperAndCounter>();
21
+
22
+
23
+ public OCIWrapper open(Object key, String dbName, String userName, String password, TableDefinition tableDefinition) throws SQLException
24
+ {
25
+ synchronized(ociAndCounters) {
26
+ OCIWrapperAndCounter ociAndCounter;
27
+ if (ociAndCounters.containsKey(key)) {
28
+ ociAndCounter = ociAndCounters.get(key);
29
+ } else {
30
+ logger.info(String.format("OCI : open for %s.", key));
31
+ ociAndCounter = new OCIWrapperAndCounter();
32
+ ociAndCounter.oci = new OCIWrapper();
33
+ ociAndCounter.oci.open(dbName, userName, password);
34
+ ociAndCounter.oci.prepareLoad(tableDefinition);
35
+ ociAndCounters.put(key, ociAndCounter);
36
+ }
37
+ ociAndCounter.counter++;
38
+ return ociAndCounter.oci;
39
+ }
40
+ }
41
+
42
+ public OCIWrapper get(Object key)
43
+ {
44
+ synchronized(ociAndCounters) {
45
+ return ociAndCounters.get(key).oci;
46
+ }
47
+ }
48
+
49
+ public void close(Object key) throws SQLException
50
+ {
51
+ synchronized(ociAndCounters) {
52
+ OCIWrapperAndCounter ociAndCounter = ociAndCounters.get(key);
53
+ if (ociAndCounter != null) {
54
+ ociAndCounter.counter--;
55
+ if (ociAndCounter.counter == 0) {
56
+ logger.info(String.format("OCI : close for %s.", key));
57
+ ociAndCounter.oci.close();
58
+ ociAndCounters.remove(key);
59
+ }
60
+ }
61
+ }
62
+ }
63
+
64
+ }