embulk-output-oracle 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +160 -160
- data/build.gradle +6 -6
- data/classpath/{embulk-output-jdbc-0.4.0.jar → embulk-output-jdbc-0.4.1.jar} +0 -0
- data/classpath/{embulk-output-oracle-0.4.0.jar → embulk-output-oracle-0.4.1.jar} +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +153 -153
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -165
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +124 -120
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +105 -100
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +38 -14
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +540 -484
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -0
- data/src/test/resources/yml/test-insert-direct-empty.yml +25 -0
- data/src/test/resources/yml/{test-insert-oci-split.yml → test-insert-direct-oci-method-split.yml} +28 -28
- data/src/test/resources/yml/{test-insert-oci.yml → test-insert-direct-oci-method.yml} +28 -28
- data/src/test/resources/yml/test-insert-direct.yml +25 -26
- data/src/test/resources/yml/test-insert-empty.yml +27 -0
- data/src/test/resources/yml/test-insert.yml +27 -25
- data/src/test/resources/yml/test-replace-empty.yml +31 -0
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name.yml +31 -31
- data/src/test/resources/yml/test-replace.yml +31 -31
- data/src/test/resources/yml/test-string-timestamp.yml +28 -28
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +10 -6
@@ -1,187 +1,187 @@
|
|
1
|
-
package org.embulk.input.filesplit;
|
2
|
-
|
3
|
-
import java.io.BufferedInputStream;
|
4
|
-
import java.io.ByteArrayInputStream;
|
5
|
-
import java.io.ByteArrayOutputStream;
|
6
|
-
import java.io.File;
|
7
|
-
import java.io.FileInputStream;
|
8
|
-
import java.io.IOException;
|
9
|
-
import java.io.InputStream;
|
10
|
-
import java.io.SequenceInputStream;
|
11
|
-
import java.util.ArrayList;
|
12
|
-
import java.util.List;
|
13
|
-
|
14
|
-
import org.embulk.config.CommitReport;
|
15
|
-
import org.embulk.config.Config;
|
16
|
-
import org.embulk.config.ConfigDefault;
|
17
|
-
import org.embulk.config.ConfigDiff;
|
18
|
-
import org.embulk.config.ConfigInject;
|
19
|
-
import org.embulk.config.ConfigSource;
|
20
|
-
import org.embulk.config.Task;
|
21
|
-
import org.embulk.config.TaskSource;
|
22
|
-
import org.embulk.spi.BufferAllocator;
|
23
|
-
import org.embulk.spi.Exec;
|
24
|
-
import org.embulk.spi.FileInputPlugin;
|
25
|
-
import org.embulk.spi.TransactionalFileInput;
|
26
|
-
import org.embulk.spi.util.InputStreamFileInput;
|
27
|
-
|
28
|
-
import com.google.common.base.Optional;
|
29
|
-
|
30
|
-
|
31
|
-
public class LocalFileSplitInputPlugin
|
32
|
-
implements FileInputPlugin
|
33
|
-
{
|
34
|
-
public interface PluginTask
|
35
|
-
extends Task
|
36
|
-
{
|
37
|
-
@Config("path")
|
38
|
-
public String getPath();
|
39
|
-
|
40
|
-
@Config("tasks")
|
41
|
-
@ConfigDefault("null")
|
42
|
-
public Optional<Integer> getTasks();
|
43
|
-
|
44
|
-
@Config("header_line")
|
45
|
-
@ConfigDefault("false")
|
46
|
-
public boolean getHeaderLine();
|
47
|
-
|
48
|
-
public List<PartialFile> getFiles();
|
49
|
-
public void setFiles(List<PartialFile> files);
|
50
|
-
|
51
|
-
@ConfigInject
|
52
|
-
public BufferAllocator getBufferAllocator();
|
53
|
-
}
|
54
|
-
|
55
|
-
@Override
|
56
|
-
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
57
|
-
{
|
58
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
59
|
-
|
60
|
-
int tasks;
|
61
|
-
if (task.getTasks().isPresent()) {
|
62
|
-
tasks = task.getTasks().get();
|
63
|
-
if (tasks <= 0) {
|
64
|
-
throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
|
65
|
-
}
|
66
|
-
} else {
|
67
|
-
tasks = Runtime.getRuntime().availableProcessors() * 2;
|
68
|
-
}
|
69
|
-
|
70
|
-
long size = new File(task.getPath()).length();
|
71
|
-
List<PartialFile> files = new ArrayList<PartialFile>();
|
72
|
-
for (int i = 0; i < tasks; i++) {
|
73
|
-
long start = size * i / tasks;
|
74
|
-
long end = size * (i + 1) / tasks;
|
75
|
-
if (start < end) {
|
76
|
-
files.add(new PartialFile(task.getPath(), start, end));
|
77
|
-
}
|
78
|
-
}
|
79
|
-
|
80
|
-
task.setFiles(files);
|
81
|
-
|
82
|
-
return resume(task.dump(), task.getFiles().size(), control);
|
83
|
-
}
|
84
|
-
|
85
|
-
@Override
|
86
|
-
public ConfigDiff resume(TaskSource taskSource,
|
87
|
-
int taskCount,
|
88
|
-
FileInputPlugin.Control control)
|
89
|
-
{
|
90
|
-
control.run(taskSource, taskCount);
|
91
|
-
|
92
|
-
return Exec.newConfigDiff();
|
93
|
-
}
|
94
|
-
|
95
|
-
@Override
|
96
|
-
public void cleanup(TaskSource taskSource,
|
97
|
-
int taskCount,
|
98
|
-
List<CommitReport> successCommitReports)
|
99
|
-
{ }
|
100
|
-
|
101
|
-
@Override
|
102
|
-
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
|
103
|
-
{
|
104
|
-
PluginTask task = taskSource.loadTask(PluginTask.class);
|
105
|
-
return new LocalFileSplitInput(task, taskIndex);
|
106
|
-
}
|
107
|
-
|
108
|
-
public static class LocalFileSplitInput
|
109
|
-
extends InputStreamFileInput
|
110
|
-
implements TransactionalFileInput
|
111
|
-
{
|
112
|
-
public static class FileSplitProvider
|
113
|
-
implements InputStreamFileInput.Provider
|
114
|
-
{
|
115
|
-
private final PartialFile file;
|
116
|
-
private final boolean hasHeader;
|
117
|
-
private boolean opened = false;
|
118
|
-
|
119
|
-
public FileSplitProvider(PartialFile file, boolean hasHeader)
|
120
|
-
{
|
121
|
-
this.file = file;
|
122
|
-
this.hasHeader = hasHeader;
|
123
|
-
}
|
124
|
-
|
125
|
-
@Override
|
126
|
-
public InputStream openNext() throws IOException
|
127
|
-
{
|
128
|
-
if (opened) {
|
129
|
-
return null;
|
130
|
-
}
|
131
|
-
opened = true;
|
132
|
-
|
133
|
-
InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
|
134
|
-
if (file.getStart() > 0 && hasHeader) {
|
135
|
-
in = new SequenceInputStream(openHeader(file.getPath()), in);
|
136
|
-
}
|
137
|
-
return in;
|
138
|
-
}
|
139
|
-
|
140
|
-
@Override
|
141
|
-
public void close() { }
|
142
|
-
|
143
|
-
private InputStream openHeader(String path) throws IOException
|
144
|
-
{
|
145
|
-
ByteArrayOutputStream header = new ByteArrayOutputStream();
|
146
|
-
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
|
147
|
-
while (true) {
|
148
|
-
int c = in.read();
|
149
|
-
if (c < 0) {
|
150
|
-
break;
|
151
|
-
}
|
152
|
-
|
153
|
-
header.write(c);
|
154
|
-
|
155
|
-
if (c == '\n') {
|
156
|
-
break;
|
157
|
-
}
|
158
|
-
|
159
|
-
if (c == '\r') {
|
160
|
-
int c2 = in.read();
|
161
|
-
if (c2 == '\n') {
|
162
|
-
header.write(c2);
|
163
|
-
}
|
164
|
-
break;
|
165
|
-
}
|
166
|
-
}
|
167
|
-
}
|
168
|
-
header.close();
|
169
|
-
return new ByteArrayInputStream(header.toByteArray());
|
170
|
-
}
|
171
|
-
}
|
172
|
-
|
173
|
-
public LocalFileSplitInput(PluginTask task, int taskIndex)
|
174
|
-
{
|
175
|
-
super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
|
176
|
-
}
|
177
|
-
|
178
|
-
@Override
|
179
|
-
public void abort() { }
|
180
|
-
|
181
|
-
@Override
|
182
|
-
public CommitReport commit()
|
183
|
-
{
|
184
|
-
return Exec.newCommitReport();
|
185
|
-
}
|
186
|
-
}
|
187
|
-
}
|
1
|
+
package org.embulk.input.filesplit;
|
2
|
+
|
3
|
+
import java.io.BufferedInputStream;
|
4
|
+
import java.io.ByteArrayInputStream;
|
5
|
+
import java.io.ByteArrayOutputStream;
|
6
|
+
import java.io.File;
|
7
|
+
import java.io.FileInputStream;
|
8
|
+
import java.io.IOException;
|
9
|
+
import java.io.InputStream;
|
10
|
+
import java.io.SequenceInputStream;
|
11
|
+
import java.util.ArrayList;
|
12
|
+
import java.util.List;
|
13
|
+
|
14
|
+
import org.embulk.config.CommitReport;
|
15
|
+
import org.embulk.config.Config;
|
16
|
+
import org.embulk.config.ConfigDefault;
|
17
|
+
import org.embulk.config.ConfigDiff;
|
18
|
+
import org.embulk.config.ConfigInject;
|
19
|
+
import org.embulk.config.ConfigSource;
|
20
|
+
import org.embulk.config.Task;
|
21
|
+
import org.embulk.config.TaskSource;
|
22
|
+
import org.embulk.spi.BufferAllocator;
|
23
|
+
import org.embulk.spi.Exec;
|
24
|
+
import org.embulk.spi.FileInputPlugin;
|
25
|
+
import org.embulk.spi.TransactionalFileInput;
|
26
|
+
import org.embulk.spi.util.InputStreamFileInput;
|
27
|
+
|
28
|
+
import com.google.common.base.Optional;
|
29
|
+
|
30
|
+
|
31
|
+
public class LocalFileSplitInputPlugin
|
32
|
+
implements FileInputPlugin
|
33
|
+
{
|
34
|
+
public interface PluginTask
|
35
|
+
extends Task
|
36
|
+
{
|
37
|
+
@Config("path")
|
38
|
+
public String getPath();
|
39
|
+
|
40
|
+
@Config("tasks")
|
41
|
+
@ConfigDefault("null")
|
42
|
+
public Optional<Integer> getTasks();
|
43
|
+
|
44
|
+
@Config("header_line")
|
45
|
+
@ConfigDefault("false")
|
46
|
+
public boolean getHeaderLine();
|
47
|
+
|
48
|
+
public List<PartialFile> getFiles();
|
49
|
+
public void setFiles(List<PartialFile> files);
|
50
|
+
|
51
|
+
@ConfigInject
|
52
|
+
public BufferAllocator getBufferAllocator();
|
53
|
+
}
|
54
|
+
|
55
|
+
@Override
|
56
|
+
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
57
|
+
{
|
58
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
59
|
+
|
60
|
+
int tasks;
|
61
|
+
if (task.getTasks().isPresent()) {
|
62
|
+
tasks = task.getTasks().get();
|
63
|
+
if (tasks <= 0) {
|
64
|
+
throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
|
65
|
+
}
|
66
|
+
} else {
|
67
|
+
tasks = Runtime.getRuntime().availableProcessors() * 2;
|
68
|
+
}
|
69
|
+
|
70
|
+
long size = new File(task.getPath()).length();
|
71
|
+
List<PartialFile> files = new ArrayList<PartialFile>();
|
72
|
+
for (int i = 0; i < tasks; i++) {
|
73
|
+
long start = size * i / tasks;
|
74
|
+
long end = size * (i + 1) / tasks;
|
75
|
+
if (start < end) {
|
76
|
+
files.add(new PartialFile(task.getPath(), start, end));
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
task.setFiles(files);
|
81
|
+
|
82
|
+
return resume(task.dump(), task.getFiles().size(), control);
|
83
|
+
}
|
84
|
+
|
85
|
+
@Override
|
86
|
+
public ConfigDiff resume(TaskSource taskSource,
|
87
|
+
int taskCount,
|
88
|
+
FileInputPlugin.Control control)
|
89
|
+
{
|
90
|
+
control.run(taskSource, taskCount);
|
91
|
+
|
92
|
+
return Exec.newConfigDiff();
|
93
|
+
}
|
94
|
+
|
95
|
+
@Override
|
96
|
+
public void cleanup(TaskSource taskSource,
|
97
|
+
int taskCount,
|
98
|
+
List<CommitReport> successCommitReports)
|
99
|
+
{ }
|
100
|
+
|
101
|
+
@Override
|
102
|
+
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
|
103
|
+
{
|
104
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
105
|
+
return new LocalFileSplitInput(task, taskIndex);
|
106
|
+
}
|
107
|
+
|
108
|
+
public static class LocalFileSplitInput
|
109
|
+
extends InputStreamFileInput
|
110
|
+
implements TransactionalFileInput
|
111
|
+
{
|
112
|
+
public static class FileSplitProvider
|
113
|
+
implements InputStreamFileInput.Provider
|
114
|
+
{
|
115
|
+
private final PartialFile file;
|
116
|
+
private final boolean hasHeader;
|
117
|
+
private boolean opened = false;
|
118
|
+
|
119
|
+
public FileSplitProvider(PartialFile file, boolean hasHeader)
|
120
|
+
{
|
121
|
+
this.file = file;
|
122
|
+
this.hasHeader = hasHeader;
|
123
|
+
}
|
124
|
+
|
125
|
+
@Override
|
126
|
+
public InputStream openNext() throws IOException
|
127
|
+
{
|
128
|
+
if (opened) {
|
129
|
+
return null;
|
130
|
+
}
|
131
|
+
opened = true;
|
132
|
+
|
133
|
+
InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
|
134
|
+
if (file.getStart() > 0 && hasHeader) {
|
135
|
+
in = new SequenceInputStream(openHeader(file.getPath()), in);
|
136
|
+
}
|
137
|
+
return in;
|
138
|
+
}
|
139
|
+
|
140
|
+
@Override
|
141
|
+
public void close() { }
|
142
|
+
|
143
|
+
private InputStream openHeader(String path) throws IOException
|
144
|
+
{
|
145
|
+
ByteArrayOutputStream header = new ByteArrayOutputStream();
|
146
|
+
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
|
147
|
+
while (true) {
|
148
|
+
int c = in.read();
|
149
|
+
if (c < 0) {
|
150
|
+
break;
|
151
|
+
}
|
152
|
+
|
153
|
+
header.write(c);
|
154
|
+
|
155
|
+
if (c == '\n') {
|
156
|
+
break;
|
157
|
+
}
|
158
|
+
|
159
|
+
if (c == '\r') {
|
160
|
+
int c2 = in.read();
|
161
|
+
if (c2 == '\n') {
|
162
|
+
header.write(c2);
|
163
|
+
}
|
164
|
+
break;
|
165
|
+
}
|
166
|
+
}
|
167
|
+
}
|
168
|
+
header.close();
|
169
|
+
return new ByteArrayInputStream(header.toByteArray());
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
public LocalFileSplitInput(PluginTask task, int taskIndex)
|
174
|
+
{
|
175
|
+
super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
|
176
|
+
}
|
177
|
+
|
178
|
+
@Override
|
179
|
+
public void abort() { }
|
180
|
+
|
181
|
+
@Override
|
182
|
+
public CommitReport commit()
|
183
|
+
{
|
184
|
+
return Exec.newCommitReport();
|
185
|
+
}
|
186
|
+
}
|
187
|
+
}
|
@@ -1,50 +1,50 @@
|
|
1
|
-
package org.embulk.input.filesplit;
|
2
|
-
|
3
|
-
public class PartialFile
|
4
|
-
{
|
5
|
-
private String path;
|
6
|
-
private long start;
|
7
|
-
private long end;
|
8
|
-
|
9
|
-
|
10
|
-
public PartialFile(String path, long start, long end)
|
11
|
-
{
|
12
|
-
this.path = path;
|
13
|
-
this.start = start;
|
14
|
-
this.end = end;
|
15
|
-
}
|
16
|
-
|
17
|
-
public PartialFile() {
|
18
|
-
}
|
19
|
-
|
20
|
-
public String getPath()
|
21
|
-
{
|
22
|
-
return path;
|
23
|
-
}
|
24
|
-
|
25
|
-
public void setPath(String path)
|
26
|
-
{
|
27
|
-
this.path = path;
|
28
|
-
}
|
29
|
-
|
30
|
-
|
31
|
-
public long getStart()
|
32
|
-
{
|
33
|
-
return start;
|
34
|
-
}
|
35
|
-
|
36
|
-
public void setStart(long start)
|
37
|
-
{
|
38
|
-
this.start = start;
|
39
|
-
}
|
40
|
-
|
41
|
-
public long getEnd()
|
42
|
-
{
|
43
|
-
return end;
|
44
|
-
}
|
45
|
-
|
46
|
-
public void setEnd(long end)
|
47
|
-
{
|
48
|
-
this.end = end;
|
49
|
-
}
|
1
|
+
package org.embulk.input.filesplit;
|
2
|
+
|
3
|
+
public class PartialFile
|
4
|
+
{
|
5
|
+
private String path;
|
6
|
+
private long start;
|
7
|
+
private long end;
|
8
|
+
|
9
|
+
|
10
|
+
public PartialFile(String path, long start, long end)
|
11
|
+
{
|
12
|
+
this.path = path;
|
13
|
+
this.start = start;
|
14
|
+
this.end = end;
|
15
|
+
}
|
16
|
+
|
17
|
+
public PartialFile() {
|
18
|
+
}
|
19
|
+
|
20
|
+
public String getPath()
|
21
|
+
{
|
22
|
+
return path;
|
23
|
+
}
|
24
|
+
|
25
|
+
public void setPath(String path)
|
26
|
+
{
|
27
|
+
this.path = path;
|
28
|
+
}
|
29
|
+
|
30
|
+
|
31
|
+
public long getStart()
|
32
|
+
{
|
33
|
+
return start;
|
34
|
+
}
|
35
|
+
|
36
|
+
public void setStart(long start)
|
37
|
+
{
|
38
|
+
this.start = start;
|
39
|
+
}
|
40
|
+
|
41
|
+
public long getEnd()
|
42
|
+
{
|
43
|
+
return end;
|
44
|
+
}
|
45
|
+
|
46
|
+
public void setEnd(long end)
|
47
|
+
{
|
48
|
+
this.end = end;
|
49
|
+
}
|
50
50
|
}
|