embulk-output-oracle 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +160 -160
- data/build.gradle +6 -6
- data/classpath/{embulk-output-jdbc-0.4.0.jar → embulk-output-jdbc-0.4.1.jar} +0 -0
- data/classpath/{embulk-output-oracle-0.4.0.jar → embulk-output-oracle-0.4.1.jar} +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +153 -153
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -165
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +124 -120
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +105 -100
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +38 -14
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +540 -484
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -0
- data/src/test/resources/yml/test-insert-direct-empty.yml +25 -0
- data/src/test/resources/yml/{test-insert-oci-split.yml → test-insert-direct-oci-method-split.yml} +28 -28
- data/src/test/resources/yml/{test-insert-oci.yml → test-insert-direct-oci-method.yml} +28 -28
- data/src/test/resources/yml/test-insert-direct.yml +25 -26
- data/src/test/resources/yml/test-insert-empty.yml +27 -0
- data/src/test/resources/yml/test-insert.yml +27 -25
- data/src/test/resources/yml/test-replace-empty.yml +31 -0
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name.yml +31 -31
- data/src/test/resources/yml/test-replace.yml +31 -31
- data/src/test/resources/yml/test-string-timestamp.yml +28 -28
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +10 -6
@@ -1,187 +1,187 @@
|
|
1
|
-
package org.embulk.input.filesplit;
|
2
|
-
|
3
|
-
import java.io.BufferedInputStream;
|
4
|
-
import java.io.ByteArrayInputStream;
|
5
|
-
import java.io.ByteArrayOutputStream;
|
6
|
-
import java.io.File;
|
7
|
-
import java.io.FileInputStream;
|
8
|
-
import java.io.IOException;
|
9
|
-
import java.io.InputStream;
|
10
|
-
import java.io.SequenceInputStream;
|
11
|
-
import java.util.ArrayList;
|
12
|
-
import java.util.List;
|
13
|
-
|
14
|
-
import org.embulk.config.CommitReport;
|
15
|
-
import org.embulk.config.Config;
|
16
|
-
import org.embulk.config.ConfigDefault;
|
17
|
-
import org.embulk.config.ConfigDiff;
|
18
|
-
import org.embulk.config.ConfigInject;
|
19
|
-
import org.embulk.config.ConfigSource;
|
20
|
-
import org.embulk.config.Task;
|
21
|
-
import org.embulk.config.TaskSource;
|
22
|
-
import org.embulk.spi.BufferAllocator;
|
23
|
-
import org.embulk.spi.Exec;
|
24
|
-
import org.embulk.spi.FileInputPlugin;
|
25
|
-
import org.embulk.spi.TransactionalFileInput;
|
26
|
-
import org.embulk.spi.util.InputStreamFileInput;
|
27
|
-
|
28
|
-
import com.google.common.base.Optional;
|
29
|
-
|
30
|
-
|
31
|
-
public class LocalFileSplitInputPlugin
|
32
|
-
implements FileInputPlugin
|
33
|
-
{
|
34
|
-
public interface PluginTask
|
35
|
-
extends Task
|
36
|
-
{
|
37
|
-
@Config("path")
|
38
|
-
public String getPath();
|
39
|
-
|
40
|
-
@Config("tasks")
|
41
|
-
@ConfigDefault("null")
|
42
|
-
public Optional<Integer> getTasks();
|
43
|
-
|
44
|
-
@Config("header_line")
|
45
|
-
@ConfigDefault("false")
|
46
|
-
public boolean getHeaderLine();
|
47
|
-
|
48
|
-
public List<PartialFile> getFiles();
|
49
|
-
public void setFiles(List<PartialFile> files);
|
50
|
-
|
51
|
-
@ConfigInject
|
52
|
-
public BufferAllocator getBufferAllocator();
|
53
|
-
}
|
54
|
-
|
55
|
-
@Override
|
56
|
-
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
57
|
-
{
|
58
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
59
|
-
|
60
|
-
int tasks;
|
61
|
-
if (task.getTasks().isPresent()) {
|
62
|
-
tasks = task.getTasks().get();
|
63
|
-
if (tasks <= 0) {
|
64
|
-
throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
|
65
|
-
}
|
66
|
-
} else {
|
67
|
-
tasks = Runtime.getRuntime().availableProcessors() * 2;
|
68
|
-
}
|
69
|
-
|
70
|
-
long size = new File(task.getPath()).length();
|
71
|
-
List<PartialFile> files = new ArrayList<PartialFile>();
|
72
|
-
for (int i = 0; i < tasks; i++) {
|
73
|
-
long start = size * i / tasks;
|
74
|
-
long end = size * (i + 1) / tasks;
|
75
|
-
if (start < end) {
|
76
|
-
files.add(new PartialFile(task.getPath(), start, end));
|
77
|
-
}
|
78
|
-
}
|
79
|
-
|
80
|
-
task.setFiles(files);
|
81
|
-
|
82
|
-
return resume(task.dump(), task.getFiles().size(), control);
|
83
|
-
}
|
84
|
-
|
85
|
-
@Override
|
86
|
-
public ConfigDiff resume(TaskSource taskSource,
|
87
|
-
int taskCount,
|
88
|
-
FileInputPlugin.Control control)
|
89
|
-
{
|
90
|
-
control.run(taskSource, taskCount);
|
91
|
-
|
92
|
-
return Exec.newConfigDiff();
|
93
|
-
}
|
94
|
-
|
95
|
-
@Override
|
96
|
-
public void cleanup(TaskSource taskSource,
|
97
|
-
int taskCount,
|
98
|
-
List<CommitReport> successCommitReports)
|
99
|
-
{ }
|
100
|
-
|
101
|
-
@Override
|
102
|
-
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
|
103
|
-
{
|
104
|
-
PluginTask task = taskSource.loadTask(PluginTask.class);
|
105
|
-
return new LocalFileSplitInput(task, taskIndex);
|
106
|
-
}
|
107
|
-
|
108
|
-
public static class LocalFileSplitInput
|
109
|
-
extends InputStreamFileInput
|
110
|
-
implements TransactionalFileInput
|
111
|
-
{
|
112
|
-
public static class FileSplitProvider
|
113
|
-
implements InputStreamFileInput.Provider
|
114
|
-
{
|
115
|
-
private final PartialFile file;
|
116
|
-
private final boolean hasHeader;
|
117
|
-
private boolean opened = false;
|
118
|
-
|
119
|
-
public FileSplitProvider(PartialFile file, boolean hasHeader)
|
120
|
-
{
|
121
|
-
this.file = file;
|
122
|
-
this.hasHeader = hasHeader;
|
123
|
-
}
|
124
|
-
|
125
|
-
@Override
|
126
|
-
public InputStream openNext() throws IOException
|
127
|
-
{
|
128
|
-
if (opened) {
|
129
|
-
return null;
|
130
|
-
}
|
131
|
-
opened = true;
|
132
|
-
|
133
|
-
InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
|
134
|
-
if (file.getStart() > 0 && hasHeader) {
|
135
|
-
in = new SequenceInputStream(openHeader(file.getPath()), in);
|
136
|
-
}
|
137
|
-
return in;
|
138
|
-
}
|
139
|
-
|
140
|
-
@Override
|
141
|
-
public void close() { }
|
142
|
-
|
143
|
-
private InputStream openHeader(String path) throws IOException
|
144
|
-
{
|
145
|
-
ByteArrayOutputStream header = new ByteArrayOutputStream();
|
146
|
-
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
|
147
|
-
while (true) {
|
148
|
-
int c = in.read();
|
149
|
-
if (c < 0) {
|
150
|
-
break;
|
151
|
-
}
|
152
|
-
|
153
|
-
header.write(c);
|
154
|
-
|
155
|
-
if (c == '\n') {
|
156
|
-
break;
|
157
|
-
}
|
158
|
-
|
159
|
-
if (c == '\r') {
|
160
|
-
int c2 = in.read();
|
161
|
-
if (c2 == '\n') {
|
162
|
-
header.write(c2);
|
163
|
-
}
|
164
|
-
break;
|
165
|
-
}
|
166
|
-
}
|
167
|
-
}
|
168
|
-
header.close();
|
169
|
-
return new ByteArrayInputStream(header.toByteArray());
|
170
|
-
}
|
171
|
-
}
|
172
|
-
|
173
|
-
public LocalFileSplitInput(PluginTask task, int taskIndex)
|
174
|
-
{
|
175
|
-
super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
|
176
|
-
}
|
177
|
-
|
178
|
-
@Override
|
179
|
-
public void abort() { }
|
180
|
-
|
181
|
-
@Override
|
182
|
-
public CommitReport commit()
|
183
|
-
{
|
184
|
-
return Exec.newCommitReport();
|
185
|
-
}
|
186
|
-
}
|
187
|
-
}
|
1
|
+
package org.embulk.input.filesplit;
|
2
|
+
|
3
|
+
import java.io.BufferedInputStream;
|
4
|
+
import java.io.ByteArrayInputStream;
|
5
|
+
import java.io.ByteArrayOutputStream;
|
6
|
+
import java.io.File;
|
7
|
+
import java.io.FileInputStream;
|
8
|
+
import java.io.IOException;
|
9
|
+
import java.io.InputStream;
|
10
|
+
import java.io.SequenceInputStream;
|
11
|
+
import java.util.ArrayList;
|
12
|
+
import java.util.List;
|
13
|
+
|
14
|
+
import org.embulk.config.CommitReport;
|
15
|
+
import org.embulk.config.Config;
|
16
|
+
import org.embulk.config.ConfigDefault;
|
17
|
+
import org.embulk.config.ConfigDiff;
|
18
|
+
import org.embulk.config.ConfigInject;
|
19
|
+
import org.embulk.config.ConfigSource;
|
20
|
+
import org.embulk.config.Task;
|
21
|
+
import org.embulk.config.TaskSource;
|
22
|
+
import org.embulk.spi.BufferAllocator;
|
23
|
+
import org.embulk.spi.Exec;
|
24
|
+
import org.embulk.spi.FileInputPlugin;
|
25
|
+
import org.embulk.spi.TransactionalFileInput;
|
26
|
+
import org.embulk.spi.util.InputStreamFileInput;
|
27
|
+
|
28
|
+
import com.google.common.base.Optional;
|
29
|
+
|
30
|
+
|
31
|
+
public class LocalFileSplitInputPlugin
|
32
|
+
implements FileInputPlugin
|
33
|
+
{
|
34
|
+
public interface PluginTask
|
35
|
+
extends Task
|
36
|
+
{
|
37
|
+
@Config("path")
|
38
|
+
public String getPath();
|
39
|
+
|
40
|
+
@Config("tasks")
|
41
|
+
@ConfigDefault("null")
|
42
|
+
public Optional<Integer> getTasks();
|
43
|
+
|
44
|
+
@Config("header_line")
|
45
|
+
@ConfigDefault("false")
|
46
|
+
public boolean getHeaderLine();
|
47
|
+
|
48
|
+
public List<PartialFile> getFiles();
|
49
|
+
public void setFiles(List<PartialFile> files);
|
50
|
+
|
51
|
+
@ConfigInject
|
52
|
+
public BufferAllocator getBufferAllocator();
|
53
|
+
}
|
54
|
+
|
55
|
+
@Override
|
56
|
+
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
57
|
+
{
|
58
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
59
|
+
|
60
|
+
int tasks;
|
61
|
+
if (task.getTasks().isPresent()) {
|
62
|
+
tasks = task.getTasks().get();
|
63
|
+
if (tasks <= 0) {
|
64
|
+
throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
|
65
|
+
}
|
66
|
+
} else {
|
67
|
+
tasks = Runtime.getRuntime().availableProcessors() * 2;
|
68
|
+
}
|
69
|
+
|
70
|
+
long size = new File(task.getPath()).length();
|
71
|
+
List<PartialFile> files = new ArrayList<PartialFile>();
|
72
|
+
for (int i = 0; i < tasks; i++) {
|
73
|
+
long start = size * i / tasks;
|
74
|
+
long end = size * (i + 1) / tasks;
|
75
|
+
if (start < end) {
|
76
|
+
files.add(new PartialFile(task.getPath(), start, end));
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
task.setFiles(files);
|
81
|
+
|
82
|
+
return resume(task.dump(), task.getFiles().size(), control);
|
83
|
+
}
|
84
|
+
|
85
|
+
@Override
|
86
|
+
public ConfigDiff resume(TaskSource taskSource,
|
87
|
+
int taskCount,
|
88
|
+
FileInputPlugin.Control control)
|
89
|
+
{
|
90
|
+
control.run(taskSource, taskCount);
|
91
|
+
|
92
|
+
return Exec.newConfigDiff();
|
93
|
+
}
|
94
|
+
|
95
|
+
@Override
|
96
|
+
public void cleanup(TaskSource taskSource,
|
97
|
+
int taskCount,
|
98
|
+
List<CommitReport> successCommitReports)
|
99
|
+
{ }
|
100
|
+
|
101
|
+
@Override
|
102
|
+
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
|
103
|
+
{
|
104
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
105
|
+
return new LocalFileSplitInput(task, taskIndex);
|
106
|
+
}
|
107
|
+
|
108
|
+
public static class LocalFileSplitInput
|
109
|
+
extends InputStreamFileInput
|
110
|
+
implements TransactionalFileInput
|
111
|
+
{
|
112
|
+
public static class FileSplitProvider
|
113
|
+
implements InputStreamFileInput.Provider
|
114
|
+
{
|
115
|
+
private final PartialFile file;
|
116
|
+
private final boolean hasHeader;
|
117
|
+
private boolean opened = false;
|
118
|
+
|
119
|
+
public FileSplitProvider(PartialFile file, boolean hasHeader)
|
120
|
+
{
|
121
|
+
this.file = file;
|
122
|
+
this.hasHeader = hasHeader;
|
123
|
+
}
|
124
|
+
|
125
|
+
@Override
|
126
|
+
public InputStream openNext() throws IOException
|
127
|
+
{
|
128
|
+
if (opened) {
|
129
|
+
return null;
|
130
|
+
}
|
131
|
+
opened = true;
|
132
|
+
|
133
|
+
InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
|
134
|
+
if (file.getStart() > 0 && hasHeader) {
|
135
|
+
in = new SequenceInputStream(openHeader(file.getPath()), in);
|
136
|
+
}
|
137
|
+
return in;
|
138
|
+
}
|
139
|
+
|
140
|
+
@Override
|
141
|
+
public void close() { }
|
142
|
+
|
143
|
+
private InputStream openHeader(String path) throws IOException
|
144
|
+
{
|
145
|
+
ByteArrayOutputStream header = new ByteArrayOutputStream();
|
146
|
+
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
|
147
|
+
while (true) {
|
148
|
+
int c = in.read();
|
149
|
+
if (c < 0) {
|
150
|
+
break;
|
151
|
+
}
|
152
|
+
|
153
|
+
header.write(c);
|
154
|
+
|
155
|
+
if (c == '\n') {
|
156
|
+
break;
|
157
|
+
}
|
158
|
+
|
159
|
+
if (c == '\r') {
|
160
|
+
int c2 = in.read();
|
161
|
+
if (c2 == '\n') {
|
162
|
+
header.write(c2);
|
163
|
+
}
|
164
|
+
break;
|
165
|
+
}
|
166
|
+
}
|
167
|
+
}
|
168
|
+
header.close();
|
169
|
+
return new ByteArrayInputStream(header.toByteArray());
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
public LocalFileSplitInput(PluginTask task, int taskIndex)
|
174
|
+
{
|
175
|
+
super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
|
176
|
+
}
|
177
|
+
|
178
|
+
@Override
|
179
|
+
public void abort() { }
|
180
|
+
|
181
|
+
@Override
|
182
|
+
public CommitReport commit()
|
183
|
+
{
|
184
|
+
return Exec.newCommitReport();
|
185
|
+
}
|
186
|
+
}
|
187
|
+
}
|
@@ -1,50 +1,50 @@
|
|
1
|
-
package org.embulk.input.filesplit;
|
2
|
-
|
3
|
-
public class PartialFile
|
4
|
-
{
|
5
|
-
private String path;
|
6
|
-
private long start;
|
7
|
-
private long end;
|
8
|
-
|
9
|
-
|
10
|
-
public PartialFile(String path, long start, long end)
|
11
|
-
{
|
12
|
-
this.path = path;
|
13
|
-
this.start = start;
|
14
|
-
this.end = end;
|
15
|
-
}
|
16
|
-
|
17
|
-
public PartialFile() {
|
18
|
-
}
|
19
|
-
|
20
|
-
public String getPath()
|
21
|
-
{
|
22
|
-
return path;
|
23
|
-
}
|
24
|
-
|
25
|
-
public void setPath(String path)
|
26
|
-
{
|
27
|
-
this.path = path;
|
28
|
-
}
|
29
|
-
|
30
|
-
|
31
|
-
public long getStart()
|
32
|
-
{
|
33
|
-
return start;
|
34
|
-
}
|
35
|
-
|
36
|
-
public void setStart(long start)
|
37
|
-
{
|
38
|
-
this.start = start;
|
39
|
-
}
|
40
|
-
|
41
|
-
public long getEnd()
|
42
|
-
{
|
43
|
-
return end;
|
44
|
-
}
|
45
|
-
|
46
|
-
public void setEnd(long end)
|
47
|
-
{
|
48
|
-
this.end = end;
|
49
|
-
}
|
1
|
+
package org.embulk.input.filesplit;
|
2
|
+
|
3
|
+
public class PartialFile
|
4
|
+
{
|
5
|
+
private String path;
|
6
|
+
private long start;
|
7
|
+
private long end;
|
8
|
+
|
9
|
+
|
10
|
+
public PartialFile(String path, long start, long end)
|
11
|
+
{
|
12
|
+
this.path = path;
|
13
|
+
this.start = start;
|
14
|
+
this.end = end;
|
15
|
+
}
|
16
|
+
|
17
|
+
public PartialFile() {
|
18
|
+
}
|
19
|
+
|
20
|
+
public String getPath()
|
21
|
+
{
|
22
|
+
return path;
|
23
|
+
}
|
24
|
+
|
25
|
+
public void setPath(String path)
|
26
|
+
{
|
27
|
+
this.path = path;
|
28
|
+
}
|
29
|
+
|
30
|
+
|
31
|
+
public long getStart()
|
32
|
+
{
|
33
|
+
return start;
|
34
|
+
}
|
35
|
+
|
36
|
+
public void setStart(long start)
|
37
|
+
{
|
38
|
+
this.start = start;
|
39
|
+
}
|
40
|
+
|
41
|
+
public long getEnd()
|
42
|
+
{
|
43
|
+
return end;
|
44
|
+
}
|
45
|
+
|
46
|
+
public void setEnd(long end)
|
47
|
+
{
|
48
|
+
this.end = end;
|
49
|
+
}
|
50
50
|
}
|