embulk-filter-copy 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/CHANGELOG.md +3 -0
- data/LICENSE.txt +21 -0
- data/README.md +67 -0
- data/build.gradle +104 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/example/config.yml +28 -0
- data/example/data.tsv +5 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +169 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/filter/copy.rb +8 -0
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java +149 -0
- data/src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java +17 -0
- data/src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java +147 -0
- data/src/main/java/org/embulk/filter/copy/forward/InForwardService.java +187 -0
- data/src/main/java/org/embulk/filter/copy/forward/InForwardVisitor.java +63 -0
- data/src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java +135 -0
- data/src/main/java/org/embulk/filter/copy/forward/OutForwardService.java +170 -0
- data/src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java +63 -0
- data/src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java +111 -0
- data/src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java +111 -0
- data/src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java +64 -0
- data/src/main/java/org/embulk/filter/copy/util/ElapsedTime.java +165 -0
- data/src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java +5 -0
- data/src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java +5 -0
- metadata +111 -0
@@ -0,0 +1,64 @@
|
|
1
|
+
package org.embulk.filter.copy.service;
|
2
|
+
|
3
|
+
import org.embulk.spi.Column;
|
4
|
+
import org.embulk.spi.ColumnVisitor;
|
5
|
+
import org.embulk.spi.PageBuilder;
|
6
|
+
import org.embulk.spi.PageReader;
|
7
|
+
|
8
|
+
public class StandardColumnVisitor
|
9
|
+
implements ColumnVisitor
|
10
|
+
{
|
11
|
+
private final PageReader reader;
|
12
|
+
private final PageBuilder builder;
|
13
|
+
|
14
|
+
public StandardColumnVisitor(PageReader reader, PageBuilder builder)
|
15
|
+
{
|
16
|
+
this.reader = reader;
|
17
|
+
this.builder = builder;
|
18
|
+
}
|
19
|
+
|
20
|
+
private void nullOr(Column column, Runnable runnable)
|
21
|
+
{
|
22
|
+
if (reader.isNull(column)) {
|
23
|
+
builder.setNull(column);
|
24
|
+
return;
|
25
|
+
}
|
26
|
+
runnable.run();
|
27
|
+
}
|
28
|
+
|
29
|
+
@Override
|
30
|
+
public void booleanColumn(Column column)
|
31
|
+
{
|
32
|
+
nullOr(column, () -> builder.setBoolean(column, reader.getBoolean(column)));
|
33
|
+
}
|
34
|
+
|
35
|
+
@Override
|
36
|
+
public void longColumn(Column column)
|
37
|
+
{
|
38
|
+
nullOr(column, () -> builder.setLong(column, reader.getLong(column)));
|
39
|
+
}
|
40
|
+
|
41
|
+
@Override
|
42
|
+
public void doubleColumn(Column column)
|
43
|
+
{
|
44
|
+
nullOr(column, () -> builder.setDouble(column, reader.getDouble(column)));
|
45
|
+
}
|
46
|
+
|
47
|
+
@Override
|
48
|
+
public void stringColumn(Column column)
|
49
|
+
{
|
50
|
+
nullOr(column, () -> builder.setString(column, reader.getString(column)));
|
51
|
+
}
|
52
|
+
|
53
|
+
@Override
|
54
|
+
public void timestampColumn(Column column)
|
55
|
+
{
|
56
|
+
nullOr(column, () -> builder.setTimestamp(column, reader.getTimestamp(column)));
|
57
|
+
}
|
58
|
+
|
59
|
+
@Override
|
60
|
+
public void jsonColumn(Column column)
|
61
|
+
{
|
62
|
+
nullOr(column, () -> builder.setJson(column, reader.getJson(column)));
|
63
|
+
}
|
64
|
+
}
|
@@ -0,0 +1,165 @@
|
|
1
|
+
package org.embulk.filter.copy.util;
|
2
|
+
|
3
|
+
import org.slf4j.Logger;
|
4
|
+
|
5
|
+
import java.util.function.Supplier;
|
6
|
+
|
7
|
+
public class ElapsedTime
|
8
|
+
{
|
9
|
+
private ElapsedTime()
|
10
|
+
{
|
11
|
+
}
|
12
|
+
|
13
|
+
private final static String TEMPLATE = " ({} ms)";
|
14
|
+
private enum LogLevel {TRACE, DEBUG, INFO, WARN, ERROR}
|
15
|
+
private enum Status {START, RUNNING, FINISHED}
|
16
|
+
|
17
|
+
private static void log(Logger logger, LogLevel level, String message, long elapsed)
|
18
|
+
{
|
19
|
+
String msgWithTmpl = message + TEMPLATE;
|
20
|
+
switch (level)
|
21
|
+
{
|
22
|
+
case TRACE:
|
23
|
+
logger.trace(msgWithTmpl, elapsed);
|
24
|
+
break;
|
25
|
+
case DEBUG:
|
26
|
+
logger.debug(msgWithTmpl, elapsed);
|
27
|
+
break;
|
28
|
+
case INFO:
|
29
|
+
logger.info(msgWithTmpl, elapsed);
|
30
|
+
break;
|
31
|
+
case WARN:
|
32
|
+
logger.warn(msgWithTmpl, elapsed);
|
33
|
+
break;
|
34
|
+
case ERROR:
|
35
|
+
logger.error(msgWithTmpl, elapsed);
|
36
|
+
break;
|
37
|
+
}
|
38
|
+
}
|
39
|
+
|
40
|
+
private static <T>T run(Logger logger, LogLevel level, String message, Supplier<T> proc)
|
41
|
+
{
|
42
|
+
long start = System.currentTimeMillis();
|
43
|
+
try {
|
44
|
+
return proc.get();
|
45
|
+
}
|
46
|
+
finally {
|
47
|
+
log(logger, level, message, System.currentTimeMillis() - start);
|
48
|
+
}
|
49
|
+
}
|
50
|
+
|
51
|
+
private static void run(Logger logger, LogLevel level, String message, Runnable proc)
|
52
|
+
{
|
53
|
+
long start = System.currentTimeMillis();
|
54
|
+
try {
|
55
|
+
proc.run();
|
56
|
+
}
|
57
|
+
finally {
|
58
|
+
log(logger, level, message, System.currentTimeMillis() - start);
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
public static <T>T trace(Logger logger, String message, Supplier<T> proc)
|
63
|
+
{
|
64
|
+
return run(logger, LogLevel.TRACE, message, proc);
|
65
|
+
}
|
66
|
+
|
67
|
+
public static <T>T debug(Logger logger, String message, Supplier<T> proc)
|
68
|
+
{
|
69
|
+
return run(logger, LogLevel.DEBUG, message, proc);
|
70
|
+
}
|
71
|
+
|
72
|
+
public static <T>T info(Logger logger, String message, Supplier<T> proc)
|
73
|
+
{
|
74
|
+
return run(logger, LogLevel.INFO, message, proc);
|
75
|
+
}
|
76
|
+
|
77
|
+
public static <T>T warn(Logger logger, String message, Supplier<T> proc)
|
78
|
+
{
|
79
|
+
return run(logger, LogLevel.WARN, message, proc);
|
80
|
+
}
|
81
|
+
|
82
|
+
public static <T>T error(Logger logger, String message, Supplier<T> proc)
|
83
|
+
{
|
84
|
+
return run(logger, LogLevel.ERROR, message, proc);
|
85
|
+
}
|
86
|
+
|
87
|
+
public static void trace(Logger logger, String message, Runnable proc)
|
88
|
+
{
|
89
|
+
run(logger, LogLevel.TRACE, message, proc);
|
90
|
+
}
|
91
|
+
|
92
|
+
public static void debug(Logger logger, String message, Runnable proc)
|
93
|
+
{
|
94
|
+
run(logger, LogLevel.DEBUG, message, proc);
|
95
|
+
}
|
96
|
+
|
97
|
+
public static void info(Logger logger, String message, Runnable proc)
|
98
|
+
{
|
99
|
+
run(logger, LogLevel.INFO, message, proc);
|
100
|
+
}
|
101
|
+
|
102
|
+
public static void warn(Logger logger, String message, Runnable proc)
|
103
|
+
{
|
104
|
+
run(logger, LogLevel.WARN, message, proc);
|
105
|
+
}
|
106
|
+
|
107
|
+
public static void error(Logger logger, String message, Runnable proc)
|
108
|
+
{
|
109
|
+
run(logger, LogLevel.ERROR, message, proc);
|
110
|
+
}
|
111
|
+
|
112
|
+
private static void logUntil(Supplier<Boolean> condition,
|
113
|
+
Logger logger, LogLevel level, String message, long sleep)
|
114
|
+
{
|
115
|
+
long start = System.currentTimeMillis();
|
116
|
+
log(logger, level, String.format("(%s) %s", Status.START, message),
|
117
|
+
System.currentTimeMillis() - start);
|
118
|
+
while (true) {
|
119
|
+
if (condition.get()) {
|
120
|
+
log(logger, level, String.format("(%s) %s", Status.FINISHED, message),
|
121
|
+
System.currentTimeMillis() - start);
|
122
|
+
break;
|
123
|
+
}
|
124
|
+
try {
|
125
|
+
Thread.sleep(sleep);
|
126
|
+
}
|
127
|
+
catch (InterruptedException e) {
|
128
|
+
logger.warn(e.getMessage(), e);
|
129
|
+
}
|
130
|
+
|
131
|
+
log(logger, level, String.format("(%s) %s", Status.RUNNING, message),
|
132
|
+
System.currentTimeMillis() - start);
|
133
|
+
}
|
134
|
+
}
|
135
|
+
|
136
|
+
public static void traceUntil(Supplier<Boolean> condition,
|
137
|
+
Logger logger, String message, long sleep)
|
138
|
+
{
|
139
|
+
logUntil(condition, logger, LogLevel.TRACE, message, sleep);
|
140
|
+
}
|
141
|
+
|
142
|
+
public static void debugUntil(Supplier<Boolean> condition,
|
143
|
+
Logger logger, String message, long sleep)
|
144
|
+
{
|
145
|
+
logUntil(condition, logger, LogLevel.DEBUG, message, sleep);
|
146
|
+
}
|
147
|
+
|
148
|
+
public static void infoUntil(Supplier<Boolean> condition,
|
149
|
+
Logger logger, String message, long sleep)
|
150
|
+
{
|
151
|
+
logUntil(condition, logger, LogLevel.INFO, message, sleep);
|
152
|
+
}
|
153
|
+
|
154
|
+
public static void warnUntil(Supplier<Boolean> condition,
|
155
|
+
Logger logger, String message, long sleep)
|
156
|
+
{
|
157
|
+
logUntil(condition, logger, LogLevel.WARN, message, sleep);
|
158
|
+
}
|
159
|
+
|
160
|
+
public static void errorUntil(Supplier<Boolean> condition,
|
161
|
+
Logger logger, String message, long sleep)
|
162
|
+
{
|
163
|
+
logUntil(condition, logger, LogLevel.ERROR, message, sleep);
|
164
|
+
}
|
165
|
+
}
|
metadata
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-filter-copy
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Civitaspo
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-06-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ~>
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.0'
|
19
|
+
name: bundler
|
20
|
+
prerelease: false
|
21
|
+
type: :development
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '10.0'
|
33
|
+
name: rake
|
34
|
+
prerelease: false
|
35
|
+
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
description: Copy records and run another embulk by using them as input data source.
|
42
|
+
email:
|
43
|
+
- civitaspo@gmail.com
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- .gitignore
|
49
|
+
- CHANGELOG.md
|
50
|
+
- LICENSE.txt
|
51
|
+
- README.md
|
52
|
+
- build.gradle
|
53
|
+
- config/checkstyle/checkstyle.xml
|
54
|
+
- config/checkstyle/default.xml
|
55
|
+
- example/config.yml
|
56
|
+
- example/data.tsv
|
57
|
+
- gradle/wrapper/gradle-wrapper.jar
|
58
|
+
- gradle/wrapper/gradle-wrapper.properties
|
59
|
+
- gradlew
|
60
|
+
- gradlew.bat
|
61
|
+
- lib/embulk/filter/copy.rb
|
62
|
+
- settings.gradle
|
63
|
+
- src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java
|
64
|
+
- src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java
|
65
|
+
- src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java
|
66
|
+
- src/main/java/org/embulk/filter/copy/forward/InForwardService.java
|
67
|
+
- src/main/java/org/embulk/filter/copy/forward/InForwardVisitor.java
|
68
|
+
- src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java
|
69
|
+
- src/main/java/org/embulk/filter/copy/forward/OutForwardService.java
|
70
|
+
- src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java
|
71
|
+
- src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java
|
72
|
+
- src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java
|
73
|
+
- src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java
|
74
|
+
- src/main/java/org/embulk/filter/copy/util/ElapsedTime.java
|
75
|
+
- src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java
|
76
|
+
- src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java
|
77
|
+
- classpath/embulk-filter-copy-0.0.1.jar
|
78
|
+
- classpath/fluency-1.1.0.jar
|
79
|
+
- classpath/influent-java-0.2.0.jar
|
80
|
+
- classpath/influent-transport-0.2.0.jar
|
81
|
+
- classpath/jackson-annotations-2.7.0.jar
|
82
|
+
- classpath/jackson-core-2.7.1.jar
|
83
|
+
- classpath/jackson-databind-2.7.1.jar
|
84
|
+
- classpath/jackson-dataformat-msgpack-0.8.11.jar
|
85
|
+
- classpath/phi-accural-failure-detector-0.0.4.jar
|
86
|
+
- classpath/slf4j-api-1.7.22.jar
|
87
|
+
homepage: https://github.com/civitaspo/embulk-filter-copy
|
88
|
+
licenses:
|
89
|
+
- MIT
|
90
|
+
metadata: {}
|
91
|
+
post_install_message:
|
92
|
+
rdoc_options: []
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - '>='
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project:
|
107
|
+
rubygems_version: 2.1.9
|
108
|
+
signing_key:
|
109
|
+
specification_version: 4
|
110
|
+
summary: Copy filter plugin for Embulk
|
111
|
+
test_files: []
|