embulk-filter-copy 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/CHANGELOG.md +3 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +67 -0
  6. data/build.gradle +104 -0
  7. data/config/checkstyle/checkstyle.xml +128 -0
  8. data/config/checkstyle/default.xml +108 -0
  9. data/example/config.yml +28 -0
  10. data/example/data.tsv +5 -0
  11. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  12. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  13. data/gradlew +169 -0
  14. data/gradlew.bat +84 -0
  15. data/lib/embulk/filter/copy.rb +8 -0
  16. data/settings.gradle +1 -0
  17. data/src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java +149 -0
  18. data/src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java +17 -0
  19. data/src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java +147 -0
  20. data/src/main/java/org/embulk/filter/copy/forward/InForwardService.java +187 -0
  21. data/src/main/java/org/embulk/filter/copy/forward/InForwardVisitor.java +63 -0
  22. data/src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java +135 -0
  23. data/src/main/java/org/embulk/filter/copy/forward/OutForwardService.java +170 -0
  24. data/src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java +63 -0
  25. data/src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java +111 -0
  26. data/src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java +111 -0
  27. data/src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java +64 -0
  28. data/src/main/java/org/embulk/filter/copy/util/ElapsedTime.java +165 -0
  29. data/src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java +5 -0
  30. data/src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java +5 -0
  31. metadata +111 -0
@@ -0,0 +1,64 @@
1
+ package org.embulk.filter.copy.service;
2
+
3
+ import org.embulk.spi.Column;
4
+ import org.embulk.spi.ColumnVisitor;
5
+ import org.embulk.spi.PageBuilder;
6
+ import org.embulk.spi.PageReader;
7
+
8
+ public class StandardColumnVisitor
9
+ implements ColumnVisitor
10
+ {
11
+ private final PageReader reader;
12
+ private final PageBuilder builder;
13
+
14
+ public StandardColumnVisitor(PageReader reader, PageBuilder builder)
15
+ {
16
+ this.reader = reader;
17
+ this.builder = builder;
18
+ }
19
+
20
+ private void nullOr(Column column, Runnable runnable)
21
+ {
22
+ if (reader.isNull(column)) {
23
+ builder.setNull(column);
24
+ return;
25
+ }
26
+ runnable.run();
27
+ }
28
+
29
+ @Override
30
+ public void booleanColumn(Column column)
31
+ {
32
+ nullOr(column, () -> builder.setBoolean(column, reader.getBoolean(column)));
33
+ }
34
+
35
+ @Override
36
+ public void longColumn(Column column)
37
+ {
38
+ nullOr(column, () -> builder.setLong(column, reader.getLong(column)));
39
+ }
40
+
41
+ @Override
42
+ public void doubleColumn(Column column)
43
+ {
44
+ nullOr(column, () -> builder.setDouble(column, reader.getDouble(column)));
45
+ }
46
+
47
+ @Override
48
+ public void stringColumn(Column column)
49
+ {
50
+ nullOr(column, () -> builder.setString(column, reader.getString(column)));
51
+ }
52
+
53
+ @Override
54
+ public void timestampColumn(Column column)
55
+ {
56
+ nullOr(column, () -> builder.setTimestamp(column, reader.getTimestamp(column)));
57
+ }
58
+
59
+ @Override
60
+ public void jsonColumn(Column column)
61
+ {
62
+ nullOr(column, () -> builder.setJson(column, reader.getJson(column)));
63
+ }
64
+ }
@@ -0,0 +1,165 @@
1
+ package org.embulk.filter.copy.util;
2
+
3
+ import org.slf4j.Logger;
4
+
5
+ import java.util.function.Supplier;
6
+
7
+ public class ElapsedTime
8
+ {
9
+ private ElapsedTime()
10
+ {
11
+ }
12
+
13
+ private final static String TEMPLATE = " ({} ms)";
14
+ private enum LogLevel {TRACE, DEBUG, INFO, WARN, ERROR}
15
+ private enum Status {START, RUNNING, FINISHED}
16
+
17
+ private static void log(Logger logger, LogLevel level, String message, long elapsed)
18
+ {
19
+ String msgWithTmpl = message + TEMPLATE;
20
+ switch (level)
21
+ {
22
+ case TRACE:
23
+ logger.trace(msgWithTmpl, elapsed);
24
+ break;
25
+ case DEBUG:
26
+ logger.debug(msgWithTmpl, elapsed);
27
+ break;
28
+ case INFO:
29
+ logger.info(msgWithTmpl, elapsed);
30
+ break;
31
+ case WARN:
32
+ logger.warn(msgWithTmpl, elapsed);
33
+ break;
34
+ case ERROR:
35
+ logger.error(msgWithTmpl, elapsed);
36
+ break;
37
+ }
38
+ }
39
+
40
+ private static <T>T run(Logger logger, LogLevel level, String message, Supplier<T> proc)
41
+ {
42
+ long start = System.currentTimeMillis();
43
+ try {
44
+ return proc.get();
45
+ }
46
+ finally {
47
+ log(logger, level, message, System.currentTimeMillis() - start);
48
+ }
49
+ }
50
+
51
+ private static void run(Logger logger, LogLevel level, String message, Runnable proc)
52
+ {
53
+ long start = System.currentTimeMillis();
54
+ try {
55
+ proc.run();
56
+ }
57
+ finally {
58
+ log(logger, level, message, System.currentTimeMillis() - start);
59
+ }
60
+ }
61
+
62
+ public static <T>T trace(Logger logger, String message, Supplier<T> proc)
63
+ {
64
+ return run(logger, LogLevel.TRACE, message, proc);
65
+ }
66
+
67
+ public static <T>T debug(Logger logger, String message, Supplier<T> proc)
68
+ {
69
+ return run(logger, LogLevel.DEBUG, message, proc);
70
+ }
71
+
72
+ public static <T>T info(Logger logger, String message, Supplier<T> proc)
73
+ {
74
+ return run(logger, LogLevel.INFO, message, proc);
75
+ }
76
+
77
+ public static <T>T warn(Logger logger, String message, Supplier<T> proc)
78
+ {
79
+ return run(logger, LogLevel.WARN, message, proc);
80
+ }
81
+
82
+ public static <T>T error(Logger logger, String message, Supplier<T> proc)
83
+ {
84
+ return run(logger, LogLevel.ERROR, message, proc);
85
+ }
86
+
87
+ public static void trace(Logger logger, String message, Runnable proc)
88
+ {
89
+ run(logger, LogLevel.TRACE, message, proc);
90
+ }
91
+
92
+ public static void debug(Logger logger, String message, Runnable proc)
93
+ {
94
+ run(logger, LogLevel.DEBUG, message, proc);
95
+ }
96
+
97
+ public static void info(Logger logger, String message, Runnable proc)
98
+ {
99
+ run(logger, LogLevel.INFO, message, proc);
100
+ }
101
+
102
+ public static void warn(Logger logger, String message, Runnable proc)
103
+ {
104
+ run(logger, LogLevel.WARN, message, proc);
105
+ }
106
+
107
+ public static void error(Logger logger, String message, Runnable proc)
108
+ {
109
+ run(logger, LogLevel.ERROR, message, proc);
110
+ }
111
+
112
+ private static void logUntil(Supplier<Boolean> condition,
113
+ Logger logger, LogLevel level, String message, long sleep)
114
+ {
115
+ long start = System.currentTimeMillis();
116
+ log(logger, level, String.format("(%s) %s", Status.START, message),
117
+ System.currentTimeMillis() - start);
118
+ while (true) {
119
+ if (condition.get()) {
120
+ log(logger, level, String.format("(%s) %s", Status.FINISHED, message),
121
+ System.currentTimeMillis() - start);
122
+ break;
123
+ }
124
+ try {
125
+ Thread.sleep(sleep);
126
+ }
127
+ catch (InterruptedException e) {
128
+ logger.warn(e.getMessage(), e);
129
+ }
130
+
131
+ log(logger, level, String.format("(%s) %s", Status.RUNNING, message),
132
+ System.currentTimeMillis() - start);
133
+ }
134
+ }
135
+
136
+ public static void traceUntil(Supplier<Boolean> condition,
137
+ Logger logger, String message, long sleep)
138
+ {
139
+ logUntil(condition, logger, LogLevel.TRACE, message, sleep);
140
+ }
141
+
142
+ public static void debugUntil(Supplier<Boolean> condition,
143
+ Logger logger, String message, long sleep)
144
+ {
145
+ logUntil(condition, logger, LogLevel.DEBUG, message, sleep);
146
+ }
147
+
148
+ public static void infoUntil(Supplier<Boolean> condition,
149
+ Logger logger, String message, long sleep)
150
+ {
151
+ logUntil(condition, logger, LogLevel.INFO, message, sleep);
152
+ }
153
+
154
+ public static void warnUntil(Supplier<Boolean> condition,
155
+ Logger logger, String message, long sleep)
156
+ {
157
+ logUntil(condition, logger, LogLevel.WARN, message, sleep);
158
+ }
159
+
160
+ public static void errorUntil(Supplier<Boolean> condition,
161
+ Logger logger, String message, long sleep)
162
+ {
163
+ logUntil(condition, logger, LogLevel.ERROR, message, sleep);
164
+ }
165
+ }
@@ -0,0 +1,5 @@
1
+ package org.embulk.filter.copy;
2
+
3
+ public class TestCopyFilterPlugin
4
+ {
5
+ }
@@ -0,0 +1,5 @@
1
+ package org.embulk.filter.copy.plugin;
2
+
3
+ public class TestInternalForwardInputPlugin
4
+ {
5
+ }
metadata ADDED
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-filter-copy
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Civitaspo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-06-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ~>
17
+ - !ruby/object:Gem::Version
18
+ version: '1.0'
19
+ name: bundler
20
+ prerelease: false
21
+ type: :development
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '10.0'
33
+ name: rake
34
+ prerelease: false
35
+ type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ description: Copy records and run another embulk by using them as input data source.
42
+ email:
43
+ - civitaspo@gmail.com
44
+ executables: []
45
+ extensions: []
46
+ extra_rdoc_files: []
47
+ files:
48
+ - .gitignore
49
+ - CHANGELOG.md
50
+ - LICENSE.txt
51
+ - README.md
52
+ - build.gradle
53
+ - config/checkstyle/checkstyle.xml
54
+ - config/checkstyle/default.xml
55
+ - example/config.yml
56
+ - example/data.tsv
57
+ - gradle/wrapper/gradle-wrapper.jar
58
+ - gradle/wrapper/gradle-wrapper.properties
59
+ - gradlew
60
+ - gradlew.bat
61
+ - lib/embulk/filter/copy.rb
62
+ - settings.gradle
63
+ - src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java
64
+ - src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java
65
+ - src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java
66
+ - src/main/java/org/embulk/filter/copy/forward/InForwardService.java
67
+ - src/main/java/org/embulk/filter/copy/forward/InForwardVisitor.java
68
+ - src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java
69
+ - src/main/java/org/embulk/filter/copy/forward/OutForwardService.java
70
+ - src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java
71
+ - src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java
72
+ - src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java
73
+ - src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java
74
+ - src/main/java/org/embulk/filter/copy/util/ElapsedTime.java
75
+ - src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java
76
+ - src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java
77
+ - classpath/embulk-filter-copy-0.0.1.jar
78
+ - classpath/fluency-1.1.0.jar
79
+ - classpath/influent-java-0.2.0.jar
80
+ - classpath/influent-transport-0.2.0.jar
81
+ - classpath/jackson-annotations-2.7.0.jar
82
+ - classpath/jackson-core-2.7.1.jar
83
+ - classpath/jackson-databind-2.7.1.jar
84
+ - classpath/jackson-dataformat-msgpack-0.8.11.jar
85
+ - classpath/phi-accural-failure-detector-0.0.4.jar
86
+ - classpath/slf4j-api-1.7.22.jar
87
+ homepage: https://github.com/civitaspo/embulk-filter-copy
88
+ licenses:
89
+ - MIT
90
+ metadata: {}
91
+ post_install_message:
92
+ rdoc_options: []
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - '>='
98
+ - !ruby/object:Gem::Version
99
+ version: '0'
100
+ required_rubygems_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - '>='
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ requirements: []
106
+ rubyforge_project:
107
+ rubygems_version: 2.1.9
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: Copy filter plugin for Embulk
111
+ test_files: []