embulk-filter-copy 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/CHANGELOG.md +3 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +67 -0
  6. data/build.gradle +104 -0
  7. data/config/checkstyle/checkstyle.xml +128 -0
  8. data/config/checkstyle/default.xml +108 -0
  9. data/example/config.yml +28 -0
  10. data/example/data.tsv +5 -0
  11. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  12. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  13. data/gradlew +169 -0
  14. data/gradlew.bat +84 -0
  15. data/lib/embulk/filter/copy.rb +8 -0
  16. data/settings.gradle +1 -0
  17. data/src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java +149 -0
  18. data/src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java +17 -0
  19. data/src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java +147 -0
  20. data/src/main/java/org/embulk/filter/copy/forward/InForwardService.java +187 -0
  21. data/src/main/java/org/embulk/filter/copy/forward/InForwardVisitor.java +63 -0
  22. data/src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java +135 -0
  23. data/src/main/java/org/embulk/filter/copy/forward/OutForwardService.java +170 -0
  24. data/src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java +63 -0
  25. data/src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java +111 -0
  26. data/src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java +111 -0
  27. data/src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java +64 -0
  28. data/src/main/java/org/embulk/filter/copy/util/ElapsedTime.java +165 -0
  29. data/src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java +5 -0
  30. data/src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java +5 -0
  31. metadata +111 -0
@@ -0,0 +1,17 @@
1
+ package org.embulk.filter.copy.forward;
2
+
3
+ import org.embulk.config.Config;
4
+ import org.embulk.config.ConfigDefault;
5
+ import org.embulk.config.Task;
6
+
7
+ public interface ForwardBaseTask
8
+ extends Task
9
+ {
10
+ @Config("shutdown_tag")
11
+ @ConfigDefault("\"shutdown\"")
12
+ String getShutdownTag();
13
+
14
+ @Config("message_tag")
15
+ @ConfigDefault("\"message\"")
16
+ String getMessageTag();
17
+ }
@@ -0,0 +1,147 @@
1
+ package org.embulk.filter.copy.forward;
2
+
3
+ import com.google.common.collect.ImmutableMap;
4
+ import com.google.common.collect.Maps;
5
+ import influent.EventStream;
6
+ import org.embulk.spi.Column;
7
+ import org.embulk.spi.Schema;
8
+ import org.embulk.spi.json.JsonParser;
9
+ import org.embulk.spi.time.Timestamp;
10
+ import org.embulk.spi.time.TimestampParser;
11
+ import org.msgpack.value.Value;
12
+
13
+ import java.util.Map;
14
+
15
+ public class InForwardEventReader
16
+ {
17
+ private final Schema schema;
18
+ private final Map<String, Column> columnMap;
19
+
20
+ private final JsonParser jsonParser = new JsonParser();
21
+ private final TimestampParser timestampParser;
22
+
23
+ private EventStream event = null;
24
+ private int eventMessageCount = 0;
25
+
26
+ private int readCount = 0;
27
+ private Map<String, Value> message = Maps.newHashMap();
28
+
29
+ public InForwardEventReader(Schema schema, TimestampParser timestampParser)
30
+ {
31
+ this.schema = schema;
32
+ ImmutableMap.Builder<String, Column> builder = ImmutableMap.builder();
33
+ schema.getColumns().forEach(column -> builder.put(column.getName(), column));
34
+ this.columnMap = builder.build();
35
+ this.timestampParser = timestampParser;
36
+ }
37
+
38
+ public Schema getSchema()
39
+ {
40
+ return schema;
41
+ }
42
+
43
+ private Column getColumn(String columnName)
44
+ {
45
+ return columnMap.get(columnName);
46
+ }
47
+
48
+ private Column getColumn(int columnIndex)
49
+ {
50
+ return getSchema().getColumn(columnIndex);
51
+ }
52
+
53
+ private Value getValue(Column column)
54
+ {
55
+ return message.get(column.getName());
56
+ }
57
+
58
+ public void setEvent(EventStream event)
59
+ {
60
+ this.event = event;
61
+ this.eventMessageCount = event.getEntries().size();
62
+ }
63
+
64
+ public boolean isNull(Column column)
65
+ {
66
+ return getValue(column).isNilValue();
67
+ }
68
+
69
+ public boolean isNull(int columnIndex)
70
+ {
71
+ return isNull(getColumn(columnIndex));
72
+ }
73
+
74
+ public boolean getBoolean(Column column)
75
+ {
76
+ return getValue(column).asBooleanValue().getBoolean();
77
+ }
78
+
79
+ public boolean getBoolean(int columnIndex)
80
+ {
81
+ return getBoolean(getColumn(columnIndex));
82
+ }
83
+
84
+ public long getLong(Column column)
85
+ {
86
+ return getValue(column).asIntegerValue().asLong();
87
+ }
88
+
89
+ public long getLong(int columnIndex)
90
+ {
91
+ return getLong(getColumn(columnIndex));
92
+ }
93
+
94
+ public double getDouble(Column column)
95
+ {
96
+ return getValue(column).asFloatValue().toDouble();
97
+ }
98
+
99
+ public double genDouble(int columnIndex)
100
+ {
101
+ return getDouble(getColumn(columnIndex));
102
+ }
103
+
104
+ public String getString(Column column)
105
+ {
106
+ return getValue(column).toString();
107
+ }
108
+
109
+ public String getString(int columnIndex)
110
+ {
111
+ return getString(getColumn(columnIndex));
112
+ }
113
+
114
+ public Timestamp getTimestamp(Column column)
115
+ {
116
+ return timestampParser.parse(getString(column));
117
+ }
118
+
119
+ public Timestamp getTimestamp(int columnIndex)
120
+ {
121
+ return getTimestamp(getColumn(columnIndex));
122
+ }
123
+
124
+ public Value getJson(Column column)
125
+ {
126
+ return jsonParser.parse(getString(column));
127
+ }
128
+
129
+ public Value getJson(int columnIndex)
130
+ {
131
+ return getJson(getColumn(columnIndex));
132
+ }
133
+
134
+ public boolean nextMessage()
135
+ {
136
+ if (eventMessageCount <= readCount) {
137
+ return false;
138
+ }
139
+
140
+ ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
141
+ event.getEntries().get(readCount++).getRecord().entrySet().forEach(
142
+ entry -> builder.put(entry.getKey().toString(), entry.getValue())
143
+ );
144
+ message = builder.build();
145
+ return true;
146
+ }
147
+ }
@@ -0,0 +1,187 @@
1
+ package org.embulk.filter.copy.forward;
2
+
3
+ import com.google.common.base.Optional;
4
+ import influent.EventStream;
5
+ import influent.Tag;
6
+ import influent.forward.ForwardCallback;
7
+ import influent.forward.ForwardServer;
8
+ import org.embulk.config.Config;
9
+ import org.embulk.config.ConfigDefault;
10
+ import org.embulk.filter.copy.util.ElapsedTime;
11
+ import org.embulk.spi.Exec;
12
+ import org.slf4j.Logger;
13
+
14
+ import java.util.concurrent.CompletableFuture;
15
+ import java.util.concurrent.Executor;
16
+ import java.util.concurrent.ExecutorService;
17
+ import java.util.concurrent.Executors;
18
+ import java.util.concurrent.atomic.AtomicBoolean;
19
+ import java.util.function.Consumer;
20
+
21
+ public class InForwardService
22
+ {
23
+ private final static Logger logger = Exec.getLogger(InForwardService.class);
24
+
25
+ public interface InForwardTask
26
+ extends org.embulk.config.Task
27
+ {
28
+ @Config("port")
29
+ @ConfigDefault("24224")
30
+ int getPort();
31
+
32
+ @Config("chunk_size_limit")
33
+ @ConfigDefault("null")
34
+ Optional<Long> getChunkSizeLimit();
35
+
36
+ @Config("backlog")
37
+ @ConfigDefault("null")
38
+ Optional<Integer> getBacklog();
39
+
40
+ @Config("send_buffer_size")
41
+ @ConfigDefault("null")
42
+ Optional<Integer> getSendBufferSize();
43
+
44
+ @Config("receive_buffer_size")
45
+ @ConfigDefault("null")
46
+ Optional<Integer> getReceiveBufferSize();
47
+
48
+ @Config("keep_alive_enabled")
49
+ @ConfigDefault("null")
50
+ Optional<Boolean> getKeepAliveEnabled();
51
+
52
+ @Config("tcp_no_delay_enabled")
53
+ @ConfigDefault("null")
54
+ Optional<Boolean> getTcpNoDelayEnabled();
55
+ }
56
+
57
+ public interface Task
58
+ extends ForwardBaseTask
59
+ {
60
+ @Config("in_forward")
61
+ @ConfigDefault("{}")
62
+ InForwardTask getInForwardTask();
63
+ }
64
+
65
+ public static class Builder
66
+ {
67
+ private Task task;
68
+ private Consumer<EventStream> eventConsumer;
69
+
70
+ public Builder()
71
+ {
72
+ }
73
+
74
+ public Builder task(Task task)
75
+ {
76
+ this.task = task;
77
+ return this;
78
+ }
79
+
80
+ public Builder forEachEventCallback(Consumer<EventStream> eventConsumer)
81
+ {
82
+ this.eventConsumer = eventConsumer;
83
+ return this;
84
+ }
85
+
86
+ public InForwardService build()
87
+ {
88
+ return new InForwardService(task, eventConsumer);
89
+ }
90
+ }
91
+
92
+ public static Builder builder()
93
+ {
94
+ return new Builder();
95
+ }
96
+
97
+ // TODO: configure?
98
+ private static final String THREAD_NAME = "in_forward";
99
+ private static final int NUM_THREADS = 1;
100
+
101
+ private final Task task;
102
+ private final ForwardServer server;
103
+ private final ExecutorService callbackThread;
104
+ private final AtomicBoolean shouldShutdown = new AtomicBoolean(false);
105
+
106
+ private InForwardService(Task task, Consumer<EventStream> eventConsumer)
107
+ {
108
+ this.task = task;
109
+ this.callbackThread = Executors.newFixedThreadPool(
110
+ NUM_THREADS,
111
+ r -> new Thread(r, THREAD_NAME));
112
+ this.server = buildServer(task.getInForwardTask(), eventConsumer, callbackThread);
113
+ }
114
+
115
+ private ForwardServer buildServer(InForwardTask t, Consumer<EventStream> eventConsumer, Executor callbackThread)
116
+ {
117
+ ForwardServer.Builder builder = new ForwardServer.Builder(
118
+ ForwardCallback.ofSyncConsumer(
119
+ wrapEventConsumer(eventConsumer),
120
+ callbackThread));
121
+
122
+ builder.localAddress(t.getPort());
123
+
124
+ if (t.getChunkSizeLimit().isPresent()) {
125
+ builder.chunkSizeLimit(t.getChunkSizeLimit().get());
126
+ }
127
+ if (t.getBacklog().isPresent()) {
128
+ builder.backlog(t.getBacklog().get());
129
+ }
130
+ if (t.getSendBufferSize().isPresent()) {
131
+ builder.sendBufferSize(t.getSendBufferSize().get());
132
+ }
133
+ if (t.getReceiveBufferSize().isPresent()) {
134
+ builder.receiveBufferSize(t.getReceiveBufferSize().get());
135
+ }
136
+ if (t.getKeepAliveEnabled().isPresent()) {
137
+ builder.keepAliveEnabled(t.getKeepAliveEnabled().get());
138
+ }
139
+ if (t.getTcpNoDelayEnabled().isPresent()) {
140
+ builder.tcpNoDelayEnabled(t.getTcpNoDelayEnabled().get());
141
+ }
142
+
143
+ return builder.build();
144
+ }
145
+
146
+ private Consumer<EventStream> wrapEventConsumer(Consumer<EventStream> eventConsumer)
147
+ {
148
+ return eventStream ->
149
+ {
150
+ if (isShutdownTag(eventStream.getTag())) {
151
+ logger.info("Receive shutdown tag: {}", eventStream.getTag());
152
+ shouldShutdown.set(true);
153
+ }
154
+ else if (isMessageTag(eventStream.getTag())) {
155
+ eventConsumer.accept(eventStream);
156
+ }
157
+ else {
158
+ throw new RuntimeException(String.format("Unknown Tag received: %s", eventStream.getTag().getName()));
159
+ }
160
+ };
161
+ }
162
+
163
+ private boolean isShutdownTag(Tag tag)
164
+ {
165
+ return tag.getName().contentEquals(task.getShutdownTag());
166
+ }
167
+
168
+ private boolean isMessageTag(Tag tag)
169
+ {
170
+ return tag.getName().contentEquals(task.getMessageTag());
171
+ }
172
+
173
+ public void runUntilShouldShutdown()
174
+ {
175
+ server.start();
176
+ ElapsedTime.debugUntil(shouldShutdown::get, logger, "in_forward server", 1000L);
177
+ shutdown();
178
+ }
179
+
180
+ private void shutdown()
181
+ {
182
+ ElapsedTime.debug(logger, "shutdown in_forward callback", callbackThread::shutdown);
183
+ CompletableFuture<Void> shutdown = server.shutdown();
184
+ ElapsedTime.debugUntil(() -> shutdown.isCancelled() || shutdown.isCompletedExceptionally() || shutdown.isDone(),
185
+ logger, "shutdown in_forward server", 1000L);
186
+ }
187
+ }
@@ -0,0 +1,63 @@
1
+ package org.embulk.filter.copy.forward;
2
+
3
+ import org.embulk.spi.Column;
4
+ import org.embulk.spi.ColumnVisitor;
5
+ import org.embulk.spi.PageBuilder;
6
+
7
+ public class InForwardVisitor
8
+ implements ColumnVisitor
9
+ {
10
+ private final InForwardEventReader reader;
11
+ private final PageBuilder builder;
12
+
13
+ public InForwardVisitor(InForwardEventReader reader, PageBuilder builder)
14
+ {
15
+ this.reader = reader;
16
+ this.builder = builder;
17
+ }
18
+
19
+ private void nullOr(Column column, Runnable r)
20
+ {
21
+ if (reader.isNull(column)) {
22
+ builder.setNull(column);
23
+ return;
24
+ }
25
+ r.run();
26
+ }
27
+
28
+ @Override
29
+ public void booleanColumn(Column column)
30
+ {
31
+ nullOr(column, () -> builder.setBoolean(column, reader.getBoolean(column)));
32
+ }
33
+
34
+ @Override
35
+ public void longColumn(Column column)
36
+ {
37
+ nullOr(column, () -> builder.setLong(column, reader.getLong(column)));
38
+ }
39
+
40
+ @Override
41
+ public void doubleColumn(Column column)
42
+ {
43
+ nullOr(column, () -> builder.setDouble(column, reader.getDouble(column)));
44
+ }
45
+
46
+ @Override
47
+ public void stringColumn(Column column)
48
+ {
49
+ nullOr(column, () -> builder.setString(column, reader.getString(column)));
50
+ }
51
+
52
+ @Override
53
+ public void timestampColumn(Column column)
54
+ {
55
+ nullOr(column, () -> builder.setTimestamp(column, reader.getTimestamp(column)));
56
+ }
57
+
58
+ @Override
59
+ public void jsonColumn(Column column)
60
+ {
61
+ nullOr(column, () -> builder.setJson(column, reader.getJson(column)));
62
+ }
63
+ }
@@ -0,0 +1,135 @@
1
+ package org.embulk.filter.copy.forward;
2
+
3
+ import com.google.common.collect.Maps;
4
+ import org.embulk.spi.Column;
5
+ import org.embulk.spi.Schema;
6
+ import org.embulk.spi.time.Timestamp;
7
+ import org.embulk.spi.time.TimestampFormatter;
8
+ import org.msgpack.value.Value;
9
+
10
+ import java.util.Map;
11
+
12
+ public class OutForwardEventBuilder
13
+ {
14
+ private final Schema schema;
15
+ private final TimestampFormatter timestampFormatter;
16
+
17
+ private Map<String, Object> message;
18
+
19
+ public OutForwardEventBuilder(
20
+ Schema schema,
21
+ TimestampFormatter timestampFormatter)
22
+ {
23
+
24
+ this.schema = schema;
25
+ this.timestampFormatter = timestampFormatter;
26
+
27
+ setNewMessage();
28
+ }
29
+
30
+ private void setNewMessage()
31
+ {
32
+ this.message = Maps.newHashMap();
33
+ }
34
+
35
+ public void emitMessage(OutForwardService outForward)
36
+ {
37
+ outForward.emit(message);
38
+ setNewMessage();
39
+ }
40
+
41
+ public Schema getSchema()
42
+ {
43
+ return schema;
44
+ }
45
+
46
+ private Column getColumn(int columnIndex)
47
+ {
48
+ return getSchema().getColumn(columnIndex);
49
+ }
50
+
51
+ private void setValue(String columnName, Object value)
52
+ {
53
+ message.put(columnName, value);
54
+ }
55
+
56
+ private void setValue(Column column, Object value)
57
+ {
58
+ setValue(column.getName(), value);
59
+ }
60
+
61
+ public void setValue(int columnIndex, Object value)
62
+ {
63
+ setValue(getColumn(columnIndex), value);
64
+ }
65
+
66
+ public void setNull(Column column)
67
+ {
68
+ setValue(column, null);
69
+ }
70
+
71
+ private void setNull(int columnIndex)
72
+ {
73
+ setNull(getColumn(columnIndex));
74
+ }
75
+
76
+ public void setBoolean(Column column, boolean value)
77
+ {
78
+ setValue(column, value);
79
+ }
80
+
81
+ public void setBoolean(int columnIndex, boolean value)
82
+ {
83
+ setBoolean(getColumn(columnIndex), value);
84
+ }
85
+
86
+ public void setLong(Column column, long value)
87
+ {
88
+ setValue(column, value);
89
+ }
90
+
91
+ public void setLong(int columnIndex, long value)
92
+ {
93
+ setLong(getColumn(columnIndex), value);
94
+ }
95
+
96
+ public void setDouble(Column column, double value)
97
+ {
98
+ setValue(column, value);
99
+ }
100
+
101
+ public void setDouble(int columnIndex, double value)
102
+ {
103
+ setDouble(getColumn(columnIndex), value);
104
+ }
105
+
106
+ public void setString(Column column, String value)
107
+ {
108
+ setValue(column, value);
109
+ }
110
+
111
+ public void setString(int columnIndex, String value)
112
+ {
113
+ setString(getColumn(columnIndex), value);
114
+ }
115
+
116
+ public void setTimestamp(Column column, Timestamp value)
117
+ {
118
+ setValue(column, timestampFormatter.format(value));
119
+ }
120
+
121
+ public void setTimestamp(int columnIndex, Timestamp value)
122
+ {
123
+ setTimestamp(getColumn(columnIndex), value);
124
+ }
125
+
126
+ public void setJson(Column column, Value value)
127
+ {
128
+ setValue(column, value.toJson());
129
+ }
130
+
131
+ public void setJson(int columnIndex, Value value)
132
+ {
133
+ setJson(getColumn(columnIndex), value);
134
+ }
135
+ }