embulk-filter-copy 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3044893452487bfd8e401d210931f2cd8c95b750
4
- data.tar.gz: bcd788f6fcbf6f5e50c1867bbb85869fbe7781b1
3
+ metadata.gz: d2500f6ad78f1db21f25444a8454312d190d3229
4
+ data.tar.gz: 6b3f59c7329235aeaff13ee152a235617b282ab1
5
5
  SHA512:
6
- metadata.gz: e7e68823705f1747c65bae165c6a573e5f3be6befb1b2ff03fedb95bac628350250405915445a247edb98cd6388dd3d73b57e631b45454bf8bdc44823c31d323
7
- data.tar.gz: 0c16332e95084910ffe7f128abf68cac89e67c05151bf1bbbe86be666ab43e3ecb57e18c0e6725d17c143cf6a368796d033344d6e70d1ad3d250f8323b9e3099
6
+ metadata.gz: 6498b12ec28031b054e31cd14a35a37540942a23fcd6b0dd95149609afa0797a201badadc0ccddecb1baf49dfdfbfbd281dc5da9fa95d875bee69e25be0c90b8
7
+ data.tar.gz: 0f1552016b5e7417a69a868a73820c26451c7f7e2b13fb0dc773d031be8c70e4fbb660de74fcf8bf32fed9bf28c8adb8f2567379da6a47615284ad94f8db1a22
@@ -1,3 +1,8 @@
1
+ 0.0.2 (2017-07-01)
2
+ ==================
3
+ * Write a default values test.
4
+ * Change EmbulkExecutorService to EmbulkExecutor/LocalThreadExecutor for #6
5
+
1
6
  0.0.1 (2017-06-07)
2
7
  ==================
3
- - first release.
8
+ * first release.
data/README.md CHANGED
@@ -34,6 +34,8 @@ filters:
34
34
  - This plugin has no test yet, so may have some bugs.
35
35
  - This plugin does not work on [embulk-executor-mapreduce](https://github.com/embulk/embulk-executor-mapreduce) yet.
36
36
  - This plugin uses lots of memory now, because embulk run twice.
37
+ - Specification about error handling is not yet fixed.
38
+ - If you have any problems or opinions, I'm glad if you raise Issue.
37
39
 
38
40
  ## Dependencies
39
41
  - https://github.com/okumin/influent
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.0.1"
17
+ version = "0.0.2"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
@@ -27,7 +27,9 @@ dependencies {
27
27
  // This plugin use 1.1.0 for using the same msgpack version as embulk.
28
28
  // ref. https://github.com/komamitsu/fluency/compare/1.1.0...1.3.0
29
29
  compile "org.komamitsu:fluency:1.1.0"
30
+
30
31
  testCompile "junit:junit:4.+"
32
+ testCompile "org.embulk:embulk-core:0.8.23:tests"
31
33
  }
32
34
 
33
35
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -5,13 +5,13 @@ import org.embulk.config.ConfigDefault;
5
5
  import org.embulk.config.ConfigSource;
6
6
  import org.embulk.config.Task;
7
7
  import org.embulk.config.TaskSource;
8
+ import org.embulk.filter.copy.executor.EmbulkExecutor;
8
9
  import org.embulk.filter.copy.forward.InForwardService;
9
10
  import org.embulk.filter.copy.forward.OutForwardEventBuilder;
10
11
  import org.embulk.filter.copy.forward.OutForwardService;
11
12
  import org.embulk.filter.copy.forward.OutForwardVisitor;
12
13
  import org.embulk.filter.copy.plugin.InternalForwardInputPlugin;
13
- import org.embulk.filter.copy.service.EmbulkExecutorService;
14
- import org.embulk.filter.copy.service.StandardColumnVisitor;
14
+ import org.embulk.filter.copy.util.StandardColumnVisitor;
15
15
  import org.embulk.spi.Exec;
16
16
  import org.embulk.spi.FilterPlugin;
17
17
  import org.embulk.spi.Page;
@@ -45,7 +45,7 @@ public class CopyFilterPlugin
45
45
  }
46
46
 
47
47
  public interface PluginTask
48
- extends Task, TimestampFormatter.Task,
48
+ extends Task, EmbulkExecutor.Task, TimestampFormatter.Task,
49
49
  OutForwardService.Task, InForwardService.Task
50
50
  {
51
51
  @Config("config")
@@ -72,15 +72,15 @@ public class CopyFilterPlugin
72
72
  return embulkRunConfig;
73
73
  }
74
74
 
75
- private void withEmbulkRun(ConfigSource config, Runnable r)
75
+ private void withEmbulkRun(EmbulkExecutor executor, ConfigSource config, Runnable r)
76
76
  {
77
- EmbulkExecutorService embulkExecutorService = new EmbulkExecutorService(Exec.getInjector());
78
- embulkExecutorService.executeAsync(config);
77
+ executor.setup();
78
+ executor.executeAsync(config);
79
79
 
80
80
  r.run();
81
81
 
82
- embulkExecutorService.waitExecutionFinished();
83
- embulkExecutorService.shutdown();
82
+ executor.waitUntilExecutionFinished();
83
+ executor.shutdown();
84
84
  }
85
85
 
86
86
  @Override
@@ -89,11 +89,11 @@ public class CopyFilterPlugin
89
89
  {
90
90
  PluginTask task = config.loadConfig(PluginTask.class);
91
91
  ConfigSource embulkRunConfig = configure(task, inputSchema);
92
+ EmbulkExecutor embulkExecutor = EmbulkExecutor.buildExecutor(task);
92
93
 
93
- withEmbulkRun(embulkRunConfig, () ->
94
- {
95
- Schema outputSchema = inputSchema;
96
- control.run(task.dump(), outputSchema);
94
+ withEmbulkRun(embulkExecutor, embulkRunConfig, () -> {
95
+ Schema outputSchema1 = inputSchema;
96
+ control.run(task.dump(), outputSchema1);
97
97
  OutForwardService.sendShutdownMessage(task);
98
98
  });
99
99
  }
@@ -0,0 +1,105 @@
1
+ package org.embulk.filter.copy.executor;
2
+
3
+ import com.fasterxml.jackson.annotation.JsonCreator;
4
+ import com.fasterxml.jackson.annotation.JsonValue;
5
+ import com.google.common.base.Joiner;
6
+ import com.google.inject.Injector;
7
+ import org.embulk.EmbulkEmbed;
8
+ import org.embulk.config.Config;
9
+ import org.embulk.config.ConfigDefault;
10
+ import org.embulk.config.ConfigException;
11
+ import org.embulk.config.ConfigSource;
12
+ import org.embulk.guice.LifeCycleInjector;
13
+ import org.embulk.spi.Exec;
14
+ import org.slf4j.Logger;
15
+
16
+ import java.lang.reflect.Constructor;
17
+ import java.lang.reflect.InvocationTargetException;
18
+ import java.util.Locale;
19
+
20
+ public abstract class EmbulkExecutor
21
+ {
22
+ static final Logger logger = Exec.getLogger(EmbulkExecutor.class);
23
+
24
+ public interface ExecutorTask
25
+ extends org.embulk.config.Task
26
+ {
27
+ @Config("\"type\"")
28
+ @ConfigDefault("\"local_thread\"")
29
+ ExecutorType getType();
30
+
31
+ enum ExecutorType
32
+ {
33
+ LOCAL_THREAD;
34
+
35
+ @JsonValue
36
+ @Override
37
+ public String toString()
38
+ {
39
+ return name().toLowerCase(Locale.ENGLISH);
40
+ }
41
+
42
+ @JsonCreator
43
+ public static ExecutorType fromString(String value)
44
+ {
45
+ switch (value) {
46
+ case "local_thread":
47
+ return LOCAL_THREAD;
48
+ default:
49
+ String values = Joiner.on(", ").join(values()).toLowerCase(Locale.ENGLISH);
50
+ throw new ConfigException(String.format("Unknown executor type '%s'. Supported executor types are %s", value, values));
51
+ }
52
+ }
53
+ }
54
+
55
+ }
56
+
57
+ public interface Task
58
+ extends org.embulk.config.Task
59
+ {
60
+ @Config("executor")
61
+ @ConfigDefault("{}")
62
+ ExecutorTask getExecutorTask();
63
+ }
64
+
65
+ public static EmbulkExecutor buildExecutor(Task task)
66
+ {
67
+ switch (task.getExecutorTask().getType()) {
68
+ case LOCAL_THREAD:
69
+ return new LocalThreadExecutor(task.getExecutorTask());
70
+ default:
71
+ String values = Joiner.on(", ").join(ExecutorTask.ExecutorType.values()).toLowerCase(Locale.ENGLISH);
72
+ throw new ConfigException(String.format("Unknown executor type '%s'. Supported executor types are %s", task.getExecutorTask().getType(), values));
73
+ }
74
+ }
75
+
76
+ final ExecutorTask task;
77
+ final Injector injector;
78
+
79
+ EmbulkExecutor(ExecutorTask task)
80
+ {
81
+ this.task = task;
82
+ this.injector = Exec.getInjector();
83
+ }
84
+
85
+ public abstract void setup();
86
+
87
+ public abstract void executeAsync(ConfigSource config);
88
+
89
+ public abstract void waitUntilExecutionFinished();
90
+
91
+ public abstract void shutdown();
92
+
93
+ EmbulkEmbed newEmbulkEmbed()
94
+ {
95
+ try {
96
+ Constructor<EmbulkEmbed> constructor = EmbulkEmbed.class
97
+ .getDeclaredConstructor(ConfigSource.class, LifeCycleInjector.class);
98
+ constructor.setAccessible(true);
99
+ return constructor.newInstance(null, injector);
100
+ }
101
+ catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
102
+ throw new ConfigException(e);
103
+ }
104
+ }
105
+ }
@@ -1,39 +1,30 @@
1
- package org.embulk.filter.copy.service;
1
+ package org.embulk.filter.copy.executor;
2
2
 
3
3
  import com.google.common.util.concurrent.FutureCallback;
4
4
  import com.google.common.util.concurrent.Futures;
5
5
  import com.google.common.util.concurrent.ListenableFuture;
6
6
  import com.google.common.util.concurrent.ListeningExecutorService;
7
7
  import com.google.common.util.concurrent.MoreExecutors;
8
- import com.google.inject.Injector;
9
- import org.embulk.EmbulkEmbed;
10
- import org.embulk.config.ConfigException;
11
8
  import org.embulk.config.ConfigSource;
12
9
  import org.embulk.exec.ExecutionResult;
13
10
  import org.embulk.filter.copy.util.ElapsedTime;
14
- import org.embulk.guice.LifeCycleInjector;
15
- import org.embulk.spi.Exec;
16
- import org.slf4j.Logger;
17
11
 
18
12
  import javax.annotation.Nullable;
19
13
 
20
- import java.lang.reflect.Constructor;
21
- import java.lang.reflect.InvocationTargetException;
22
14
  import java.util.concurrent.Callable;
23
15
  import java.util.concurrent.Executors;
24
16
 
25
- public class EmbulkExecutorService
17
+ public class LocalThreadExecutor
18
+ extends EmbulkExecutor
26
19
  {
27
- private final static String THREAD_NAME = "embulk executor service";
20
+ private static final String THREAD_NAME = "embulk executor service";
28
21
  private static final int NUM_THREADS = 1;
29
- private final static Logger logger = Exec.getLogger(EmbulkExecutorService.class);
30
- private final Injector injector;
31
22
  private final ListeningExecutorService es;
32
23
  private ListenableFuture<ExecutionResult> future;
33
24
 
34
- public EmbulkExecutorService(Injector injector)
25
+ LocalThreadExecutor(ExecutorTask task)
35
26
  {
36
- this.injector = injector;
27
+ super(task);
37
28
  this.es = MoreExecutors.listeningDecorator(
38
29
  Executors.newFixedThreadPool(
39
30
  NUM_THREADS,
@@ -41,6 +32,12 @@ public class EmbulkExecutorService
41
32
  ));
42
33
  }
43
34
 
35
+ @Override
36
+ public void setup()
37
+ {
38
+ }
39
+
40
+ @Override
44
41
  public void executeAsync(ConfigSource config)
45
42
  {
46
43
  logger.debug("execute with this config: {}", config);
@@ -51,40 +48,28 @@ public class EmbulkExecutorService
51
48
  Futures.addCallback(future, resultFutureCallback());
52
49
  }
53
50
 
54
- public void shutdown()
55
- {
56
- ElapsedTime.info(
57
- logger,
58
- "embulk executor service shutdown",
59
- es::shutdown);
60
- }
61
-
62
- public void waitExecutionFinished()
51
+ @Override
52
+ public void waitUntilExecutionFinished()
63
53
  {
64
54
  if (future == null) {
65
55
  throw new NullPointerException();
66
56
  }
67
-
68
57
  ElapsedTime.debugUntil(() -> future.isDone() || future.isCancelled(),
69
58
  logger, "embulk executor", 3000L);
70
59
  }
71
60
 
72
- private Callable<ExecutionResult> embulkRun(ConfigSource config)
61
+ @Override
62
+ public void shutdown()
73
63
  {
74
- return () -> newEmbulkEmbed(injector).run(config);
64
+ ElapsedTime.info(
65
+ logger,
66
+ "embulk executor service shutdown",
67
+ es::shutdown);
75
68
  }
76
69
 
77
- private EmbulkEmbed newEmbulkEmbed(Injector injector)
70
+ private Callable<ExecutionResult> embulkRun(ConfigSource config)
78
71
  {
79
- try {
80
- Constructor<EmbulkEmbed> constructor = EmbulkEmbed.class
81
- .getDeclaredConstructor(ConfigSource.class, LifeCycleInjector.class);
82
- constructor.setAccessible(true);
83
- return constructor.newInstance(Exec.newConfigSource(), injector);
84
- }
85
- catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
86
- throw new ConfigException(e);
87
- }
72
+ return () -> newEmbulkEmbed().run(config);
88
73
  }
89
74
 
90
75
  private FutureCallback<ExecutionResult> resultFutureCallback()
@@ -1,4 +1,4 @@
1
- package org.embulk.filter.copy.service;
1
+ package org.embulk.filter.copy.util;
2
2
 
3
3
  import org.embulk.spi.Column;
4
4
  import org.embulk.spi.ColumnVisitor;
@@ -1,5 +1,113 @@
1
1
  package org.embulk.filter.copy;
2
2
 
3
+ import com.google.common.collect.Lists;
4
+ import org.embulk.EmbulkTestRuntime;
5
+ import org.embulk.config.ConfigLoader;
6
+ import org.embulk.config.ConfigSource;
7
+ import org.embulk.filter.copy.executor.EmbulkExecutor.ExecutorTask;
8
+ import org.embulk.spi.Exec;
9
+ import org.embulk.spi.Schema;
10
+ import org.embulk.spi.type.Type;
11
+ import org.embulk.spi.type.Types;
12
+ import org.joda.time.DateTimeZone;
13
+ import org.junit.Before;
14
+ import org.junit.Rule;
15
+ import org.junit.Test;
16
+ import org.junit.rules.ExpectedException;
17
+
18
+ import static com.google.common.base.Optional.absent;
19
+ import static org.embulk.filter.copy.CopyFilterPlugin.EmbulkConfig;
20
+ import static org.embulk.filter.copy.CopyFilterPlugin.PluginTask;
21
+ import static org.embulk.filter.copy.executor.EmbulkExecutor.ExecutorTask.ExecutorType.LOCAL_THREAD;
22
+ import static org.embulk.filter.copy.forward.InForwardService.InForwardTask;
23
+ import static org.embulk.filter.copy.forward.OutForwardService.OutForwardTask;
24
+ import static org.junit.Assert.assertEquals;
25
+
3
26
  public class TestCopyFilterPlugin
4
27
  {
28
+ @Rule
29
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
30
+
31
+ @Rule
32
+ public ExpectedException exception = ExpectedException.none();
33
+
34
+ private Schema schema;
35
+ private CopyFilterPlugin plugin;
36
+
37
+ private static Schema schema(Object... nameAndTypes)
38
+ {
39
+ Schema.Builder builder = Schema.builder();
40
+ for (int i = 0; i < nameAndTypes.length; i += 2) {
41
+ builder.add((String) nameAndTypes[i], (Type) nameAndTypes[i + 1]);
42
+ }
43
+ return builder.build();
44
+ }
45
+
46
+ private static ConfigSource fromYaml(String yaml)
47
+ {
48
+ ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
49
+ return loader.fromYamlString(yaml);
50
+ }
51
+
52
+ @Before
53
+ public void createResources()
54
+ {
55
+ schema = schema("_c0", Types.STRING, "_c1", Types.STRING); // default schema
56
+ }
57
+
58
+ @Test
59
+ public void testPluginTaskDefault()
60
+ {
61
+ String yaml = String.join("\n",
62
+ "type: copy",
63
+ "config:",
64
+ " out:",
65
+ " type: stdout"
66
+ );
67
+
68
+ ConfigSource config = fromYaml(yaml);
69
+ PluginTask task = config.loadConfig(PluginTask.class);
70
+
71
+ // EmbulkConfig
72
+ EmbulkConfig embulkConfig = task.getConfig();
73
+ assertEquals(Exec.newConfigSource(), embulkConfig.getExecConfig());
74
+ assertEquals(Lists.<ConfigSource>newArrayList(), embulkConfig.getFilterConfig());
75
+
76
+ // EmbulkExecutor
77
+ ExecutorTask executorTask = task.getExecutorTask();
78
+ assertEquals(LOCAL_THREAD, executorTask.getType());
79
+
80
+ // TimestampFormatter
81
+ assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
82
+ assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
83
+
84
+ // ForwardBaseTask
85
+ assertEquals("message", task.getMessageTag());
86
+ assertEquals("shutdown", task.getShutdownTag());
87
+
88
+ // InForwardService
89
+ InForwardTask inForward = task.getInForwardTask();
90
+ assertEquals(24224, inForward.getPort());
91
+ assertEquals(absent(), inForward.getChunkSizeLimit());
92
+ assertEquals(absent(), inForward.getBacklog());
93
+ assertEquals(absent(), inForward.getSendBufferSize());
94
+ assertEquals(absent(), inForward.getReceiveBufferSize());
95
+ assertEquals(absent(), inForward.getKeepAliveEnabled());
96
+ assertEquals(absent(), inForward.getTcpNoDelayEnabled());
97
+
98
+ // OutForwardService
99
+ OutForwardTask outForward = task.getOutForwardTask();
100
+ assertEquals("localhost", outForward.getHost());
101
+ assertEquals(24224, outForward.getPort());
102
+ assertEquals(absent(), outForward.getMaxBufferSize());
103
+ assertEquals(absent(), outForward.getBufferChunkInitialSize());
104
+ assertEquals(absent(), outForward.getBufferChunkRetentionSize());
105
+ assertEquals(absent(), outForward.getFlushIntervalMillis());
106
+ assertEquals(absent(), outForward.getSenderMaxRetryCount());
107
+ assertEquals(absent(), outForward.getAckResponseMode());
108
+ assertEquals(absent(), outForward.getFileBackupDir());
109
+ assertEquals(absent(), outForward.getWaitUntilBufferFlushed());
110
+ assertEquals(absent(), outForward.getWaitUntilFlusherTerminated());
111
+ }
112
+
5
113
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-copy
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-06-06 00:00:00.000000000 Z
11
+ date: 2017-07-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -61,6 +61,8 @@ files:
61
61
  - lib/embulk/filter/copy.rb
62
62
  - settings.gradle
63
63
  - src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java
64
+ - src/main/java/org/embulk/filter/copy/executor/EmbulkExecutor.java
65
+ - src/main/java/org/embulk/filter/copy/executor/LocalThreadExecutor.java
64
66
  - src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java
65
67
  - src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java
66
68
  - src/main/java/org/embulk/filter/copy/forward/InForwardService.java
@@ -69,12 +71,11 @@ files:
69
71
  - src/main/java/org/embulk/filter/copy/forward/OutForwardService.java
70
72
  - src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java
71
73
  - src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java
72
- - src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java
73
- - src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java
74
74
  - src/main/java/org/embulk/filter/copy/util/ElapsedTime.java
75
+ - src/main/java/org/embulk/filter/copy/util/StandardColumnVisitor.java
75
76
  - src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java
76
77
  - src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java
77
- - classpath/embulk-filter-copy-0.0.1.jar
78
+ - classpath/embulk-filter-copy-0.0.2.jar
78
79
  - classpath/fluency-1.1.0.jar
79
80
  - classpath/influent-java-0.2.0.jar
80
81
  - classpath/influent-transport-0.2.0.jar