embulk 0.6.16 → 0.6.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +33 -45
  3. data/build.gradle +3 -3
  4. data/embulk-core/src/main/java/org/embulk/spi/Exec.java +6 -0
  5. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +73 -1
  6. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
  7. data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
  8. data/embulk-docs/src/_static/embulk-logo.svg +133 -0
  9. data/embulk-docs/src/release.rst +1 -0
  10. data/embulk-docs/src/release/release-0.6.17.rst +39 -0
  11. data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +2 -17
  12. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +3 -22
  13. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +2 -2
  14. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +0 -1
  15. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +18 -42
  16. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +3 -3
  17. data/lib/embulk/command/embulk_new_plugin.rb +40 -14
  18. data/lib/embulk/command/embulk_run.rb +3 -3
  19. data/lib/embulk/data/new/README.md.erb +18 -17
  20. data/lib/embulk/data/new/java/build.gradle.erb +1 -1
  21. data/lib/embulk/data/new/java/decoder.java.erb +53 -9
  22. data/lib/embulk/data/new/java/encoder.java.erb +54 -8
  23. data/lib/embulk/data/new/java/file_input.java.erb +91 -12
  24. data/lib/embulk/data/new/java/file_output.java.erb +35 -8
  25. data/lib/embulk/data/new/java/filter.java.erb +16 -7
  26. data/lib/embulk/data/new/java/formatter.java.erb +16 -7
  27. data/lib/embulk/data/new/java/input.java.erb +18 -14
  28. data/lib/embulk/data/new/java/output.java.erb +16 -8
  29. data/lib/embulk/data/new/java/parser.java.erb +17 -8
  30. data/lib/embulk/data/new/java/plugin_loader.rb.erb +1 -1
  31. data/lib/embulk/data/new/java/test.java.erb +1 -1
  32. data/lib/embulk/data/new/ruby/filter.rb.erb +6 -4
  33. data/lib/embulk/data/new/ruby/formatter.rb.erb +6 -4
  34. data/lib/embulk/data/new/ruby/gemspec.erb +2 -2
  35. data/lib/embulk/data/new/ruby/input.rb.erb +6 -4
  36. data/lib/embulk/data/new/ruby/output.rb.erb +6 -4
  37. data/lib/embulk/data/new/ruby/parser.rb.erb +6 -4
  38. data/lib/embulk/file_input.rb +4 -0
  39. data/lib/embulk/file_output.rb +4 -0
  40. data/lib/embulk/version.rb +1 -1
  41. metadata +8 -4
@@ -57,7 +57,7 @@ Gem::Specification.new do |spec|
57
57
  spec.description = %[<%= description %>]
58
58
  spec.email = [<%= email.dump %>]
59
59
  spec.licenses = ["MIT"]
60
- # TODO set this: spec.homepage = <%= "https://github.com/#{email[/([^@]*)/]}/#{project_name}".dump %>
60
+ # TODO set this: spec.homepage = <%= "https://github.com/#{email[/([^@]*)/]}/#{full_project_name}".dump %>
61
61
 
62
62
  spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
63
63
  spec.test_files = spec.files.grep(%r"^(test|spec)/")
@@ -1,12 +1,19 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
+ import java.io.InputStream;
4
+ import java.io.IOException;
5
+ import com.google.common.base.Optional;
3
6
  import org.embulk.config.Config;
4
7
  import org.embulk.config.ConfigDefault;
8
+ import org.embulk.config.ConfigInject;
5
9
  import org.embulk.config.ConfigSource;
6
10
  import org.embulk.config.Task;
7
11
  import org.embulk.config.TaskSource;
8
12
  import org.embulk.spi.DecoderPlugin;
9
13
  import org.embulk.spi.FileInput;
14
+ import org.embulk.spi.BufferAllocator;
15
+ import org.embulk.spi.util.FileInputInputStream;
16
+ import org.embulk.spi.util.InputStreamFileInput;
10
17
 
11
18
  public class <%= java_class_name %>
12
19
  implements DecoderPlugin
@@ -14,12 +21,22 @@ public class <%= java_class_name %>
14
21
  public interface PluginTask
15
22
  extends Task
16
23
  {
17
- @Config("property1")
18
- public String getProperty1();
24
+ // configuration option 1 (required integer)
25
+ @Config("option1")
26
+ public int getOption1();
19
27
 
20
- @Config("property2")
21
- @ConfigDefault("0")
22
- public int getProperty2();
28
+ // configuration option 2 (optional string, null is not allowed)
29
+ @Config("optoin2")
30
+ @ConfigDefault("\"myvalue\"")
31
+ public String getOption2();
32
+
33
+ // configuration option 3 (optional string, null is allowed)
34
+ @Config("optoin3")
35
+ @ConfigDefault("null")
36
+ public Optional<String> getOption3();
37
+
38
+ @ConfigInject
39
+ public BufferAllocator getBufferAllocator();
23
40
  }
24
41
 
25
42
  @Override
@@ -31,10 +48,37 @@ public class <%= java_class_name %>
31
48
  }
32
49
 
33
50
  @Override
34
- public FileInput open(TaskSource taskSource, FileInput input)
51
+ public FileInput open(TaskSource taskSource, FileInput fileInput)
35
52
  {
36
- PluginTask task = taskSource.loadTask(PluginTask.class);
53
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
54
+
55
+ // Write your code here :)
56
+ throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
37
57
 
38
- // TODO
58
+ // If expect InputStream, you can use this code:
59
+
60
+ //final FileInputInputStream files = new FileInputInputStream(fileInput);
61
+ //
62
+ //return new InputStreamFileInput(
63
+ // task.getBufferAllocator(),
64
+ // new InputStreamFileInput.Provider() {
65
+ // public InputStream openNext() throws IOException
66
+ // {
67
+ // if (!files.nextFile()) {
68
+ // return null;
69
+ // }
70
+ // return newDecoderInputStream(task, files);
71
+ // }
72
+ //
73
+ // public void close() throws IOException
74
+ // {
75
+ // files.close();
76
+ // }
77
+ // });
39
78
  }
79
+
80
+ //private static InputStream newDecoderInputStream(PluginTask task, InputStream file) throws IOException
81
+ //{
82
+ // return new MyInputStream(file);
83
+ //}
40
84
  }
@@ -1,12 +1,19 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
+ import java.io.OutputStream;
4
+ import java.io.IOException;
5
+ import com.google.common.base.Optional;
3
6
  import org.embulk.config.Config;
4
7
  import org.embulk.config.ConfigDefault;
8
+ import org.embulk.config.ConfigInject;
5
9
  import org.embulk.config.ConfigSource;
6
10
  import org.embulk.config.Task;
7
11
  import org.embulk.config.TaskSource;
8
12
  import org.embulk.spi.EncoderPlugin;
9
13
  import org.embulk.spi.FileOutput;
14
+ import org.embulk.spi.BufferAllocator;
15
+ import org.embulk.spi.util.FileOutputOutputStream;
16
+ import org.embulk.spi.util.OutputStreamFileOutput;
10
17
 
11
18
  public class <%= java_class_name %>
12
19
  implements EncoderPlugin
@@ -14,12 +21,22 @@ public class <%= java_class_name %>
14
21
  public interface PluginTask
15
22
  extends Task
16
23
  {
17
- @Config("property1")
18
- public String getProperty1();
24
+ // configuration option 1 (required integer)
25
+ @Config("option1")
26
+ public int getOption1();
19
27
 
20
- @Config("property2")
21
- @ConfigDefault("0")
22
- public int getProperty2();
28
+ // configuration option 2 (optional string, null is not allowed)
29
+ @Config("optoin2")
30
+ @ConfigDefault("\"myvalue\"")
31
+ public String getOption2();
32
+
33
+ // configuration option 3 (optional string, null is allowed)
34
+ @Config("optoin3")
35
+ @ConfigDefault("null")
36
+ public Optional<String> getOption3();
37
+
38
+ @ConfigInject
39
+ public BufferAllocator getBufferAllocator();
23
40
  }
24
41
 
25
42
  @Override
@@ -33,8 +50,37 @@ public class <%= java_class_name %>
33
50
  @Override
34
51
  public FileOutput open(TaskSource taskSource, FileOutput fileOutput)
35
52
  {
36
- PluginTask task = taskSource.loadTask(PluginTask.class);
53
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
54
+
55
+ // Write your code here :)
56
+ throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
37
57
 
38
- // TODO
58
+ // If expect OutputStream, you can use this code:
59
+
60
+ //final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput,
61
+ // task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
62
+ //
63
+ //return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
64
+ // public OutputStream openNext() throws IOException
65
+ // {
66
+ // output.nextFile();
67
+ // return newEncoderOutputStream(output);
68
+ // }
69
+ //
70
+ // public void finish() throws IOException
71
+ // {
72
+ // fileOutput.finish();
73
+ // }
74
+ //
75
+ // public void close() throws IOException
76
+ // {
77
+ // fileOutput.close();
78
+ // }
79
+ //});
39
80
  }
81
+
82
+ //private static OutputStream newEncoderOutputStream(PluginTask task, OutputStream file) throws IOException
83
+ //{
84
+ // return new MyOutputStream(file);
85
+ //}
40
86
  }
@@ -1,16 +1,22 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
3
  import java.util.List;
4
+ import java.util.ArrayList;
5
+ import com.google.common.base.Optional;
6
+ import com.google.common.collect.ImmutableList;
4
7
  import org.embulk.config.CommitReport;
5
8
  import org.embulk.config.Config;
6
9
  import org.embulk.config.ConfigDefault;
10
+ import org.embulk.config.ConfigInject;
7
11
  import org.embulk.config.ConfigDiff;
8
12
  import org.embulk.config.ConfigSource;
9
13
  import org.embulk.config.Task;
10
14
  import org.embulk.config.TaskSource;
11
15
  import org.embulk.spi.Exec;
12
16
  import org.embulk.spi.FileInputPlugin;
17
+ import org.embulk.spi.BufferAllocator;
13
18
  import org.embulk.spi.TransactionalFileInput;
19
+ import org.embulk.spi.util.InputStreamTransactionalFileInput;
14
20
 
15
21
  public class <%= java_class_name %>
16
22
  implements FileInputPlugin
@@ -18,12 +24,33 @@ public class <%= java_class_name %>
18
24
  public interface PluginTask
19
25
  extends Task
20
26
  {
21
- @Config("property1")
22
- public String getProperty1();
27
+ // configuration option 1 (required integer)
28
+ @Config("option1")
29
+ public int getOption1();
23
30
 
24
- @Config("property2")
25
- @ConfigDefault("0")
26
- public int getProperty2();
31
+ // configuration option 2 (optional string, null is not allowed)
32
+ @Config("optoin2")
33
+ @ConfigDefault("\"myvalue\"")
34
+ public String getOption2();
35
+
36
+ // configuration option 3 (optional string, null is allowed)
37
+ @Config("optoin3")
38
+ @ConfigDefault("null")
39
+ public Optional<String> getOption3();
40
+
41
+ //@Config("path_prefix")
42
+ //public String getPathPrefix();
43
+
44
+ //@Config("last_path")
45
+ //@ConfigDefault("null")
46
+ //public Optional<String> getLastPath();
47
+
48
+ // usually, you store list of files in task to pass them from transaction() to run().
49
+ //public List<String> getFiles();
50
+ //public void setFiles(List<String> files);
51
+
52
+ @ConfigInject
53
+ public BufferAllocator getBufferAllocator();
27
54
  }
28
55
 
29
56
  @Override
@@ -31,19 +58,50 @@ public class <%= java_class_name %>
31
58
  {
32
59
  PluginTask task = config.loadConfig(PluginTask.class);
33
60
 
34
- // taskCount is usually number of input files
35
- int taskCount = 1; // number of run() method calls
61
+ // run() method is called for this number of times in parallel.
62
+ int taskCount = 1;
63
+
64
+ // usually, taskCount is number of input files.
65
+ //task.setFiles(listFiles(task));
66
+ //int taskCount = task.getFiles().size();
36
67
 
37
68
  return resume(task.dump(), taskCount, control);
38
69
  }
39
70
 
71
+ // usually, you have an method to create list of files
72
+ //List<String> listFiles(PluginTask task)
73
+ //{
74
+ // final ImmutableList.Builder<String> builder = ImmutableList.builder();
75
+ // for (String path : listFilesWithPrefix(task.getPathPrefix())) {
76
+ // if (task.getLastPath().isPresent() && path.compareTo(task.getLastPath().get())) {
77
+ // continue;
78
+ // }
79
+ // builder.add(path);
80
+ // }
81
+ // return builder.build();
82
+ //}
83
+
40
84
  @Override
41
85
  public ConfigDiff resume(TaskSource taskSource,
42
86
  int taskCount,
43
87
  FileInputPlugin.Control control)
44
88
  {
45
89
  control.run(taskSource, taskCount);
46
- return Exec.newConfigDiff();
90
+
91
+ ConfigDiff configDiff = Exec.newConfigDiff();
92
+
93
+ // usually, yo uset last_path
94
+ //if (task.getFiles().isEmpty()) {
95
+ // if (task.getLastPath().isPresent()) {
96
+ // configDiff.set("last_path", task.getLastPath().get());
97
+ // }
98
+ //} else {
99
+ // List<String> files = new ArrayList<String>(task.getFiles());
100
+ // Collections.sort(files);
101
+ // configDiff.set("last_path", files.get(files.size() - 1));
102
+ //}
103
+
104
+ return configDiff;
47
105
  }
48
106
 
49
107
  @Override
@@ -56,9 +114,30 @@ public class <%= java_class_name %>
56
114
  @Override
57
115
  public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
58
116
  {
59
- PluginTask task = taskSource.loadTask(PluginTask.class);
117
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
60
118
 
61
- // TODO
62
- throw new UnsupportedOperationException("The 'open' method needs to be implemented");
119
+ // Write your code here :)
120
+ throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
121
+
122
+ // if you expect InputStream, you can use this code:
123
+
124
+ //InputStream input = openInputStream(task, task.getFiles().get(taskIndex));
125
+ //
126
+ //return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input) {
127
+ // @Override
128
+ // public void abort()
129
+ // { }
130
+ //
131
+ // @Override
132
+ // public CommitReport commit()
133
+ // {
134
+ // return Exec.newCommitReport();
135
+ // }
136
+ //}
63
137
  }
138
+
139
+ //private static InputStream openInputStream(PluginTask task, String path)
140
+ //{
141
+ // return new MyInputStream(file);
142
+ //}
64
143
  }
@@ -1,6 +1,7 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
3
  import java.util.List;
4
+ import com.google.common.base.Optional;
4
5
  import org.embulk.config.CommitReport;
5
6
  import org.embulk.config.Config;
6
7
  import org.embulk.config.ConfigDefault;
@@ -18,12 +19,35 @@ public class <%= java_class_name %>
18
19
  public interface PluginTask
19
20
  extends Task
20
21
  {
21
- @Config("property1")
22
- public String getProperty1();
22
+ // configuration option 1 (required integer)
23
+ @Config("option1")
24
+ public int getOption1();
23
25
 
24
- @Config("property2")
25
- @ConfigDefault("0")
26
- public int getProperty2();
26
+ // configuration option 2 (optional string, null is not allowed)
27
+ @Config("optoin2")
28
+ @ConfigDefault("\"myvalue\"")
29
+ public String getOption2();
30
+
31
+ // configuration option 3 (optional string, null is allowed)
32
+ @Config("optoin3")
33
+ @ConfigDefault("null")
34
+ public Optional<String> getOption3();
35
+
36
+ // usually, run() method needs to write multiple files because size of a file
37
+ // can be very large. So, file name will be:
38
+ //
39
+ // path_prefix + String.format(sequence_format, taskIndex, sequenceCounterInRunMethod) + file_ext
40
+ //
41
+
42
+ //@Config("path_prefix")
43
+ //public String getPathPrefix();
44
+
45
+ //@Config("file_ext")
46
+ //public String getFileNameExtension();
47
+
48
+ //@Config("sequence_format")
49
+ //@ConfigDefault("\"%03d.%02d.\"")
50
+ //public String getSequenceFormat();
27
51
  }
28
52
 
29
53
  @Override
@@ -60,7 +84,10 @@ public class <%= java_class_name %>
60
84
  {
61
85
  PluginTask task = taskSource.loadTask(PluginTask.class);
62
86
 
63
- // TODO
64
- throw new UnsupportedOperationException("The 'open' method needs to be implemented");
87
+ // Write your code here :)
88
+ throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
89
+
90
+ // See LocalFileOutputPlugin as an example implementation:
91
+ // https://github.com/embulk/embulk/blob/master/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java
65
92
  }
66
93
  }
@@ -1,5 +1,6 @@
1
1
  package org.embulk.<%= embulk_category %>;
2
2
 
3
+ import com.google.common.base.Optional;
3
4
  import org.embulk.config.Config;
4
5
  import org.embulk.config.ConfigDefault;
5
6
  import org.embulk.config.ConfigDiff;
@@ -17,12 +18,19 @@ public class <%= java_class_name %>
17
18
  public interface PluginTask
18
19
  extends Task
19
20
  {
20
- @Config("property1")
21
- public String getProperty1();
22
-
23
- @Config("property2")
24
- @ConfigDefault("0")
25
- public int getProperty2();
21
+ // configuration option 1 (required integer)
22
+ @Config("option1")
23
+ public int getOption1();
24
+
25
+ // configuration option 2 (optional string, null is not allowed)
26
+ @Config("optoin2")
27
+ @ConfigDefault("\"myvalue\"")
28
+ public String getOption2();
29
+
30
+ // configuration option 3 (optional string, null is allowed)
31
+ @Config("optoin3")
32
+ @ConfigDefault("null")
33
+ public Optional<String> getOption3();
26
34
  }
27
35
 
28
36
  @Override
@@ -42,6 +50,7 @@ public class <%= java_class_name %>
42
50
  {
43
51
  PluginTask task = taskSource.loadTask(PluginTask.class);
44
52
 
45
- // TODO
53
+ // Write your code here :)
54
+ throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
46
55
  }
47
56
  }
@@ -1,5 +1,6 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
+ import com.google.common.base.Optional;
3
4
  import org.embulk.config.Config;
4
5
  import org.embulk.config.ConfigDefault;
5
6
  import org.embulk.config.ConfigDiff;
@@ -17,12 +18,19 @@ public class <%= java_class_name %>
17
18
  public interface PluginTask
18
19
  extends Task
19
20
  {
20
- @Config("property1")
21
- public String getProperty1();
21
+ // configuration option 1 (required integer)
22
+ @Config("option1")
23
+ public int getOption1();
22
24
 
23
- @Config("property2")
24
- @ConfigDefault("0")
25
- public int getProperty2();
25
+ // configuration option 2 (optional string, null is not allowed)
26
+ @Config("optoin2")
27
+ @ConfigDefault("\"myvalue\"")
28
+ public String getOption2();
29
+
30
+ // configuration option 3 (optional string, null is allowed)
31
+ @Config("optoin3")
32
+ @ConfigDefault("null")
33
+ public Optional<String> getOption3();
26
34
  }
27
35
 
28
36
  @Override
@@ -40,6 +48,7 @@ public class <%= java_class_name %>
40
48
  {
41
49
  PluginTask task = taskSource.loadTask(PluginTask.class);
42
50
 
43
- // TODO
51
+ // Write your code here :)
52
+ throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
44
53
  }
45
54
  }