embulk 0.6.16 → 0.6.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +33 -45
  3. data/build.gradle +3 -3
  4. data/embulk-core/src/main/java/org/embulk/spi/Exec.java +6 -0
  5. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +73 -1
  6. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
  7. data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
  8. data/embulk-docs/src/_static/embulk-logo.svg +133 -0
  9. data/embulk-docs/src/release.rst +1 -0
  10. data/embulk-docs/src/release/release-0.6.17.rst +39 -0
  11. data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +2 -17
  12. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +3 -22
  13. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +2 -2
  14. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +0 -1
  15. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +18 -42
  16. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +3 -3
  17. data/lib/embulk/command/embulk_new_plugin.rb +40 -14
  18. data/lib/embulk/command/embulk_run.rb +3 -3
  19. data/lib/embulk/data/new/README.md.erb +18 -17
  20. data/lib/embulk/data/new/java/build.gradle.erb +1 -1
  21. data/lib/embulk/data/new/java/decoder.java.erb +53 -9
  22. data/lib/embulk/data/new/java/encoder.java.erb +54 -8
  23. data/lib/embulk/data/new/java/file_input.java.erb +91 -12
  24. data/lib/embulk/data/new/java/file_output.java.erb +35 -8
  25. data/lib/embulk/data/new/java/filter.java.erb +16 -7
  26. data/lib/embulk/data/new/java/formatter.java.erb +16 -7
  27. data/lib/embulk/data/new/java/input.java.erb +18 -14
  28. data/lib/embulk/data/new/java/output.java.erb +16 -8
  29. data/lib/embulk/data/new/java/parser.java.erb +17 -8
  30. data/lib/embulk/data/new/java/plugin_loader.rb.erb +1 -1
  31. data/lib/embulk/data/new/java/test.java.erb +1 -1
  32. data/lib/embulk/data/new/ruby/filter.rb.erb +6 -4
  33. data/lib/embulk/data/new/ruby/formatter.rb.erb +6 -4
  34. data/lib/embulk/data/new/ruby/gemspec.erb +2 -2
  35. data/lib/embulk/data/new/ruby/input.rb.erb +6 -4
  36. data/lib/embulk/data/new/ruby/output.rb.erb +6 -4
  37. data/lib/embulk/data/new/ruby/parser.rb.erb +6 -4
  38. data/lib/embulk/file_input.rb +4 -0
  39. data/lib/embulk/file_output.rb +4 -0
  40. data/lib/embulk/version.rb +1 -1
  41. metadata +8 -4
@@ -1,6 +1,7 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
3
  import java.util.List;
4
+ import com.google.common.base.Optional;
4
5
  import org.embulk.config.CommitReport;
5
6
  import org.embulk.config.Config;
6
7
  import org.embulk.config.ConfigDefault;
@@ -20,14 +21,21 @@ public class <%= java_class_name %>
20
21
  public interface PluginTask
21
22
  extends Task
22
23
  {
23
- @Config("property1")
24
- public String getProperty1();
24
+ // configuration option 1 (required integer)
25
+ @Config("option1")
26
+ public int getOption1();
25
27
 
26
- @Config("property2")
27
- @ConfigDefault("0")
28
- public int getProperty2();
28
+ // configuration option 2 (optional string, null is not allowed)
29
+ @Config("optoin2")
30
+ @ConfigDefault("\"myvalue\"")
31
+ public String getOption2();
29
32
 
30
- // TODO get schema from config or data source
33
+ // configuration option 3 (optional string, null is allowed)
34
+ @Config("optoin3")
35
+ @ConfigDefault("null")
36
+ public Optional<String> getOption3();
37
+
38
+ // if you get schema from config
31
39
  @Config("columns")
32
40
  public SchemaConfig getColumns();
33
41
  }
@@ -67,17 +75,13 @@ public class <%= java_class_name %>
67
75
  {
68
76
  PluginTask task = taskSource.loadTask(PluginTask.class);
69
77
 
70
- // TODO
71
- throw new UnsupportedOperationException("The 'run' method needs to be implemented");
78
+ // Write your code here :)
79
+ throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
72
80
  }
73
81
 
74
82
  @Override
75
83
  public ConfigDiff guess(ConfigSource config)
76
84
  {
77
- // TODO
78
- throw new UnsupportedOperationException("'<%= name %>' input plugin does not support guessing.");
79
- //ConfigDiff diff = Exec.newConfigDiff();
80
- //diff.set("property1", "value");
81
- //return diff;
85
+ return Exec.newConfigDiff();
82
86
  }
83
87
  }
@@ -1,6 +1,7 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
3
  import java.util.List;
4
+ import com.google.common.base.Optional;
4
5
  import org.embulk.config.CommitReport;
5
6
  import org.embulk.config.Config;
6
7
  import org.embulk.config.ConfigDefault;
@@ -20,12 +21,19 @@ public class <%= java_class_name %>
20
21
  public interface PluginTask
21
22
  extends Task
22
23
  {
23
- @Config("property1")
24
- public String getProperty1();
24
+ // configuration option 1 (required integer)
25
+ @Config("option1")
26
+ public int getOption1();
25
27
 
26
- @Config("property2")
27
- @ConfigDefault("0")
28
- public int getProperty2();
28
+ // configuration option 2 (optional string, null is not allowed)
29
+ @Config("optoin2")
30
+ @ConfigDefault("\"myvalue\"")
31
+ public String getOption2();
32
+
33
+ // configuration option 3 (optional string, null is allowed)
34
+ @Config("optoin3")
35
+ @ConfigDefault("null")
36
+ public Optional<String> getOption3();
29
37
  }
30
38
 
31
39
  @Override
@@ -63,7 +71,7 @@ public class <%= java_class_name %>
63
71
  {
64
72
  PluginTask task = taskSource.loadTask(PluginTask.class);
65
73
 
66
- // TODO
67
- throw new UnsupportedOperationException("The 'open' method needs to be implemented");
74
+ // Write your code here :)
75
+ throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
68
76
  }
69
77
  }
@@ -1,5 +1,6 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
+ import com.google.common.base.Optional;
3
4
  import org.embulk.config.Config;
4
5
  import org.embulk.config.ConfigDefault;
5
6
  import org.embulk.config.ConfigDiff;
@@ -18,14 +19,21 @@ public class <%= java_class_name %>
18
19
  public interface PluginTask
19
20
  extends Task
20
21
  {
21
- @Config("property1")
22
- public String getProperty1();
22
+ // configuration option 1 (required integer)
23
+ @Config("option1")
24
+ public int getOption1();
23
25
 
24
- @Config("property2")
25
- @ConfigDefault("0")
26
- public int getProperty2();
26
+ // configuration option 2 (optional string, null is not allowed)
27
+ @Config("optoin2")
28
+ @ConfigDefault("\"myvalue\"")
29
+ public String getOption2();
27
30
 
28
- // TODO get schema from config or data source
31
+ // configuration option 3 (optional string, null is allowed)
32
+ @Config("optoin3")
33
+ @ConfigDefault("null")
34
+ public Optional<String> getOption3();
35
+
36
+ // if you get schema from config or data source
29
37
  @Config("columns")
30
38
  public SchemaConfig getColumns();
31
39
  }
@@ -46,6 +54,7 @@ public class <%= java_class_name %>
46
54
  {
47
55
  PluginTask task = taskSource.loadTask(PluginTask.class);
48
56
 
49
- // TODO
57
+ // Write your code here :)
58
+ throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
50
59
  }
51
60
  }
@@ -1,3 +1,3 @@
1
1
  Embulk::JavaPlugin.register_<%= embulk_category %>(
2
- <%= name.dump %>, <%= "org.embulk.#{embulk_category}.#{java_class_name}".dump %>,
2
+ <%= name.dump %>, <%= "#{java_package_name}.#{java_class_name}".dump %>,
3
3
  File.expand_path('../../../../classpath', __FILE__))
@@ -1,4 +1,4 @@
1
- package org.embulk.<%= embulk_category %>;
1
+ package <%= java_package_name %>;
2
2
 
3
3
  public class Test<%= java_class_name %>
4
4
  {
@@ -7,8 +7,9 @@ module Embulk
7
7
  def self.transaction(config, in_schema, &control)
8
8
  # configuration code:
9
9
  task = {
10
- "property1" => config.param("property1", :string),
11
- "property2" => config.param("property2", :integer, default: 0),
10
+ "option1" => config.param("option1", :integer), # integer, required
11
+ "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
+ "option3" => config.param("option3", :string, default: nil), # string, optional
12
13
  }
13
14
 
14
15
  yield(task, out_columns)
@@ -16,8 +17,9 @@ module Embulk
16
17
 
17
18
  def init
18
19
  # initialization code:
19
- @property1 = task["property1"]
20
- @property2 = task["property2"]
20
+ @option1 = task["option1"]
21
+ @option2 = task["option2"]
22
+ @option3 = task["option3"]
21
23
  end
22
24
 
23
25
  def close
@@ -7,8 +7,9 @@ module Embulk
7
7
  def self.transaction(config, schema, &control)
8
8
  # configuration code:
9
9
  task = {
10
- "property1" => config.param("property1", :string),
11
- "property2" => config.param("property2", :integer, default: 0),
10
+ "option1" => config.param("option1", :integer), # integer, required
11
+ "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
+ "option3" => config.param("option3", :string, default: nil), # string, optional
12
13
  }
13
14
 
14
15
  yield(task)
@@ -16,8 +17,9 @@ module Embulk
16
17
 
17
18
  def init
18
19
  # initialization code:
19
- @property1 = task["property1"]
20
- @property2 = task["property2"]
20
+ @option1 = task["option1"]
21
+ @option2 = task["option2"]
22
+ @option3 = task["option3"]
21
23
 
22
24
  # your data
23
25
  @current_file == nil
@@ -1,13 +1,13 @@
1
1
 
2
2
  Gem::Specification.new do |spec|
3
- spec.name = "<%= project_name %>"
3
+ spec.name = "<%= full_project_name %>"
4
4
  spec.version = "0.1.0"
5
5
  spec.authors = [<%= author.dump %>]
6
6
  spec.summary = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
7
7
  spec.description = <%= "#{description}".dump %>
8
8
  spec.email = [<%= email.dump %>]
9
9
  spec.licenses = ["MIT"]
10
- # TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%= project_name %>"
10
+ # TODO set this: spec.homepage = "https://github.com/<%= email[/([^@]*)/] %>/<%= full_project_name %>"
11
11
 
12
12
  spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
13
  spec.test_files = spec.files.grep(%r{^(test|spec)/})
@@ -7,8 +7,9 @@ module Embulk
7
7
  def self.transaction(config, &control)
8
8
  # configuration code:
9
9
  task = {
10
- "property1" => config.param("property1", :string),
11
- "property2" => config.param("property2", :integer, default: 0),
10
+ "option1" => config.param("option1", :integer), # integer, required
11
+ "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
+ "option3" => config.param("option3", :string, default: nil), # string, optional
12
13
  }
13
14
 
14
15
  columns = [
@@ -39,8 +40,9 @@ module Embulk
39
40
 
40
41
  def init
41
42
  # initialization code:
42
- @property1 = task["property1"]
43
- @property2 = task["property2"]
43
+ @option1 = task["option1"]
44
+ @option2 = task["option2"]
45
+ @option3 = task["option3"]
44
46
  end
45
47
 
46
48
  def run
@@ -7,8 +7,9 @@ module Embulk
7
7
  def self.transaction(config, schema, count, &control)
8
8
  # configuration code:
9
9
  task = {
10
- "property1" => config.param("property1", :string),
11
- "property2" => config.param("property2", :integer, default: 0),
10
+ "option1" => config.param("option1", :integer), # integer, required
11
+ "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
+ "option3" => config.param("option3", :string, default: nil), # string, optional
12
13
  }
13
14
 
14
15
  # resumable output:
@@ -29,8 +30,9 @@ module Embulk
29
30
 
30
31
  def init
31
32
  # initialization code:
32
- @property1 = task["property1"]
33
- @property2 = task["property2"]
33
+ @option1 = task["option1"]
34
+ @option2 = task["option2"]
35
+ @option3 = task["option3"]
34
36
  end
35
37
 
36
38
  def close
@@ -7,8 +7,9 @@ module Embulk
7
7
  def self.transaction(config, &control)
8
8
  # configuration code:
9
9
  task = {
10
- "property1" => config.param("property1", :string),
11
- "property2" => config.param("property2", :integer, default: 0),
10
+ "option1" => config.param("option1", :integer), # integer, required
11
+ "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
+ "option3" => config.param("option3", :string, default: nil), # string, optional
12
13
  }
13
14
 
14
15
  columns = [
@@ -22,8 +23,9 @@ module Embulk
22
23
 
23
24
  def init
24
25
  # initialization code:
25
- @property1 = task["property1"]
26
- @property2 = task["property2"]
26
+ @option1 = task["option1"]
27
+ @option2 = task["option2"]
28
+ @option3 = task["option3"]
27
29
  end
28
30
 
29
31
  def run(file_input)
@@ -79,5 +79,9 @@ module Embulk
79
79
  def close
80
80
  @java_file_input.close
81
81
  end
82
+
83
+ def to_java
84
+ @java_file_input
85
+ end
82
86
  end
83
87
  end
@@ -47,6 +47,10 @@ module Embulk
47
47
  def close
48
48
  @java_file_output.close
49
49
  end
50
+
51
+ def to_java
52
+ @java_file_output
53
+ end
50
54
  end
51
55
 
52
56
  end
@@ -1,3 +1,3 @@
1
1
  module Embulk
2
- VERSION = '0.6.16'
2
+ VERSION = '0.6.17'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.16
4
+ version: 0.6.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-02 00:00:00.000000000 Z
11
+ date: 2015-07-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -234,6 +234,7 @@ files:
234
234
  - embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java
235
235
  - embulk-core/src/main/java/org/embulk/spi/util/Filters.java
236
236
  - embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java
237
+ - embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java
237
238
  - embulk-core/src/main/java/org/embulk/spi/util/Inputs.java
238
239
  - embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java
239
240
  - embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java
@@ -244,6 +245,7 @@ files:
244
245
  - embulk-core/src/main/java/org/embulk/spi/util/Pages.java
245
246
  - embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java
246
247
  - embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java
248
+ - embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java
247
249
  - embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java
248
250
  - embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java
249
251
  - embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java
@@ -290,6 +292,7 @@ files:
290
292
  - embulk-docs/push-gh-pages.sh
291
293
  - embulk-docs/src/_static/embulk-architecture.png
292
294
  - embulk-docs/src/_static/embulk-logo.png
295
+ - embulk-docs/src/_static/embulk-logo.svg
293
296
  - embulk-docs/src/built-in.rst
294
297
  - embulk-docs/src/conf.py
295
298
  - embulk-docs/src/customization.rst
@@ -329,6 +332,7 @@ files:
329
332
  - embulk-docs/src/release/release-0.6.14.rst
330
333
  - embulk-docs/src/release/release-0.6.15.rst
331
334
  - embulk-docs/src/release/release-0.6.16.rst
335
+ - embulk-docs/src/release/release-0.6.17.rst
332
336
  - embulk-docs/src/release/release-0.6.2.rst
333
337
  - embulk-docs/src/release/release-0.6.3.rst
334
338
  - embulk-docs/src/release/release-0.6.4.rst
@@ -445,8 +449,8 @@ files:
445
449
  - classpath/bval-jsr303-0.5.jar
446
450
  - classpath/commons-beanutils-core-1.8.3.jar
447
451
  - classpath/commons-lang3-3.1.jar
448
- - classpath/embulk-core-0.6.16.jar
449
- - classpath/embulk-standards-0.6.16.jar
452
+ - classpath/embulk-core-0.6.17.jar
453
+ - classpath/embulk-standards-0.6.17.jar
450
454
  - classpath/guava-18.0.jar
451
455
  - classpath/guice-4.0.jar
452
456
  - classpath/guice-multibindings-4.0.jar