embulk 0.7.1-java → 0.7.2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4caef0e11a979edc6d12dc23d1bc89c4796cf869
4
- data.tar.gz: 83ca9285638a2de602cae9e21341e8e63ffdfdef
3
+ metadata.gz: f134c6937fe0926c4f18a31c4bc43b0180673bfd
4
+ data.tar.gz: 0075a441e60c60291e38a607565036579286e9db
5
5
  SHA512:
6
- metadata.gz: 9a5a51d2cb2ebeba9ec15892cd70e12595813920906f23478d4ebb88baaf7b497b2bc5a3d6e8faaa08c0e85ff455f158345a59246d0180128b08065d2e956d04
7
- data.tar.gz: c3409ea3f3c3534c6e70d9b25bbc7ed6a0b3c30c73bddc719d576eea5698246cfec650b7716d373782a3a1befaaf4021e49d24826922abadf82948524a2926ad
6
+ metadata.gz: 1f8a1361a31eaf2a44c27bce626ac0a408b7d2f0f282614807e75e90ad8fb4791fb27ff5673be5b952559b0220147ed68b48e2f9a242aaf01ffab6dfa998dc0f
7
+ data.tar.gz: 4a5532254234c25eaeadde47e7c17aed6771551f0aa0030bcb134f3d10fb7e7d45f882529438a899342327fa539e508169fdff889cd9911831ff5b557a58cf8b
@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
16
16
 
17
17
  allprojects {
18
18
  group = 'org.embulk'
19
- version = '0.7.1'
19
+ version = '0.7.2'
20
20
 
21
21
  ext {
22
22
  jrubyVersion = '9.0.0.0'
@@ -4,6 +4,11 @@ public class ConfigException
4
4
  extends RuntimeException
5
5
  implements UserDataException
6
6
  {
7
+ protected ConfigException()
8
+ {
9
+ super();
10
+ }
11
+
7
12
  public ConfigException(String message)
8
13
  {
9
14
  super(message);
@@ -1,10 +1,9 @@
1
1
  package org.embulk.spi.time;
2
2
 
3
- import org.embulk.config.UserDataException;
3
+ import org.embulk.spi.DataException;
4
4
 
5
5
  public class TimestampParseException
6
- extends Exception
7
- implements UserDataException
6
+ extends DataException
8
7
  {
9
8
  public TimestampParseException(String message)
10
9
  {
@@ -180,6 +180,8 @@ Options
180
180
  +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
181
181
  | max\_quoted\_size\_limit | integer | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped | ``131072`` by default |
182
182
  +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
183
+ | stop\_on\_invalid\_record | boolean | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp) | ``false`` by default |
184
+ +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
183
185
  | default\_timezone | string | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo) | ``UTC`` by default |
184
186
  +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
185
187
  | newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
@@ -4,8 +4,10 @@ Release Notes
4
4
  .. toctree::
5
5
  :maxdepth: 1
6
6
 
7
+ release/release-0.7.2
7
8
  release/release-0.7.1
8
9
  release/release-0.7.0
10
+ release/release-0.6.26
9
11
  release/release-0.6.25
10
12
  release/release-0.6.24
11
13
  release/release-0.6.23
@@ -0,0 +1,17 @@
1
+ Release 0.6.26
2
+ ==================================
3
+
4
+ Built-in plugins
5
+ ------------------
6
+
7
+ * Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
8
+
9
+
10
+ Ruby Plugin API
11
+ ------------------
12
+
13
+ * Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
14
+
15
+ Release Date
16
+ ------------------
17
+ 2015-08-20
@@ -0,0 +1,25 @@
1
+ Release 0.7.2
2
+ ==================================
3
+
4
+ Built-in plugins
5
+ ------------------
6
+
7
+ * Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
8
+
9
+
10
+ Ruby Plugin API
11
+ ------------------
12
+
13
+ * Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
14
+ * Fixed ``cannot be cast to org.jruby.RubyException`` when ConfigException is raised.
15
+
16
+
17
+ General Changes
18
+ ------------------
19
+
20
+ * Fixed ``embulk selfupdate`` subcommand (@cosmo0920++)
21
+
22
+
23
+ Release Date
24
+ ------------------
25
+ 2015-08-20
@@ -90,6 +90,10 @@ public class CsvParserPlugin
90
90
  @Config("allow_extra_columns")
91
91
  @ConfigDefault("false")
92
92
  boolean getAllowExtraColumns();
93
+
94
+ @Config("stop_on_invalid_record")
95
+ @ConfigDefault("false")
96
+ boolean getStopOnInvalidRecord();
93
97
  }
94
98
 
95
99
  public static class QuoteCharacter
@@ -230,6 +234,7 @@ public class CsvParserPlugin
230
234
  final String nullStringOrNull = task.getNullString().orNull();
231
235
  final boolean allowOptionalColumns = task.getAllowOptionalColumns();
232
236
  final boolean allowExtraColumns = task.getAllowExtraColumns();
237
+ final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
233
238
  int skipHeaderLines = task.getSkipHeaderLines();
234
239
 
235
240
  try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -353,6 +358,9 @@ public class CsvParserPlugin
353
358
  } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
354
359
  long lineNumber = tokenizer.getCurrentLineNumber();
355
360
  String skippedLine = tokenizer.skipCurrentLine();
361
+ if (stopOnInvalidRecord) {
362
+ throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
363
+ }
356
364
  log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
357
365
  //exec.notice().skippedLine(skippedLine);
358
366
 
@@ -70,7 +70,9 @@ public class CsvTokenizer
70
70
  // recover lines of quoted value
71
71
  skippedLine = quotedValueLines.remove(0); // TODO optimize performance
72
72
  unreadLines.addAll(quotedValueLines);
73
- unreadLines.add(line);
73
+ if (line != null) {
74
+ unreadLines.add(line);
75
+ }
74
76
  lineNumber -= quotedValueLines.size();
75
77
  quotedValueLines.clear();
76
78
  }
@@ -56,6 +56,8 @@ module Embulk
56
56
  raise "Embulk.setup works only with JRuby."
57
57
  end
58
58
 
59
+ require 'json'
60
+
59
61
  require_classpath
60
62
 
61
63
  systemConfigJson = system_config.merge({
@@ -173,7 +173,7 @@ examples:
173
173
 
174
174
  when :selfupdate
175
175
  op.on('-f', "Skip corruption check", TrueClass) do |b|
176
- system[:force] = true
176
+ options[:force] = true
177
177
  end
178
178
  args = 0..0
179
179
 
@@ -253,7 +253,7 @@ examples:
253
253
 
254
254
  when :selfupdate
255
255
  require 'embulk/command/embulk_selfupdate'
256
- Embulk.selfupdate(system)
256
+ Embulk.selfupdate(options)
257
257
 
258
258
  else
259
259
  require 'json'
@@ -1,17 +1,26 @@
1
1
 
2
2
  module Embulk
3
- module UserDataError
4
- include Java::Config::UserDataException
3
+ # ConfigError is not a ::StandardError but is a java.lang.RuntimeException.
4
+ # "rescue => e" can rescues ConfigError.
5
+ class ConfigError < Java::Config::ConfigException
6
+ def initialize(message=nil)
7
+ if message
8
+ super(message.to_s)
9
+ else
10
+ super()
11
+ end
12
+ end
5
13
  end
6
14
 
7
- class ConfigError < StandardError
8
- include UserDataError
9
- end
10
-
11
- class DataError < StandardError
12
- include UserDataError
13
- end
14
-
15
- class PluginLoadError < StandardError
15
+ # DataError is not a ::StandardError but is a java.lang.RuntimeException.
16
+ # "rescue => e" can rescues DataError.
17
+ class DataError < Java::SPI::DataException
18
+ def initialize(message=nil)
19
+ if message
20
+ super(message.to_s)
21
+ else
22
+ super()
23
+ end
24
+ end
16
25
  end
17
26
  end
@@ -0,0 +1,8 @@
1
+
2
+ module Embulk
3
+ module Exec
4
+ def self.preview?
5
+ Java::Exec.isPreview
6
+ end
7
+ end
8
+ end
@@ -15,6 +15,7 @@ module Embulk
15
15
  require 'embulk/guess_plugin'
16
16
  require 'embulk/executor_plugin'
17
17
  require 'embulk/java_plugin' if Embulk.java?
18
+ require 'embulk/exec' if Embulk.java?
18
19
 
19
20
  class PluginManager
20
21
  def initialize
@@ -1,3 +1,3 @@
1
1
  module Embulk
2
- VERSION = '0.7.1'
2
+ VERSION = '0.7.2'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1
4
+ version: 0.7.2
5
5
  platform: java
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-19 00:00:00.000000000 Z
11
+ date: 2015-08-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: liquid
@@ -133,8 +133,8 @@ files:
133
133
  - classpath/bval-jsr303-0.5.jar
134
134
  - classpath/commons-beanutils-core-1.8.3.jar
135
135
  - classpath/commons-lang3-3.1.jar
136
- - classpath/embulk-core-0.7.1.jar
137
- - classpath/embulk-standards-0.7.1.jar
136
+ - classpath/embulk-core-0.7.2.jar
137
+ - classpath/embulk-standards-0.7.2.jar
138
138
  - classpath/guava-18.0.jar
139
139
  - classpath/guice-4.0.jar
140
140
  - classpath/guice-multibindings-4.0.jar
@@ -421,6 +421,7 @@ files:
421
421
  - embulk-docs/src/release/release-0.6.23.rst
422
422
  - embulk-docs/src/release/release-0.6.24.rst
423
423
  - embulk-docs/src/release/release-0.6.25.rst
424
+ - embulk-docs/src/release/release-0.6.26.rst
424
425
  - embulk-docs/src/release/release-0.6.3.rst
425
426
  - embulk-docs/src/release/release-0.6.4.rst
426
427
  - embulk-docs/src/release/release-0.6.5.rst
@@ -430,6 +431,7 @@ files:
430
431
  - embulk-docs/src/release/release-0.6.9.rst
431
432
  - embulk-docs/src/release/release-0.7.0.rst
432
433
  - embulk-docs/src/release/release-0.7.1.rst
434
+ - embulk-docs/src/release/release-0.7.2.rst
433
435
  - embulk-standards/build.gradle
434
436
  - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
435
437
  - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -505,6 +507,7 @@ files:
505
507
  - lib/embulk/decoder_plugin.rb
506
508
  - lib/embulk/encoder_plugin.rb
507
509
  - lib/embulk/error.rb
510
+ - lib/embulk/exec.rb
508
511
  - lib/embulk/executor_plugin.rb
509
512
  - lib/embulk/file_input.rb
510
513
  - lib/embulk/file_input_plugin.rb