embulk 0.7.1-java → 0.7.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/embulk-core/src/main/java/org/embulk/config/ConfigException.java +5 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java +2 -3
- data/embulk-docs/src/built-in.rst +2 -0
- data/embulk-docs/src/release.rst +2 -0
- data/embulk-docs/src/release/release-0.6.26.rst +17 -0
- data/embulk-docs/src/release/release-0.7.2.rst +25 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +8 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +3 -1
- data/lib/embulk.rb +2 -0
- data/lib/embulk/command/embulk_run.rb +2 -2
- data/lib/embulk/error.rb +20 -11
- data/lib/embulk/exec.rb +8 -0
- data/lib/embulk/plugin.rb +1 -0
- data/lib/embulk/version.rb +1 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f134c6937fe0926c4f18a31c4bc43b0180673bfd
|
4
|
+
data.tar.gz: 0075a441e60c60291e38a607565036579286e9db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1f8a1361a31eaf2a44c27bce626ac0a408b7d2f0f282614807e75e90ad8fb4791fb27ff5673be5b952559b0220147ed68b48e2f9a242aaf01ffab6dfa998dc0f
|
7
|
+
data.tar.gz: 4a5532254234c25eaeadde47e7c17aed6771551f0aa0030bcb134f3d10fb7e7d45f882529438a899342327fa539e508169fdff889cd9911831ff5b557a58cf8b
|
data/build.gradle
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
package org.embulk.spi.time;
|
2
2
|
|
3
|
-
import org.embulk.
|
3
|
+
import org.embulk.spi.DataException;
|
4
4
|
|
5
5
|
public class TimestampParseException
|
6
|
-
extends
|
7
|
-
implements UserDataException
|
6
|
+
extends DataException
|
8
7
|
{
|
9
8
|
public TimestampParseException(String message)
|
10
9
|
{
|
@@ -180,6 +180,8 @@ Options
|
|
180
180
|
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
181
181
|
| max\_quoted\_size\_limit | integer | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped | ``131072`` by default |
|
182
182
|
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
183
|
+
| stop\_on\_invalid\_record | boolean | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp) | ``false`` by default |
|
184
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
183
185
|
| default\_timezone | string | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo) | ``UTC`` by default |
|
184
186
|
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
185
187
|
| newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
|
data/embulk-docs/src/release.rst
CHANGED
@@ -0,0 +1,17 @@
|
|
1
|
+
Release 0.6.26
|
2
|
+
==================================
|
3
|
+
|
4
|
+
Built-in plugins
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
|
8
|
+
|
9
|
+
|
10
|
+
Ruby Plugin API
|
11
|
+
------------------
|
12
|
+
|
13
|
+
* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
|
14
|
+
|
15
|
+
Release Date
|
16
|
+
------------------
|
17
|
+
2015-08-20
|
@@ -0,0 +1,25 @@
|
|
1
|
+
Release 0.7.2
|
2
|
+
==================================
|
3
|
+
|
4
|
+
Built-in plugins
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
|
8
|
+
|
9
|
+
|
10
|
+
Ruby Plugin API
|
11
|
+
------------------
|
12
|
+
|
13
|
+
* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
|
14
|
+
* Fixed ``cannot be cast to org.jruby.RubyException`` when ConfigException is raised.
|
15
|
+
|
16
|
+
|
17
|
+
General Changes
|
18
|
+
------------------
|
19
|
+
|
20
|
+
* Fixed ``embulk selfupdate`` subcommand (@cosmo0920++)
|
21
|
+
|
22
|
+
|
23
|
+
Release Date
|
24
|
+
------------------
|
25
|
+
2015-08-20
|
@@ -90,6 +90,10 @@ public class CsvParserPlugin
|
|
90
90
|
@Config("allow_extra_columns")
|
91
91
|
@ConfigDefault("false")
|
92
92
|
boolean getAllowExtraColumns();
|
93
|
+
|
94
|
+
@Config("stop_on_invalid_record")
|
95
|
+
@ConfigDefault("false")
|
96
|
+
boolean getStopOnInvalidRecord();
|
93
97
|
}
|
94
98
|
|
95
99
|
public static class QuoteCharacter
|
@@ -230,6 +234,7 @@ public class CsvParserPlugin
|
|
230
234
|
final String nullStringOrNull = task.getNullString().orNull();
|
231
235
|
final boolean allowOptionalColumns = task.getAllowOptionalColumns();
|
232
236
|
final boolean allowExtraColumns = task.getAllowExtraColumns();
|
237
|
+
final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
|
233
238
|
int skipHeaderLines = task.getSkipHeaderLines();
|
234
239
|
|
235
240
|
try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
|
@@ -353,6 +358,9 @@ public class CsvParserPlugin
|
|
353
358
|
} catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
|
354
359
|
long lineNumber = tokenizer.getCurrentLineNumber();
|
355
360
|
String skippedLine = tokenizer.skipCurrentLine();
|
361
|
+
if (stopOnInvalidRecord) {
|
362
|
+
throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
|
363
|
+
}
|
356
364
|
log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
|
357
365
|
//exec.notice().skippedLine(skippedLine);
|
358
366
|
|
@@ -70,7 +70,9 @@ public class CsvTokenizer
|
|
70
70
|
// recover lines of quoted value
|
71
71
|
skippedLine = quotedValueLines.remove(0); // TODO optimize performance
|
72
72
|
unreadLines.addAll(quotedValueLines);
|
73
|
-
|
73
|
+
if (line != null) {
|
74
|
+
unreadLines.add(line);
|
75
|
+
}
|
74
76
|
lineNumber -= quotedValueLines.size();
|
75
77
|
quotedValueLines.clear();
|
76
78
|
}
|
data/lib/embulk.rb
CHANGED
@@ -173,7 +173,7 @@ examples:
|
|
173
173
|
|
174
174
|
when :selfupdate
|
175
175
|
op.on('-f', "Skip corruption check", TrueClass) do |b|
|
176
|
-
|
176
|
+
options[:force] = true
|
177
177
|
end
|
178
178
|
args = 0..0
|
179
179
|
|
@@ -253,7 +253,7 @@ examples:
|
|
253
253
|
|
254
254
|
when :selfupdate
|
255
255
|
require 'embulk/command/embulk_selfupdate'
|
256
|
-
Embulk.selfupdate(
|
256
|
+
Embulk.selfupdate(options)
|
257
257
|
|
258
258
|
else
|
259
259
|
require 'json'
|
data/lib/embulk/error.rb
CHANGED
@@ -1,17 +1,26 @@
|
|
1
1
|
|
2
2
|
module Embulk
|
3
|
-
|
4
|
-
|
3
|
+
# ConfigError is not a ::StandardError but is a java.lang.RuntimeException.
|
4
|
+
# "rescue => e" can rescues ConfigError.
|
5
|
+
class ConfigError < Java::Config::ConfigException
|
6
|
+
def initialize(message=nil)
|
7
|
+
if message
|
8
|
+
super(message.to_s)
|
9
|
+
else
|
10
|
+
super()
|
11
|
+
end
|
12
|
+
end
|
5
13
|
end
|
6
14
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
15
|
+
# DataError is not a ::StandardError but is a java.lang.RuntimeException.
|
16
|
+
# "rescue => e" can rescues DataError.
|
17
|
+
class DataError < Java::SPI::DataException
|
18
|
+
def initialize(message=nil)
|
19
|
+
if message
|
20
|
+
super(message.to_s)
|
21
|
+
else
|
22
|
+
super()
|
23
|
+
end
|
24
|
+
end
|
16
25
|
end
|
17
26
|
end
|
data/lib/embulk/exec.rb
ADDED
data/lib/embulk/plugin.rb
CHANGED
data/lib/embulk/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: liquid
|
@@ -133,8 +133,8 @@ files:
|
|
133
133
|
- classpath/bval-jsr303-0.5.jar
|
134
134
|
- classpath/commons-beanutils-core-1.8.3.jar
|
135
135
|
- classpath/commons-lang3-3.1.jar
|
136
|
-
- classpath/embulk-core-0.7.
|
137
|
-
- classpath/embulk-standards-0.7.
|
136
|
+
- classpath/embulk-core-0.7.2.jar
|
137
|
+
- classpath/embulk-standards-0.7.2.jar
|
138
138
|
- classpath/guava-18.0.jar
|
139
139
|
- classpath/guice-4.0.jar
|
140
140
|
- classpath/guice-multibindings-4.0.jar
|
@@ -421,6 +421,7 @@ files:
|
|
421
421
|
- embulk-docs/src/release/release-0.6.23.rst
|
422
422
|
- embulk-docs/src/release/release-0.6.24.rst
|
423
423
|
- embulk-docs/src/release/release-0.6.25.rst
|
424
|
+
- embulk-docs/src/release/release-0.6.26.rst
|
424
425
|
- embulk-docs/src/release/release-0.6.3.rst
|
425
426
|
- embulk-docs/src/release/release-0.6.4.rst
|
426
427
|
- embulk-docs/src/release/release-0.6.5.rst
|
@@ -430,6 +431,7 @@ files:
|
|
430
431
|
- embulk-docs/src/release/release-0.6.9.rst
|
431
432
|
- embulk-docs/src/release/release-0.7.0.rst
|
432
433
|
- embulk-docs/src/release/release-0.7.1.rst
|
434
|
+
- embulk-docs/src/release/release-0.7.2.rst
|
433
435
|
- embulk-standards/build.gradle
|
434
436
|
- embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
|
435
437
|
- embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
|
@@ -505,6 +507,7 @@ files:
|
|
505
507
|
- lib/embulk/decoder_plugin.rb
|
506
508
|
- lib/embulk/encoder_plugin.rb
|
507
509
|
- lib/embulk/error.rb
|
510
|
+
- lib/embulk/exec.rb
|
508
511
|
- lib/embulk/executor_plugin.rb
|
509
512
|
- lib/embulk/file_input.rb
|
510
513
|
- lib/embulk/file_input_plugin.rb
|