embulk-parser-query_string 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +2 -0
- data/embulk-parser-query_string.gemspec +2 -2
- data/lib/embulk/parser/query_string.rb +22 -10
- data/partial-config.yml +1 -1
- data/test/embulk/guess/test_query_string.rb +1 -1
- data/test/embulk/parser/test_query_string_plugin.rb +14 -21
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb9918e68aa9b4a3569fefcbb2ffa477aba99378
|
4
|
+
data.tar.gz: cbf1b7c3d4d14ecc0180c9de70f8e64b59c185b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c60a005d7be4dd2de37445d1b7e696043824ef69f8c05c4fb90304aea8de14f0a75507031f7667ddacdcf2750806bdd2a657d47ab27a6fbcd384431e939c18fc
|
7
|
+
data.tar.gz: c433316f33f2b8b3ad29b7ec052f14a393c0101ace98ac103cf0d965a5a3fd43915c1e93d2c0273393f7642f53b8793355c33515923ef6e5dc2c58f7cca92094
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.0.3 - 2015-07-08
|
2
|
+
|
3
|
+
* [enhancement] Support embulk 0.6.10 (backward compatibility) [#9](https://github.com/treasure-data/embulk-parser-query_string/pull/9)
|
4
|
+
|
1
5
|
## 0.0.2 - 2015-07-08
|
2
6
|
|
3
7
|
The name of this plugin is changed to "embulk-parser-query_string" from "embulk-parser-query-string".
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-parser-query_string"
|
4
|
-
spec.version = "0.0.
|
4
|
+
spec.version = "0.0.3"
|
5
5
|
spec.authors = ["yoshihara", "uu59"]
|
6
6
|
spec.summary = "Query String parser plugin for Embulk"
|
7
7
|
spec.description = "Parses Query String files read by other file input plugins."
|
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
14
14
|
spec.require_paths = ["lib"]
|
15
15
|
|
16
|
-
spec.add_development_dependency 'embulk', [">= 0.6.
|
16
|
+
spec.add_development_dependency 'embulk', [">= 0.6.10", "< 1.0"]
|
17
17
|
spec.add_development_dependency 'bundler', ['~> 1.0']
|
18
18
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
19
19
|
spec.add_development_dependency 'pry'
|
@@ -10,13 +10,13 @@ module Embulk
|
|
10
10
|
decoder_task = config.load_config(Java::LineDecoder::DecoderTask)
|
11
11
|
|
12
12
|
task = {
|
13
|
-
decoder
|
14
|
-
strip_quote
|
15
|
-
strip_whitespace
|
13
|
+
"decoder" => DataSource.from_java(decoder_task.dump),
|
14
|
+
"strip_quote" => config.param("strip_quote", :bool, default: true),
|
15
|
+
"strip_whitespace" => config.param("strip_whitespace", :bool, default: true),
|
16
16
|
}
|
17
17
|
|
18
18
|
columns = []
|
19
|
-
schema = config.param(
|
19
|
+
schema = config.param("schema", :array, default: [])
|
20
20
|
schema.each do |column|
|
21
21
|
name = column["name"]
|
22
22
|
type = column["type"].to_sym
|
@@ -29,11 +29,11 @@ module Embulk
|
|
29
29
|
|
30
30
|
def init
|
31
31
|
@options = {
|
32
|
-
strip_quote: task[
|
33
|
-
strip_whitespace: task[
|
32
|
+
strip_quote: task["strip_quote"],
|
33
|
+
strip_whitespace: task["strip_whitespace"],
|
34
34
|
}
|
35
35
|
|
36
|
-
@decoder = task.param(
|
36
|
+
@decoder = task.param("decoder", :hash).load_task(Java::LineDecoder::DecoderTask)
|
37
37
|
end
|
38
38
|
|
39
39
|
def run(file_input)
|
@@ -69,10 +69,22 @@ module Embulk
|
|
69
69
|
|
70
70
|
return unless record
|
71
71
|
|
72
|
-
|
73
|
-
|
72
|
+
# NOTE: this conversion is needless afrer Embulk 0.6.13
|
73
|
+
values = schema.map do |column|
|
74
|
+
name = column.name
|
75
|
+
value = record[name]
|
76
|
+
|
77
|
+
case column.type
|
78
|
+
when :long
|
79
|
+
Integer(value)
|
80
|
+
when :timestamp
|
81
|
+
Time.parse(value)
|
82
|
+
else
|
83
|
+
value.to_s
|
84
|
+
end
|
74
85
|
end
|
75
|
-
|
86
|
+
|
87
|
+
page_builder.add(values)
|
76
88
|
end
|
77
89
|
end
|
78
90
|
end
|
data/partial-config.yml
CHANGED
@@ -44,7 +44,7 @@ module Embulk
|
|
44
44
|
def test_type(data)
|
45
45
|
type, expected = data
|
46
46
|
sample_lines = self.class.sample_lines_with_same_keys
|
47
|
-
config = DataSource[{parser
|
47
|
+
config = DataSource[{"parser" => {"type" => type}}]
|
48
48
|
|
49
49
|
actual = QueryString.new.guess_lines(config, sample_lines)
|
50
50
|
assert_equal(expected, actual)
|
@@ -49,33 +49,25 @@ module Embulk
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
52
|
-
class
|
52
|
+
class TestProcessLine < self
|
53
53
|
def test_process_line
|
54
|
-
|
55
|
-
mock(page_builder).add(record.values)
|
56
|
-
end
|
57
|
-
|
54
|
+
mock(page_builder).add(["FOO", 1, Time.parse("2015-07-08T16:25:46")])
|
58
55
|
plugin.send(:process_line, line)
|
59
56
|
end
|
60
57
|
|
61
58
|
private
|
62
59
|
|
63
|
-
def records
|
64
|
-
[
|
65
|
-
{"foo" => "FOO", "bar" => "1"},
|
66
|
-
]
|
67
|
-
end
|
68
|
-
|
69
60
|
def line
|
70
|
-
"foo=FOO&bar=1"
|
61
|
+
"foo=FOO&bar=1&baz=2015-07-08T16:25:46"
|
71
62
|
end
|
72
63
|
end
|
73
64
|
|
74
65
|
def test_transaction
|
66
|
+
expected_task = task.dup
|
67
|
+
expected_task.delete("schema")
|
68
|
+
|
75
69
|
QueryString.transaction(config) do |actual_task, actual_columns|
|
76
|
-
|
77
|
-
t.delete(:schema)
|
78
|
-
assert_equal(t, actual_task)
|
70
|
+
assert_equal(expected_task, actual_task)
|
79
71
|
assert_equal(schema, actual_columns)
|
80
72
|
end
|
81
73
|
end
|
@@ -92,17 +84,18 @@ module Embulk
|
|
92
84
|
|
93
85
|
def task
|
94
86
|
{
|
95
|
-
decoder
|
96
|
-
strip_quote
|
97
|
-
strip_whitespace
|
98
|
-
schema
|
87
|
+
"decoder" => {"Charset" => "UTF-8", "Newline" => "CRLF"},
|
88
|
+
"strip_quote" => true,
|
89
|
+
"strip_whitespace" => true,
|
90
|
+
"schema" => columns,
|
99
91
|
}
|
100
92
|
end
|
101
93
|
|
102
94
|
def columns
|
103
95
|
[
|
104
|
-
{"name" => "foo", "type" =>
|
105
|
-
{"name" => "bar", "type" =>
|
96
|
+
{"name" => "foo", "type" => :string},
|
97
|
+
{"name" => "bar", "type" => :long},
|
98
|
+
{"name" => "baz", "type" => :timestamp},
|
106
99
|
]
|
107
100
|
end
|
108
101
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-parser-query_string
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yoshihara
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.6.
|
19
|
+
version: 0.6.10
|
20
20
|
- - <
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '1.0'
|
@@ -27,7 +27,7 @@ dependencies:
|
|
27
27
|
requirements:
|
28
28
|
- - '>='
|
29
29
|
- !ruby/object:Gem::Version
|
30
|
-
version: 0.6.
|
30
|
+
version: 0.6.10
|
31
31
|
- - <
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '1.0'
|