fluent-plugin-bigquery 0.2.16 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +76 -3
  3. data/Rakefile +1 -0
  4. data/fluent-plugin-bigquery.gemspec +3 -5
  5. data/lib/fluent/plugin/bigquery/schema.rb +221 -0
  6. data/lib/fluent/plugin/bigquery/version.rb +1 -1
  7. data/lib/fluent/plugin/bigquery/writer.rb +289 -0
  8. data/lib/fluent/plugin/out_bigquery.rb +159 -373
  9. data/test/helper.rb +1 -0
  10. data/test/plugin/test_out_bigquery.rb +470 -142
  11. data/test/plugin/test_record_schema.rb +173 -0
  12. metadata +17 -21
data/test/plugin/test_record_schema.rb ADDED
@@ -0,0 +1,173 @@
1
+ require 'helper'
2
+ require 'active_support/json'
3
+ require 'active_support/core_ext/hash'
4
+ require 'active_support/core_ext/object/json'
5
+
6
+ class RecordSchemaTest < Test::Unit::TestCase
7
+ def base_schema
8
+ [
9
+ {
10
+ "name" => "time",
11
+ "type" => "TIMESTAMP",
12
+ "mode" => "REQUIRED"
13
+ },
14
+ {
15
+ "name" => "tty",
16
+ "type" => "STRING",
17
+ "mode" => "NULLABLE"
18
+ },
19
+ {
20
+ "name" => "pwd",
21
+ "type" => "STRING",
22
+ "mode" => "REQUIRED"
23
+ },
24
+ {
25
+ "name" => "user",
26
+ "type" => "STRING",
27
+ "mode" => "REQUIRED"
28
+ },
29
+ {
30
+ "name" => "argv",
31
+ "type" => "STRING",
32
+ "mode" => "REPEATED"
33
+ }
34
+ ]
35
+ end
36
+
37
+ def base_schema_with_new_column
38
+ [
39
+ {
40
+ "name" => "time",
41
+ "type" => "TIMESTAMP",
42
+ "mode" => "REQUIRED"
43
+ },
44
+ {
45
+ "name" => "tty",
46
+ "type" => "STRING",
47
+ "mode" => "NULLABLE"
48
+ },
49
+ {
50
+ "name" => "pwd",
51
+ "type" => "STRING",
52
+ "mode" => "REQUIRED"
53
+ },
54
+ {
55
+ "name" => "user",
56
+ "type" => "STRING",
57
+ "mode" => "REQUIRED"
58
+ },
59
+ {
60
+ "name" => "argv",
61
+ "type" => "STRING",
62
+ "mode" => "REPEATED"
63
+ },
64
+ {
65
+ "name" => "new_column",
66
+ "type" => "STRING",
67
+ "mode" => "REQUIRED"
68
+ }
69
+ ]
70
+ end
71
+
72
+ def base_schema_with_type_changed_column
73
+ [
74
+ {
75
+ "name" => "time",
76
+ "type" => "INTEGER", # change type
77
+ "mode" => "REQUIRED"
78
+ },
79
+ {
80
+ "name" => "tty",
81
+ "type" => "STRING",
82
+ "mode" => "NULLABLE"
83
+ },
84
+ {
85
+ "name" => "pwd",
86
+ "type" => "STRING",
87
+ "mode" => "REQUIRED"
88
+ },
89
+ {
90
+ "name" => "user",
91
+ "type" => "STRING",
92
+ "mode" => "REQUIRED"
93
+ },
94
+ {
95
+ "name" => "argv",
96
+ "type" => "STRING",
97
+ "mode" => "REPEATED"
98
+ },
99
+ ]
100
+ end
101
+
102
+ def test_load_schema
103
+ fields = Fluent::BigQuery::RecordSchema.new("record")
104
+ fields.load_schema(base_schema, true)
105
+ assert { fields.to_a.as_json == base_schema }
106
+ end
107
+
108
+ def test_load_schema_allow_overwrite_with_type_changed_column
109
+ fields = Fluent::BigQuery::RecordSchema.new("record")
110
+ fields.load_schema(base_schema, true)
111
+
112
+ fields.load_schema(base_schema_with_type_changed_column, true)
113
+ assert { fields.to_a.as_json == base_schema_with_type_changed_column }
114
+ end
115
+
116
+ def test_load_schema_allow_overwrite_with_new_column
117
+ fields = Fluent::BigQuery::RecordSchema.new("record")
118
+ fields.load_schema(base_schema, true)
119
+
120
+ fields.load_schema(base_schema_with_new_column, true)
121
+ assert { fields.to_a.as_json == base_schema_with_new_column }
122
+ end
123
+
124
+ def test_load_schema_not_allow_overwrite_with_type_changed_column
125
+ fields = Fluent::BigQuery::RecordSchema.new("record")
126
+ fields.load_schema(base_schema, false)
127
+
128
+ fields.load_schema(base_schema_with_type_changed_column, false)
129
+ assert { fields.to_a.as_json == base_schema }
130
+ end
131
+
132
+ def test_load_schema_no_allow_overwrite_with_new_column
133
+ fields = Fluent::BigQuery::RecordSchema.new("record")
134
+ fields.load_schema(base_schema, false)
135
+
136
+ fields.load_schema(base_schema_with_new_column, false)
137
+ assert { fields.to_a.as_json == base_schema_with_new_column }
138
+ end
139
+
140
+ def test_format_one
141
+ fields = Fluent::BigQuery::RecordSchema.new("record")
142
+ fields.load_schema(base_schema, false)
143
+
144
+ time = Time.local(2016, 2, 7, 19, 0, 0).utc
145
+
146
+ formatted = fields.format_one({
147
+ "time" => time, "tty" => nil, "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", 42]
148
+ })
149
+ assert_equal(
150
+ formatted,
151
+ {
152
+ "time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", "42"]
153
+ }
154
+ )
155
+ end
156
+
157
+ def test_format_one_with_extra_column
158
+ fields = Fluent::BigQuery::RecordSchema.new("record")
159
+ fields.load_schema(base_schema, false)
160
+
161
+ time = Time.local(2016, 2, 7, 19, 0, 0).utc
162
+
163
+ formatted = fields.format_one({
164
+ "time" => time, "tty" => nil, "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", 42.195], "extra" => "extra_data"
165
+ })
166
+ assert_equal(
167
+ formatted,
168
+ {
169
+ "time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", "42.195"], "extra" => "extra_data"
170
+ }
171
+ )
172
+ end
173
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.16
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naoya Ito
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-16 00:00:00.000000000 Z
11
+ date: 2016-10-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -115,6 +115,9 @@ dependencies:
115
115
  - - ">="
116
116
  - !ruby/object:Gem::Version
117
117
  version: '3.2'
118
+ - - "<"
119
+ - !ruby/object:Gem::Version
120
+ version: '5'
118
121
  type: :runtime
119
122
  prerelease: false
120
123
  version_requirements: !ruby/object:Gem::Requirement
@@ -122,20 +125,23 @@ dependencies:
122
125
  - - ">="
123
126
  - !ruby/object:Gem::Version
124
127
  version: '3.2'
128
+ - - "<"
129
+ - !ruby/object:Gem::Version
130
+ version: '5'
125
131
  - !ruby/object:Gem::Dependency
126
132
  name: fluentd
127
133
  requirement: !ruby/object:Gem::Requirement
128
134
  requirements:
129
- - - ">="
135
+ - - "~>"
130
136
  - !ruby/object:Gem::Version
131
- version: '0'
137
+ version: 0.12.0
132
138
  type: :runtime
133
139
  prerelease: false
134
140
  version_requirements: !ruby/object:Gem::Requirement
135
141
  requirements:
136
- - - ">="
142
+ - - "~>"
137
143
  - !ruby/object:Gem::Version
138
- version: '0'
144
+ version: 0.12.0
139
145
  - !ruby/object:Gem::Dependency
140
146
  name: fluent-mixin-plaintextformatter
141
147
  requirement: !ruby/object:Gem::Requirement
@@ -178,20 +184,6 @@ dependencies:
178
184
  - - ">="
179
185
  - !ruby/object:Gem::Version
180
186
  version: 0.0.2
181
- - !ruby/object:Gem::Dependency
182
- name: fluent-plugin-dummydata-producer
183
- requirement: !ruby/object:Gem::Requirement
184
- requirements:
185
- - - ">="
186
- - !ruby/object:Gem::Version
187
- version: '0'
188
- type: :development
189
- prerelease: false
190
- version_requirements: !ruby/object:Gem::Requirement
191
- requirements:
192
- - - ">="
193
- - !ruby/object:Gem::Version
194
- version: '0'
195
187
  description: Fluentd plugin to store data on Google BigQuery, by load, or by stream
196
188
  inserts
197
189
  email:
@@ -207,10 +199,13 @@ files:
207
199
  - README.md
208
200
  - Rakefile
209
201
  - fluent-plugin-bigquery.gemspec
202
+ - lib/fluent/plugin/bigquery/schema.rb
210
203
  - lib/fluent/plugin/bigquery/version.rb
204
+ - lib/fluent/plugin/bigquery/writer.rb
211
205
  - lib/fluent/plugin/out_bigquery.rb
212
206
  - test/helper.rb
213
207
  - test/plugin/test_out_bigquery.rb
208
+ - test/plugin/test_record_schema.rb
214
209
  - test/plugin/testdata/apache.schema
215
210
  - test/plugin/testdata/json_key.json
216
211
  - test/plugin/testdata/sudo.schema
@@ -234,13 +229,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
234
229
  version: '0'
235
230
  requirements: []
236
231
  rubyforge_project:
237
- rubygems_version: 2.2.2
232
+ rubygems_version: 2.6.4
238
233
  signing_key:
239
234
  specification_version: 4
240
235
  summary: Fluentd plugin to store data on Google BigQuery
241
236
  test_files:
242
237
  - test/helper.rb
243
238
  - test/plugin/test_out_bigquery.rb
239
+ - test/plugin/test_record_schema.rb
244
240
  - test/plugin/testdata/apache.schema
245
241
  - test/plugin/testdata/json_key.json
246
242
  - test/plugin/testdata/sudo.schema