fluent-plugin-bigquery 0.2.16 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +76 -3
  3. data/Rakefile +1 -0
  4. data/fluent-plugin-bigquery.gemspec +3 -5
  5. data/lib/fluent/plugin/bigquery/schema.rb +221 -0
  6. data/lib/fluent/plugin/bigquery/version.rb +1 -1
  7. data/lib/fluent/plugin/bigquery/writer.rb +289 -0
  8. data/lib/fluent/plugin/out_bigquery.rb +159 -373
  9. data/test/helper.rb +1 -0
  10. data/test/plugin/test_out_bigquery.rb +470 -142
  11. data/test/plugin/test_record_schema.rb +173 -0
  12. metadata +17 -21
data/test/plugin/test_record_schema.rb ADDED
@@ -0,0 +1,173 @@
1
+ require 'helper'
2
+ require 'active_support/json'
3
+ require 'active_support/core_ext/hash'
4
+ require 'active_support/core_ext/object/json'
5
+
6
+ class RecordSchemaTest < Test::Unit::TestCase
7
+ def base_schema
8
+ [
9
+ {
10
+ "name" => "time",
11
+ "type" => "TIMESTAMP",
12
+ "mode" => "REQUIRED"
13
+ },
14
+ {
15
+ "name" => "tty",
16
+ "type" => "STRING",
17
+ "mode" => "NULLABLE"
18
+ },
19
+ {
20
+ "name" => "pwd",
21
+ "type" => "STRING",
22
+ "mode" => "REQUIRED"
23
+ },
24
+ {
25
+ "name" => "user",
26
+ "type" => "STRING",
27
+ "mode" => "REQUIRED"
28
+ },
29
+ {
30
+ "name" => "argv",
31
+ "type" => "STRING",
32
+ "mode" => "REPEATED"
33
+ }
34
+ ]
35
+ end
36
+
37
+ def base_schema_with_new_column
38
+ [
39
+ {
40
+ "name" => "time",
41
+ "type" => "TIMESTAMP",
42
+ "mode" => "REQUIRED"
43
+ },
44
+ {
45
+ "name" => "tty",
46
+ "type" => "STRING",
47
+ "mode" => "NULLABLE"
48
+ },
49
+ {
50
+ "name" => "pwd",
51
+ "type" => "STRING",
52
+ "mode" => "REQUIRED"
53
+ },
54
+ {
55
+ "name" => "user",
56
+ "type" => "STRING",
57
+ "mode" => "REQUIRED"
58
+ },
59
+ {
60
+ "name" => "argv",
61
+ "type" => "STRING",
62
+ "mode" => "REPEATED"
63
+ },
64
+ {
65
+ "name" => "new_column",
66
+ "type" => "STRING",
67
+ "mode" => "REQUIRED"
68
+ }
69
+ ]
70
+ end
71
+
72
+ def base_schema_with_type_changed_column
73
+ [
74
+ {
75
+ "name" => "time",
76
+ "type" => "INTEGER", # change type
77
+ "mode" => "REQUIRED"
78
+ },
79
+ {
80
+ "name" => "tty",
81
+ "type" => "STRING",
82
+ "mode" => "NULLABLE"
83
+ },
84
+ {
85
+ "name" => "pwd",
86
+ "type" => "STRING",
87
+ "mode" => "REQUIRED"
88
+ },
89
+ {
90
+ "name" => "user",
91
+ "type" => "STRING",
92
+ "mode" => "REQUIRED"
93
+ },
94
+ {
95
+ "name" => "argv",
96
+ "type" => "STRING",
97
+ "mode" => "REPEATED"
98
+ },
99
+ ]
100
+ end
101
+
102
+ def test_load_schema
103
+ fields = Fluent::BigQuery::RecordSchema.new("record")
104
+ fields.load_schema(base_schema, true)
105
+ assert { fields.to_a.as_json == base_schema }
106
+ end
107
+
108
+ def test_load_schema_allow_overwrite_with_type_changed_column
109
+ fields = Fluent::BigQuery::RecordSchema.new("record")
110
+ fields.load_schema(base_schema, true)
111
+
112
+ fields.load_schema(base_schema_with_type_changed_column, true)
113
+ assert { fields.to_a.as_json == base_schema_with_type_changed_column }
114
+ end
115
+
116
+ def test_load_schema_allow_overwrite_with_new_column
117
+ fields = Fluent::BigQuery::RecordSchema.new("record")
118
+ fields.load_schema(base_schema, true)
119
+
120
+ fields.load_schema(base_schema_with_new_column, true)
121
+ assert { fields.to_a.as_json == base_schema_with_new_column }
122
+ end
123
+
124
+ def test_load_schema_not_allow_overwrite_with_type_changed_column
125
+ fields = Fluent::BigQuery::RecordSchema.new("record")
126
+ fields.load_schema(base_schema, false)
127
+
128
+ fields.load_schema(base_schema_with_type_changed_column, false)
129
+ assert { fields.to_a.as_json == base_schema }
130
+ end
131
+
132
+ def test_load_schema_no_allow_overwrite_with_new_column
133
+ fields = Fluent::BigQuery::RecordSchema.new("record")
134
+ fields.load_schema(base_schema, false)
135
+
136
+ fields.load_schema(base_schema_with_new_column, false)
137
+ assert { fields.to_a.as_json == base_schema_with_new_column }
138
+ end
139
+
140
+ def test_format_one
141
+ fields = Fluent::BigQuery::RecordSchema.new("record")
142
+ fields.load_schema(base_schema, false)
143
+
144
+ time = Time.local(2016, 2, 7, 19, 0, 0).utc
145
+
146
+ formatted = fields.format_one({
147
+ "time" => time, "tty" => nil, "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", 42]
148
+ })
149
+ assert_equal(
150
+ formatted,
151
+ {
152
+ "time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", "42"]
153
+ }
154
+ )
155
+ end
156
+
157
+ def test_format_one_with_extra_column
158
+ fields = Fluent::BigQuery::RecordSchema.new("record")
159
+ fields.load_schema(base_schema, false)
160
+
161
+ time = Time.local(2016, 2, 7, 19, 0, 0).utc
162
+
163
+ formatted = fields.format_one({
164
+ "time" => time, "tty" => nil, "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", 42.195], "extra" => "extra_data"
165
+ })
166
+ assert_equal(
167
+ formatted,
168
+ {
169
+ "time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "pwd" => "/home", "user" => "joker1007", "argv" => ["foo", "42.195"], "extra" => "extra_data"
170
+ }
171
+ )
172
+ end
173
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.16
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naoya Ito
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-16 00:00:00.000000000 Z
11
+ date: 2016-10-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -115,6 +115,9 @@ dependencies:
115
115
  - - ">="
116
116
  - !ruby/object:Gem::Version
117
117
  version: '3.2'
118
+ - - "<"
119
+ - !ruby/object:Gem::Version
120
+ version: '5'
118
121
  type: :runtime
119
122
  prerelease: false
120
123
  version_requirements: !ruby/object:Gem::Requirement
@@ -122,20 +125,23 @@ dependencies:
122
125
  - - ">="
123
126
  - !ruby/object:Gem::Version
124
127
  version: '3.2'
128
+ - - "<"
129
+ - !ruby/object:Gem::Version
130
+ version: '5'
125
131
  - !ruby/object:Gem::Dependency
126
132
  name: fluentd
127
133
  requirement: !ruby/object:Gem::Requirement
128
134
  requirements:
129
- - - ">="
135
+ - - "~>"
130
136
  - !ruby/object:Gem::Version
131
- version: '0'
137
+ version: 0.12.0
132
138
  type: :runtime
133
139
  prerelease: false
134
140
  version_requirements: !ruby/object:Gem::Requirement
135
141
  requirements:
136
- - - ">="
142
+ - - "~>"
137
143
  - !ruby/object:Gem::Version
138
- version: '0'
144
+ version: 0.12.0
139
145
  - !ruby/object:Gem::Dependency
140
146
  name: fluent-mixin-plaintextformatter
141
147
  requirement: !ruby/object:Gem::Requirement
@@ -178,20 +184,6 @@ dependencies:
178
184
  - - ">="
179
185
  - !ruby/object:Gem::Version
180
186
  version: 0.0.2
181
- - !ruby/object:Gem::Dependency
182
- name: fluent-plugin-dummydata-producer
183
- requirement: !ruby/object:Gem::Requirement
184
- requirements:
185
- - - ">="
186
- - !ruby/object:Gem::Version
187
- version: '0'
188
- type: :development
189
- prerelease: false
190
- version_requirements: !ruby/object:Gem::Requirement
191
- requirements:
192
- - - ">="
193
- - !ruby/object:Gem::Version
194
- version: '0'
195
187
  description: Fluentd plugin to store data on Google BigQuery, by load, or by stream
196
188
  inserts
197
189
  email:
@@ -207,10 +199,13 @@ files:
207
199
  - README.md
208
200
  - Rakefile
209
201
  - fluent-plugin-bigquery.gemspec
202
+ - lib/fluent/plugin/bigquery/schema.rb
210
203
  - lib/fluent/plugin/bigquery/version.rb
204
+ - lib/fluent/plugin/bigquery/writer.rb
211
205
  - lib/fluent/plugin/out_bigquery.rb
212
206
  - test/helper.rb
213
207
  - test/plugin/test_out_bigquery.rb
208
+ - test/plugin/test_record_schema.rb
214
209
  - test/plugin/testdata/apache.schema
215
210
  - test/plugin/testdata/json_key.json
216
211
  - test/plugin/testdata/sudo.schema
@@ -234,13 +229,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
234
229
  version: '0'
235
230
  requirements: []
236
231
  rubyforge_project:
237
- rubygems_version: 2.2.2
232
+ rubygems_version: 2.6.4
238
233
  signing_key:
239
234
  specification_version: 4
240
235
  summary: Fluentd plugin to store data on Google BigQuery
241
236
  test_files:
242
237
  - test/helper.rb
243
238
  - test/plugin/test_out_bigquery.rb
239
+ - test/plugin/test_record_schema.rb
244
240
  - test/plugin/testdata/apache.schema
245
241
  - test/plugin/testdata/json_key.json
246
242
  - test/plugin/testdata/sudo.schema