fluent-plugin-bigquery 0.2.10 → 0.2.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fluent/plugin/bigquery/version.rb +1 -1
- data/lib/fluent/plugin/out_bigquery.rb +30 -0
- data/test/plugin/test_out_bigquery.rb +48 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e93c58d0a0c1da28a59026a81f9d060ec3b0e532
|
4
|
+
data.tar.gz: ff810e3c6177a362a28a14cd92ea3091294212d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 86e853fdad235e8ab2b014e100b4e1f355ff9e11da27605e30d66aa9676de3af888f8ab0499445b42df60c884114d5d380c4fa79f6b92ee226063554c4f1344c
|
7
|
+
data.tar.gz: a8e3d37c866073bfd54d25d15482387bb3454191ab5f6a41718c4a7b324d838e275b3dc2a8ce33eb213f57ee0d9a21b66cd878f5b3aec7f555be179238c4dedd
|
@@ -76,6 +76,10 @@ module Fluent
|
|
76
76
|
# config_param :field_record, :string, defualt: nil
|
77
77
|
# config_param :optional_data_field, :string, default: nil
|
78
78
|
|
79
|
+
REGEXP_MAX_NUM = 10
|
80
|
+
config_param :replace_record_key, :bool, default: false
|
81
|
+
(1..REGEXP_MAX_NUM).each {|i| config_param :"replace_record_key_regexp#{i}", :string, default: nil }
|
82
|
+
|
79
83
|
config_param :time_format, :string, default: nil
|
80
84
|
config_param :localtime, :bool, default: nil
|
81
85
|
config_param :utc, :bool, default: nil
|
@@ -168,6 +172,15 @@ module Fluent
|
|
168
172
|
end
|
169
173
|
end
|
170
174
|
|
175
|
+
@regexps = {}
|
176
|
+
(1..REGEXP_MAX_NUM).each do |i|
|
177
|
+
next unless conf["replace_record_key_regexp#{i}"]
|
178
|
+
regexp, replacement = conf["replace_record_key_regexp#{i}"].split(/ /, 2)
|
179
|
+
raise ConfigError, "replace_record_key_regexp#{i} does not contain 2 parameters" unless replacement
|
180
|
+
raise ConfigError, "replace_record_key_regexp#{i} contains a duplicated key, #{regexp}" if @regexps[regexp]
|
181
|
+
@regexps[regexp] = replacement
|
182
|
+
end
|
183
|
+
|
171
184
|
@localtime = false if @localtime.nil? && @utc
|
172
185
|
|
173
186
|
@timef = TimeFormatter.new(@time_format, @localtime)
|
@@ -316,10 +329,27 @@ module Fluent
|
|
316
329
|
raise NotImplementedError # TODO
|
317
330
|
end
|
318
331
|
|
332
|
+
def replace_record_key(record)
|
333
|
+
new_record = {}
|
334
|
+
record.each do |key, _|
|
335
|
+
new_key = key
|
336
|
+
@regexps.each do |regexp, replacement|
|
337
|
+
new_key = new_key.gsub(/#{regexp}/, replacement)
|
338
|
+
end
|
339
|
+
new_key = new_key.gsub(/\W/, '')
|
340
|
+
new_record.store(new_key, record[key])
|
341
|
+
end
|
342
|
+
new_record
|
343
|
+
end
|
344
|
+
|
319
345
|
def format_stream(tag, es)
|
320
346
|
super
|
321
347
|
buf = ''
|
322
348
|
es.each do |time, record|
|
349
|
+
if @replace_record_key
|
350
|
+
record = replace_record_key(record)
|
351
|
+
end
|
352
|
+
|
323
353
|
row = @fields.format(@add_time_field.call(record, time))
|
324
354
|
unless row.empty?
|
325
355
|
row = {"json" => row}
|
@@ -683,6 +683,54 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
683
683
|
driver.instance.shutdown
|
684
684
|
end
|
685
685
|
|
686
|
+
def test_replace_record_key
|
687
|
+
now = Time.now
|
688
|
+
input = [
|
689
|
+
now,
|
690
|
+
{
|
691
|
+
"vhost" => :bar,
|
692
|
+
"@referer" => "http://referer.example",
|
693
|
+
"bot_access" => true,
|
694
|
+
"login-session" => false
|
695
|
+
}
|
696
|
+
]
|
697
|
+
expected = {
|
698
|
+
"json" => {
|
699
|
+
"time" => now.to_i,
|
700
|
+
"vhost" => "bar",
|
701
|
+
"referer" => "http://referer.example",
|
702
|
+
"bot_access" => true,
|
703
|
+
"login_session" => false
|
704
|
+
}
|
705
|
+
}
|
706
|
+
|
707
|
+
driver = create_driver(<<-CONFIG)
|
708
|
+
table foo
|
709
|
+
email foo@bar.example
|
710
|
+
private_key_path /path/to/key
|
711
|
+
project yourproject_id
|
712
|
+
dataset yourdataset_id
|
713
|
+
|
714
|
+
replace_record_key true
|
715
|
+
replace_record_key_regexp1 - _
|
716
|
+
|
717
|
+
time_format %s
|
718
|
+
time_field time
|
719
|
+
|
720
|
+
field_integer time
|
721
|
+
field_string vhost, referer
|
722
|
+
field_boolean bot_access, login_session
|
723
|
+
CONFIG
|
724
|
+
mock_client(driver) do |expect|
|
725
|
+
expect.discovered_api("bigquery", "v2") { stub! }
|
726
|
+
end
|
727
|
+
driver.instance.start
|
728
|
+
buf = driver.instance.format_stream("my.tag", [input])
|
729
|
+
driver.instance.shutdown
|
730
|
+
|
731
|
+
assert_equal expected, MessagePack.unpack(buf)
|
732
|
+
end
|
733
|
+
|
686
734
|
def test_write
|
687
735
|
entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
|
688
736
|
driver = create_driver(CONFIG)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naoya Ito
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|