logstash-output-google_bigquery 4.0.1-java → 4.1.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/docs/index.asciidoc +15 -0
- data/lib/logstash/outputs/bigquery/streamclient.rb +50 -17
- data/lib/logstash/outputs/google_bigquery.rb +6 -2
- data/logstash-output-google_bigquery.gemspec +1 -1
- data/spec/fixtures/credentials.json +8 -0
- data/spec/outputs/bigquery/streamclient_spec.rb +18 -0
- data/spec/outputs/google_bigquery_spec.rb +2 -2
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 31a414a63c70fc7b35c690e076db1d1595785c79187a94e361b50d8632161496
|
4
|
+
data.tar.gz: 70622ec5c509cd8ac319e4e021b04086a841952b51619dd99fdaa4e2ac21c0a6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b0157e8e33e82c5532bf13cb758ef74d55b05c5ee32185ba53ae0f3dfefbfbc34bcd0df8eac2d1c024b01be93c034ad923e15741df31cb37a84746e944ffe69f
|
7
|
+
data.tar.gz: 31a7c30f4a4bd0d5b95cfec1f4e569961263527a81f8d92b97c0d9ec04dcff8b73c4f5fe4c118cc307543a12958d0986a33ea35e9e45f3c89e7363dcf8a501d5
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 4.1.0
|
2
|
+
- Added `skip_invalid_rows` configuration which will insert all valid rows of a BigQuery insert
|
3
|
+
and skip any invalid ones.
|
4
|
+
- Fixes [#5](https://github.com/logstash-plugins/logstash-output-google_bigquery/issues/5)
|
5
|
+
|
1
6
|
## 4.0.1
|
2
7
|
- Documentation cleanup
|
3
8
|
|
data/docs/index.asciidoc
CHANGED
@@ -99,6 +99,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
99
99
|
| <<plugins-{type}s-{plugin}-key_path>> |<<string,string>>|*Obsolete*
|
100
100
|
| <<plugins-{type}s-{plugin}-project_id>> |<<string,string>>|Yes
|
101
101
|
| <<plugins-{type}s-{plugin}-service_account>> |<<string,string>>|__Deprecated__
|
102
|
+
| <<plugins-{type}s-{plugin}-skip_invalid_rows>> |<<boolean,boolean>>|No
|
102
103
|
| <<plugins-{type}s-{plugin}-table_prefix>> |<<string,string>>|No
|
103
104
|
| <<plugins-{type}s-{plugin}-table_separator>> |<<string,string>>|No
|
104
105
|
| <<plugins-{type}s-{plugin}-temp_directory>> |<<string,string>>|__Deprecated__
|
@@ -174,6 +175,9 @@ added[4.0.0]
|
|
174
175
|
* Default value is `"/tmp/bigquery"`.
|
175
176
|
|
176
177
|
The location to store events that could not be uploaded due to errors.
|
178
|
+
By default if _any_ message in an insert is invalid all will fail.
|
179
|
+
You can use <<plugins-{type}s-{plugin}-skip_invalid_rows>> to allow partial inserts.
|
180
|
+
|
177
181
|
Consider using an additional Logstash input to pipe the contents of
|
178
182
|
these to an alert platform so you can manually fix the events.
|
179
183
|
|
@@ -298,6 +302,17 @@ deprecated[4.0.0, Replaced by `json_key_file` or by using ADC. See <<plugins-{ty
|
|
298
302
|
|
299
303
|
* Value type is <<string,string>>
|
300
304
|
|
305
|
+
[id="plugins-{type}s-{plugin}-skip_invalid_rows"]
|
306
|
+
===== `skip_invalid_rows`
|
307
|
+
|
308
|
+
added[4.1.0]
|
309
|
+
|
310
|
+
* Value type is <<boolean,boolean>>
|
311
|
+
* Default value is `false`
|
312
|
+
|
313
|
+
Insert all valid rows of a request, even if invalid rows exist.
|
314
|
+
The default value is false, which causes the entire request to fail if any invalid rows exist.
|
315
|
+
|
301
316
|
[id="plugins-{type}s-{plugin}-table_prefix"]
|
302
317
|
===== `table_prefix`
|
303
318
|
|
@@ -36,18 +36,20 @@ module LogStash
|
|
36
36
|
@bigquery.create table_info
|
37
37
|
end
|
38
38
|
|
39
|
-
def append(dataset, table, rows, ignore_unknown)
|
39
|
+
def append(dataset, table, rows, ignore_unknown, skip_invalid)
|
40
40
|
api_debug("Appending #{rows.length} rows", dataset, table)
|
41
41
|
|
42
|
-
request = build_append_request
|
42
|
+
request = build_append_request(dataset, table, rows, ignore_unknown, skip_invalid)
|
43
43
|
|
44
44
|
response = @bigquery.insertAll request
|
45
|
-
return
|
45
|
+
return [] unless response.hasErrors
|
46
46
|
|
47
|
+
failed_rows = []
|
47
48
|
response.getInsertErrors().entrySet().each{ |entry|
|
48
49
|
key = entry.getKey
|
49
|
-
|
50
|
+
failed_rows << rows[key]
|
50
51
|
|
52
|
+
errors = entry.getValue
|
51
53
|
errors.each{|bqError|
|
52
54
|
@logger.warn('Error while inserting',
|
53
55
|
key: key,
|
@@ -57,12 +59,13 @@ module LogStash
|
|
57
59
|
}
|
58
60
|
}
|
59
61
|
|
60
|
-
|
62
|
+
failed_rows
|
61
63
|
end
|
62
64
|
|
63
|
-
def build_append_request(dataset, table, rows, ignore_unknown)
|
65
|
+
def build_append_request(dataset, table, rows, ignore_unknown, skip_invalid)
|
64
66
|
request = com.google.cloud.bigquery.InsertAllRequest.newBuilder dataset, table
|
65
67
|
request.setIgnoreUnknownValues ignore_unknown
|
68
|
+
request.setSkipInvalidRows(skip_invalid)
|
66
69
|
|
67
70
|
rows.each { |serialized_row|
|
68
71
|
# deserialize rows into Java maps
|
@@ -75,7 +78,7 @@ module LogStash
|
|
75
78
|
|
76
79
|
# raises an exception if the key file is invalid
|
77
80
|
def get_key_file_error(json_key_file)
|
78
|
-
return nil if
|
81
|
+
return nil if nil_or_empty?(json_key_file)
|
79
82
|
|
80
83
|
abs = ::File.absolute_path json_key_file
|
81
84
|
unless abs == json_key_file
|
@@ -94,16 +97,10 @@ module LogStash
|
|
94
97
|
err = get_key_file_error json_key_file
|
95
98
|
raise err unless err.nil?
|
96
99
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
# TODO: set User-Agent
|
102
|
-
|
103
|
-
key_file = java.io.FileInputStream.new json_key_file
|
104
|
-
credentials = com.google.auth.oauth2.ServiceAccountCredentials.fromStream key_file
|
105
|
-
return com.google.cloud.bigquery.BigQueryOptions.newBuilder()
|
106
|
-
.setCredentials(credentials)
|
100
|
+
com.google.cloud.bigquery.BigQueryOptions.newBuilder()
|
101
|
+
.setCredentials(credentials(json_key_file))
|
102
|
+
.setHeaderProvider(http_headers)
|
103
|
+
.setRetrySettings(retry_settings)
|
107
104
|
.setProjectId(project_id)
|
108
105
|
.build()
|
109
106
|
.getService()
|
@@ -111,9 +108,45 @@ module LogStash
|
|
111
108
|
|
112
109
|
private
|
113
110
|
|
111
|
+
java_import 'com.google.auth.oauth2.GoogleCredentials'
|
112
|
+
def credentials(json_key_path)
|
113
|
+
return GoogleCredentials.getApplicationDefault() if nil_or_empty?(json_key_path)
|
114
|
+
|
115
|
+
key_file = java.io.FileInputStream.new(json_key_path)
|
116
|
+
GoogleCredentials.fromStream(key_file)
|
117
|
+
end
|
118
|
+
|
119
|
+
java_import 'com.google.api.gax.rpc.FixedHeaderProvider'
|
120
|
+
def http_headers
|
121
|
+
gem_name = 'logstash-output-google_bigquery'
|
122
|
+
gem_version = '4.1.0'
|
123
|
+
user_agent = "Elastic/#{gem_name} version/#{gem_version}"
|
124
|
+
|
125
|
+
FixedHeaderProvider.create({ 'User-Agent' => user_agent })
|
126
|
+
end
|
127
|
+
|
128
|
+
java_import 'com.google.api.gax.retrying.RetrySettings'
|
129
|
+
java_import 'org.threeten.bp.Duration'
|
130
|
+
def retry_settings
|
131
|
+
# backoff values taken from com.google.api.client.util.ExponentialBackOff
|
132
|
+
RetrySettings.newBuilder()
|
133
|
+
.setInitialRetryDelay(Duration.ofMillis(500))
|
134
|
+
.setRetryDelayMultiplier(1.5)
|
135
|
+
.setMaxRetryDelay(Duration.ofSeconds(60))
|
136
|
+
.setInitialRpcTimeout(Duration.ofSeconds(20))
|
137
|
+
.setRpcTimeoutMultiplier(1.5)
|
138
|
+
.setMaxRpcTimeout(Duration.ofSeconds(20))
|
139
|
+
.setTotalTimeout(Duration.ofMinutes(15))
|
140
|
+
.build()
|
141
|
+
end
|
142
|
+
|
114
143
|
def api_debug(message, dataset, table)
|
115
144
|
@logger.debug(message, dataset: dataset, table: table)
|
116
145
|
end
|
146
|
+
|
147
|
+
def nil_or_empty?(param)
|
148
|
+
param.nil? || param.empty?
|
149
|
+
end
|
117
150
|
end
|
118
151
|
end
|
119
152
|
end
|
@@ -160,6 +160,10 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
|
|
160
160
|
# Files names follow the pattern `[table name]-[UNIX timestamp].log`
|
161
161
|
config :error_directory, validate: :string, required: true, default: '/tmp/bigquery_errors'
|
162
162
|
|
163
|
+
# Insert all valid rows of a request, even if invalid rows exist. The default value is false,
|
164
|
+
# which causes the entire request to fail if any invalid rows exist.
|
165
|
+
config :skip_invalid_rows, validate: :boolean, default: false
|
166
|
+
|
163
167
|
# The following configuration options still exist to alert users that are using them
|
164
168
|
config :uploader_interval_secs, validate: :number, deprecated: 'No longer used.'
|
165
169
|
config :deleter_interval_secs, validate: :number, deprecated: 'No longer used.'
|
@@ -232,8 +236,8 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base
|
|
232
236
|
|
233
237
|
create_table_if_not_exists table
|
234
238
|
|
235
|
-
|
236
|
-
write_to_errors_file(
|
239
|
+
failed_rows = @bq_client.append(@dataset, table, messages, @ignore_unknown_values, @skip_invalid_rows)
|
240
|
+
write_to_errors_file(failed_rows, table) unless failed_rows.empty?
|
237
241
|
rescue StandardError => e
|
238
242
|
@logger.error 'Error uploading data.', :exception => e
|
239
243
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-google_bigquery'
|
3
|
-
s.version = '4.0
|
3
|
+
s.version = '4.1.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Writes events to Google BigQuery"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -0,0 +1,8 @@
|
|
1
|
+
{
|
2
|
+
"//": "Dummy account from https://github.com/GoogleCloudPlatform/google-cloud-java/google-cloud-clients/google-cloud-core/src/test/java/com/google/cloud/ServiceOptionsTest.java",
|
3
|
+
"private_key_id": "somekeyid",
|
4
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC+K2hSuFpAdrJI\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHgaR\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\nQP/9dJfIkIDJ9Fw9N4Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2LgczOjwWHGi99MFjxSer5m9\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\n0S31xIe3sSlgW0+UbYlF4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvLsKupSeWAW4tMj3eo/64ge\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\nCdDw/0jmZTEjpe4S1lxfHplAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FFJlbXSRsJMf/Qq39mOR2\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\nmYPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\ngUIi9REwXlGDW0Mz50dxpxcKCAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdFCd2UoGddYaOF+KNeM\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\nECR8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\ncoOvtreXCX6XqfrWDtKIvv0vjlHBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa2AY7eafmoU/nZPT\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\nJ7gSidI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\nEfeFCoOX75MxKwXs6xgrw4W//AYGGUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKkXyRDW4IG1Oa2p\nrALStNBx5Y9t0/LQnFI4w3aG\n-----END PRIVATE KEY-----\n",
|
5
|
+
"client_email": "someclientid@developer.gserviceaccount.com",
|
6
|
+
"client_id": "someclientid.apps.googleusercontent.com",
|
7
|
+
"type": "service_account"
|
8
|
+
}
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'logstash/outputs/bigquery/streamclient'
|
4
|
+
|
5
|
+
describe LogStash::Outputs::BigQuery::StreamingClient do
|
6
|
+
|
7
|
+
# This test is mostly to make sure the Java types, signatures and classes
|
8
|
+
# haven't changed being that JRuby is very relaxed.
|
9
|
+
describe '#initialize' do
|
10
|
+
let(:logger) { spy('logger') }
|
11
|
+
|
12
|
+
it 'does not throw an error when initializing' do
|
13
|
+
key_file = ::File.join('spec', 'fixtures', 'credentials.json')
|
14
|
+
key_file = ::File.absolute_path(key_file)
|
15
|
+
LogStash::Outputs::BigQuery::StreamingClient.new(key_file, 'my-project', logger)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -74,7 +74,7 @@ describe LogStash::Outputs::GoogleBigQuery do
|
|
74
74
|
|
75
75
|
it 'creates a table if it does not exist' do
|
76
76
|
allow(subject).to receive(:create_table_if_not_exists).and_return(nil)
|
77
|
-
allow(bq_client).to receive(:append).and_return(
|
77
|
+
allow(bq_client).to receive(:append).and_return([])
|
78
78
|
allow(subject).to receive(:write_to_errors_file).and_return(nil)
|
79
79
|
expect(subject).to receive(:create_table_if_not_exists)
|
80
80
|
|
@@ -83,7 +83,7 @@ describe LogStash::Outputs::GoogleBigQuery do
|
|
83
83
|
|
84
84
|
it 'writes rows to a file on failed insert' do
|
85
85
|
allow(subject).to receive(:create_table_if_not_exists).and_return(nil)
|
86
|
-
allow(bq_client).to receive(:append).and_return(
|
86
|
+
allow(bq_client).to receive(:append).and_return([0])
|
87
87
|
allow(subject).to receive(:write_to_errors_file).and_return(nil)
|
88
88
|
expect(subject).to receive(:write_to_errors_file)
|
89
89
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-google_bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0
|
4
|
+
version: 4.1.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,8 +107,10 @@ files:
|
|
107
107
|
- lib/logstash/outputs/bigquery/streamclient.rb
|
108
108
|
- lib/logstash/outputs/google_bigquery.rb
|
109
109
|
- logstash-output-google_bigquery.gemspec
|
110
|
+
- spec/fixtures/credentials.json
|
110
111
|
- spec/outputs/bigquery/batcher_spec.rb
|
111
112
|
- spec/outputs/bigquery/schema_spec.rb
|
113
|
+
- spec/outputs/bigquery/streamclient_spec.rb
|
112
114
|
- spec/outputs/google_bigquery_spec.rb
|
113
115
|
- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar
|
114
116
|
- vendor/jar-dependencies/com/google/api-client/google-api-client/1.23.0/google-api-client-1.23.0.jar
|
@@ -173,6 +175,8 @@ signing_key:
|
|
173
175
|
specification_version: 4
|
174
176
|
summary: Writes events to Google BigQuery
|
175
177
|
test_files:
|
178
|
+
- spec/fixtures/credentials.json
|
176
179
|
- spec/outputs/bigquery/batcher_spec.rb
|
177
180
|
- spec/outputs/bigquery/schema_spec.rb
|
181
|
+
- spec/outputs/bigquery/streamclient_spec.rb
|
178
182
|
- spec/outputs/google_bigquery_spec.rb
|