logstash-input-google_cloud_storage 0.9.0-java → 0.10.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/CONTRIBUTING.md +1 -1
- data/README.md +4 -3
- data/docs/index.asciidoc +224 -5
- data/lib/logstash-input-google_cloud_storage_jars.rb +32 -28
- data/lib/logstash/inputs/cloud_storage/blob_adapter.rb +0 -13
- data/lib/logstash/inputs/cloud_storage/blob_filter.rb +0 -14
- data/lib/logstash/inputs/cloud_storage/client.rb +0 -13
- data/lib/logstash/inputs/cloud_storage/file_reader.rb +2 -15
- data/lib/logstash/inputs/cloud_storage/processed_db.rb +0 -13
- data/lib/logstash/inputs/google_cloud_storage.rb +2 -15
- data/logstash-input-google_cloud_storage.gemspec +1 -1
- data/spec/inputs/cloud_storage/blob_adapter_spec.rb +0 -14
- data/spec/inputs/cloud_storage/blob_filter_spec.rb +0 -14
- data/spec/inputs/cloud_storage/client_spec.rb +1 -15
- data/spec/inputs/cloud_storage/file_reader_spec.rb +2 -16
- data/spec/inputs/cloud_storage/processed_db_spec.rb +0 -14
- data/spec/inputs/google_cloud_storage_spec.rb +1 -15
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.6/jackson-core-2.9.6.jar +0 -0
- data/vendor/jar-dependencies/com/google/api-client/google-api-client/1.27.0/google-api-client-1.27.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/api-common/1.7.0/api-common-1.7.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/gax-httpjson/0.55.0/gax-httpjson-0.55.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/gax/1.38.0/gax-1.38.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/grpc/proto-google-common-protos/{1.7.0/proto-google-common-protos-1.7.0.jar → 1.12.0/proto-google-common-protos-1.12.0.jar} +0 -0
- data/vendor/jar-dependencies/com/google/api/grpc/proto-google-iam-v1/{0.8.0/proto-google-iam-v1-0.8.0.jar → 0.12.0/proto-google-iam-v1-0.12.0.jar} +0 -0
- data/vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev20181109-1.27.0/google-api-services-storage-v1-rev20181109-1.27.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.12.0/google-auth-library-credentials-0.12.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.12.0/google-auth-library-oauth2-http-0.12.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/1.62.0/google-cloud-core-http-1.62.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.62.0/google-cloud-core-1.62.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.62.0/google-cloud-storage-1.62.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar +0 -0
- data/vendor/jar-dependencies/com/google/guava/guava/26.0-android/guava-26.0-android.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.27.0/google-http-client-appengine-1.27.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.27.0/google-http-client-jackson2-1.27.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client/1.27.0/google-http-client-1.27.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar +0 -0
- data/vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.27.0/google-oauth-client-1.27.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.6.1/protobuf-java-util-3.6.1.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.6.1/protobuf-java-3.6.1.jar +0 -0
- data/vendor/jar-dependencies/commons-codec/commons-codec/1.10/commons-codec-1.10.jar +0 -0
- data/vendor/jar-dependencies/commons-logging/commons-logging/1.2/commons-logging-1.2.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-context/1.14.0/grpc-context-1.14.0.jar +0 -0
- data/vendor/jar-dependencies/io/opencensus/opencensus-api/0.17.0/opencensus-api-0.17.0.jar +0 -0
- data/vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.17.0/opencensus-contrib-http-util-0.17.0.jar +0 -0
- data/vendor/jar-dependencies/javax/annotation/javax.annotation-api/1.2/javax.annotation-api-1.2.jar +0 -0
- data/vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.5.5/httpclient-4.5.5.jar +0 -0
- data/vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.4.9/httpcore-4.4.9.jar +0 -0
- data/vendor/jar-dependencies/org/checkerframework/checker-compat-qual/2.5.2/checker-compat-qual-2.5.2.jar +0 -0
- data/vendor/jar-dependencies/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar +0 -0
- metadata +34 -34
- data/docs/configuration.asciidoc +0 -112
- data/docs/examples.asciidoc +0 -84
- data/docs/metadata.asciidoc +0 -20
- data/docs/resources.asciidoc +0 -6
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar +0 -0
- data/vendor/jar-dependencies/com/google/api-client/google-api-client/1.23.0/google-api-client-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/api-common/1.5.0/api-common-1.5.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/gax-httpjson/0.40.0/gax-httpjson-0.40.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/api/gax/1.23.0/gax-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/apis/google-api-services-storage/v1-rev114-1.23.0/google-api-services-storage-v1-rev114-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/auth/google-auth-library-credentials/0.9.0/google-auth-library-credentials-0.9.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/auth/google-auth-library-oauth2-http/0.9.0/google-auth-library-oauth2-http-0.9.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-core-http/1.25.0/google-cloud-core-http-1.25.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-core/1.25.0/google-cloud-core-1.25.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/cloud/google-cloud-storage/1.25.0/google-cloud-storage-1.25.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/code/findbugs/jsr305/3.0.1/jsr305-3.0.1.jar +0 -0
- data/vendor/jar-dependencies/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client-appengine/1.23.0/google-http-client-appengine-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client-jackson2/1.23.0/google-http-client-jackson2-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/http-client/google-http-client/1.23.0/google-http-client-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/oauth-client/google-oauth-client/1.23.0/google-oauth-client-1.23.0.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java-util/3.5.1/protobuf-java-util-3.5.1.jar +0 -0
- data/vendor/jar-dependencies/com/google/protobuf/protobuf-java/3.5.1/protobuf-java-3.5.1.jar +0 -0
- data/vendor/jar-dependencies/commons-codec/commons-codec/1.3/commons-codec-1.3.jar +0 -0
- data/vendor/jar-dependencies/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar +0 -0
- data/vendor/jar-dependencies/io/grpc/grpc-context/1.9.0/grpc-context-1.9.0.jar +0 -0
- data/vendor/jar-dependencies/io/opencensus/opencensus-api/0.11.1/opencensus-api-0.11.1.jar +0 -0
- data/vendor/jar-dependencies/io/opencensus/opencensus-contrib-http-util/0.11.1/opencensus-contrib-http-util-0.11.1.jar +0 -0
- data/vendor/jar-dependencies/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar +0 -0
- data/vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e78557c8bf094bfd04130b57015071baadd85453f4f697589352da83e40fdeb6
|
4
|
+
data.tar.gz: 0320d090a6bf8dc24676ab59ea59c401cb6aa18c8d4206a1df69017de7901617
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c9fd651ea1779ec2e824f11975ff95e3f3b3d3be809c150e590d60e2d500ab2fc78d4f1058326ba7c7058f75a1be3d693bb7f0317ab3de4f9ad30cd8e8a73188
|
7
|
+
data.tar.gz: 5935af6d43c06e4c766fed2b64e5d5c01ebed026edfb157ab1f27ff06af3111a5dff83df35a6f36f8fa4b0304fb04c1edc4bdfefe12de3598f3539ab24b4b2de
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTING.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
|
2
2
|
|
3
|
-
Programming is not a required skill. Whatever you've seen about open source and maintainers or community members
|
3
|
+
Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
|
4
4
|
|
5
5
|
It is more important to the community that you are able to contribute.
|
6
6
|
|
data/README.md
CHANGED
@@ -4,7 +4,7 @@ Extracts events from files in a Google Cloud Storage bucket.
|
|
4
4
|
|
5
5
|
Example use-cases:
|
6
6
|
|
7
|
-
* Read https://cloud.google.com/stackdriver/
|
7
|
+
* Read [Stackdriver logs](https://cloud.google.com/stackdriver/) from a Cloud Storage bucket into Elastic.
|
8
8
|
* Read gzipped logs from cold-storage into Elastic.
|
9
9
|
* Restore data from an Elastic dump.
|
10
10
|
* Extract data from Cloud Storage, transform it with Logstash and load it into BigQuery.
|
@@ -36,6 +36,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
|
|
36
36
|
- Install dependencies
|
37
37
|
```sh
|
38
38
|
bundle install
|
39
|
+
bundle exec rake vendor
|
39
40
|
```
|
40
41
|
|
41
42
|
#### Test
|
@@ -66,7 +67,7 @@ bin/logstash-plugin install --no-verify logstash-input-google_cloud_storage
|
|
66
67
|
```
|
67
68
|
- Run Logstash with your plugin
|
68
69
|
```sh
|
69
|
-
bin/logstash -e '
|
70
|
+
bin/logstash -e 'input {google_cloud_storage {}}'
|
70
71
|
```
|
71
72
|
At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
|
72
73
|
|
@@ -88,7 +89,7 @@ bin/logstash-plugin install /your/local/plugin/logstash-input-google_cloud_stora
|
|
88
89
|
|
89
90
|
All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
|
90
91
|
|
91
|
-
Programming is not a required skill. Whatever you've seen about open source and maintainers or community members
|
92
|
+
Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
|
92
93
|
|
93
94
|
It is more important to the community that you are able to contribute.
|
94
95
|
|
data/docs/index.asciidoc
CHANGED
@@ -19,7 +19,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
|
|
19
19
|
|
20
20
|
include::{include_path}/plugin_header.asciidoc[]
|
21
21
|
|
22
|
-
|
22
|
+
[id="plugins-{type}s-{plugin}-description"]
|
23
23
|
==== Description
|
24
24
|
|
25
25
|
Extracts events from files in a Google Cloud Storage bucket.
|
@@ -33,11 +33,230 @@ Example use-cases:
|
|
33
33
|
|
34
34
|
Note: While this project is partially maintained by Google, this is not an official Google product.
|
35
35
|
|
36
|
+
[id="plugins-{type}s-{plugin}-metadata-attributes"]
|
37
|
+
==== Metadata Attributes
|
38
|
+
|
39
|
+
The plugin exposes several metadata attributes about the object being read.
|
40
|
+
You can access these later in the pipeline to augment the data or perform conditional logic.
|
41
|
+
|
42
|
+
[cols="<,<,<",options="header",]
|
43
|
+
|=======================================================================
|
44
|
+
| Key | Type | Description
|
45
|
+
| `[@metadata][gcs][bucket]` | `string` | The name of the bucket the file was read from.
|
46
|
+
| `[@metadata][gcs][name]` | `string` | The name of the object.
|
47
|
+
| `[@metadata][gcs][metadata]` | `object` | A map of metadata on the object.
|
48
|
+
| `[@metadata][gcs][md5]` | `string` | MD5 hash of the data. Encoded using base64.
|
49
|
+
| `[@metadata][gcs][crc32c]` | `string` | CRC32c checksum, as described in RFC 4960. Encoded using base64 in big-endian byte order.
|
50
|
+
| `[@metadata][gcs][generation]` | `long` | The content generation of the object. Used for object versioning
|
51
|
+
| `[@metadata][gcs][line]` | `long` | The position of the event in the file. 1 indexed.
|
52
|
+
| `[@metadata][gcs][line_id]` | `string` | A deterministic, unique ID describing this line. This lets you do idempotent inserts into Elasticsearch.
|
53
|
+
|=======================================================================
|
54
|
+
|
55
|
+
More information about object metadata can be found in the
|
56
|
+
https://cloud.google.com/storage/docs/json_api/v1/objects[official documentation].
|
57
|
+
|
58
|
+
[id="plugins-{type}s-{plugin}-example-configurations"]
|
59
|
+
==== Example Configurations
|
60
|
+
|
61
|
+
===== Basic
|
62
|
+
|
63
|
+
Basic configuration to read JSON logs every minute from `my-logs-bucket`.
|
64
|
+
For example, https://cloud.google.com/stackdriver/[Stackdriver logs].
|
65
|
+
|
66
|
+
[source,ruby]
|
67
|
+
----------------------------------
|
68
|
+
input {
|
69
|
+
google_cloud_storage {
|
70
|
+
interval => 60
|
71
|
+
bucket_id => "my-logs-bucket"
|
72
|
+
json_key_file => "/home/user/key.json"
|
73
|
+
file_matches => ".*json"
|
74
|
+
codec => "json_lines"
|
75
|
+
}
|
76
|
+
}
|
77
|
+
output { stdout { codec => rubydebug } }
|
78
|
+
----------------------------------
|
79
|
+
|
80
|
+
===== Idempotent Inserts into Elasticsearch
|
81
|
+
|
82
|
+
If your pipeline might insert the same file multiple times you can use the `line_id` metadata key as a deterministic id.
|
83
|
+
|
84
|
+
The ID has the format: `gs://<bucket_id>/<object_id>:<line_num>@<generation>`.
|
85
|
+
`line_num` represents the nth event deserialized from the file starting at 1.
|
86
|
+
`generation` is a unique id Cloud Storage generates for the object.
|
87
|
+
When an object is overwritten it gets a new generation.
|
88
|
+
|
89
|
+
[source,ruby]
|
90
|
+
----------------------------------
|
91
|
+
input {
|
92
|
+
google_cloud_storage {
|
93
|
+
bucket_id => "batch-jobs-output"
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
output {
|
98
|
+
elasticsearch {
|
99
|
+
document_id => "%{[@metadata][gcs][line_id]}"
|
100
|
+
}
|
101
|
+
}
|
102
|
+
----------------------------------
|
103
|
+
|
104
|
+
===== From Cloud Storage to BigQuery
|
105
|
+
|
106
|
+
Extract data from Cloud Storage, transform it with Logstash and load it into BigQuery.
|
107
|
+
|
108
|
+
[source,ruby]
|
109
|
+
----------------------------------
|
110
|
+
input {
|
111
|
+
google_cloud_storage {
|
112
|
+
interval => 60
|
113
|
+
bucket_id => "batch-jobs-output"
|
114
|
+
file_matches => "purchases.*.csv"
|
115
|
+
json_key_file => "/home/user/key.json"
|
116
|
+
codec => "plain"
|
117
|
+
}
|
118
|
+
}
|
119
|
+
|
120
|
+
filter {
|
121
|
+
csv {
|
122
|
+
columns => ["transaction", "sku", "price"]
|
123
|
+
convert => {
|
124
|
+
"transaction" => "integer"
|
125
|
+
"price" => "float"
|
126
|
+
}
|
127
|
+
}
|
128
|
+
}
|
129
|
+
|
130
|
+
output {
|
131
|
+
google_bigquery {
|
132
|
+
project_id => "my-project"
|
133
|
+
dataset => "logs"
|
134
|
+
csv_schema => "transaction:INTEGER,sku:INTEGER,price:FLOAT"
|
135
|
+
json_key_file => "/path/to/key.json"
|
136
|
+
error_directory => "/tmp/bigquery-errors"
|
137
|
+
ignore_unknown_values => true
|
138
|
+
}
|
139
|
+
}
|
140
|
+
----------------------------------
|
141
|
+
|
142
|
+
[id="plugins-{type}s-{plugin}-additional-resources"]
|
143
|
+
==== Additional Resources
|
144
|
+
|
145
|
+
* https://cloud.google.com/storage/[Cloud Storage Homepage]
|
146
|
+
* https://cloud.google.com/storage/pricing-summary/[Cloud Storage Pricing]
|
147
|
+
* https://cloud.google.com/iam/docs/service-accounts[IAM Service Accounts]
|
148
|
+
* https://cloud.google.com/docs/authentication/production[Application Default Credentials]
|
149
|
+
|
150
|
+
|
151
|
+
[id="plugins-{type}s-{plugin}-options"]
|
152
|
+
==== Google Cloud Storage Input Configuration Options
|
153
|
+
|
154
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
155
|
+
|
156
|
+
[cols="<,<,<",options="header",]
|
157
|
+
|=======================================================================
|
158
|
+
|Setting |Input type|Required
|
159
|
+
| <<plugins-{type}s-{plugin}-bucket_id>> |<<string,string>>|Yes
|
160
|
+
| <<plugins-{type}s-{plugin}-json_key_file>> |<<path,path>>|No
|
161
|
+
| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
|
162
|
+
| <<plugins-{type}s-{plugin}-file_matches>> |<<string,string>>|No
|
163
|
+
| <<plugins-{type}s-{plugin}-file_exclude>> |<<string,string>>|No
|
164
|
+
| <<plugins-{type}s-{plugin}-metadata_key>> |<<string,string>>|No
|
165
|
+
| <<plugins-{type}s-{plugin}-processed_db_path>> |<<path,path>>|No
|
166
|
+
| <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
|
167
|
+
| <<plugins-{type}s-{plugin}-unpack_gzip>> |<<boolean,boolean>>|No
|
168
|
+
|=======================================================================
|
169
|
+
|
170
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
171
|
+
input plugins.
|
172
|
+
|
173
|
+
|
174
|
+
[id="plugins-{type}s-{plugin}-bucket_id"]
|
175
|
+
===== `bucket_id`
|
176
|
+
|
177
|
+
* Value type is <<string,string>>
|
178
|
+
* There is no default value for this setting.
|
179
|
+
|
180
|
+
The bucket containing your log files.
|
181
|
+
|
182
|
+
[id="plugins-{type}s-{plugin}-json_key_file"]
|
183
|
+
===== `json_key_file`
|
184
|
+
|
185
|
+
* Value type is <<path,path>>
|
186
|
+
* There is no default value for this setting.
|
187
|
+
|
188
|
+
The path to the key to authenticate your user to the bucket.
|
189
|
+
This service user _should_ have the `storage.objects.update` permission so it can
|
190
|
+
create metadata on the object preventing it from being scanned multiple times.
|
191
|
+
|
192
|
+
[id="plugins-{type}s-{plugin}-interval"]
|
193
|
+
===== `interval`
|
194
|
+
|
195
|
+
* Value type is <<number,number>>
|
196
|
+
* Default is: `60`
|
197
|
+
|
198
|
+
The number of seconds between looking for new files in your bucket.
|
199
|
+
|
200
|
+
[id="plugins-{type}s-{plugin}-file_matches"]
|
201
|
+
===== `file_matches`
|
202
|
+
|
203
|
+
* Value type is <<string,string>>
|
204
|
+
* Default is: `.*\.log(\.gz)?`
|
205
|
+
|
206
|
+
A regex pattern to filter files. Only files with names matching this will be considered.
|
207
|
+
All files match by default.
|
208
|
+
|
209
|
+
[id="plugins-{type}s-{plugin}-file_exclude"]
|
210
|
+
===== `file_exclude`
|
211
|
+
|
212
|
+
* Value type is <<string,string>>
|
213
|
+
* Default is: `^$`
|
214
|
+
|
215
|
+
Any files matching this regex are excluded from processing.
|
216
|
+
No files are excluded by default.
|
217
|
+
|
218
|
+
[id="plugins-{type}s-{plugin}-metadata_key"]
|
219
|
+
===== `metadata_key`
|
220
|
+
|
221
|
+
* Value type is <<string,string>>
|
222
|
+
* Default is: `x-goog-meta-ls-gcs-input`
|
223
|
+
|
224
|
+
This key will be set on the objects after they've been processed by the plugin. That way you can
|
225
|
+
stop the plugin and not upload files again or prevent them from being uploaded by setting the
|
226
|
+
field manually.
|
227
|
+
|
228
|
+
NOTE: the key is a flag, if a file was partially processed before Logstash exited some events will be resent.
|
229
|
+
|
230
|
+
[id="plugins-{type}s-{plugin}-processed_db_path"]
|
231
|
+
===== `processed_db_path`
|
232
|
+
|
233
|
+
* Value type is <<path,path>>
|
234
|
+
* Default is: `LOGSTASH_DATA/plugins/inputs/google_cloud_storage/db`.
|
235
|
+
|
236
|
+
If set, the plugin will store the list of processed files locally.
|
237
|
+
This allows you to create a service account for the plugin that does not have write permissions.
|
238
|
+
However, the data will not be shared across multiple running instances of Logstash.
|
239
|
+
|
240
|
+
[id="plugins-{type}s-{plugin}-delete"]
|
241
|
+
===== `delete`
|
242
|
+
|
243
|
+
* Value type is <<boolean,boolean>>
|
244
|
+
* Default is: `false`
|
245
|
+
|
246
|
+
Should the log file be deleted after its contents have been updated?
|
247
|
+
|
248
|
+
[id="plugins-{type}s-{plugin}-unpack_gzip"]
|
249
|
+
===== `unpack_gzip`
|
250
|
+
|
251
|
+
* Value type is <<boolean,boolean>>
|
252
|
+
* Default is: `true`
|
36
253
|
|
37
|
-
|
254
|
+
If set to `true`, files ending in `.gz` are decompressed before they're parsed by the codec.
|
255
|
+
The file will be skipped if it has the suffix, but can't be opened as a gzip, e.g.
|
256
|
+
if it has a bad magic number.
|
38
257
|
|
39
|
-
include::examples.asciidoc[]
|
40
258
|
|
41
|
-
|
259
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
260
|
+
include::{include_path}/{type}.asciidoc[]
|
42
261
|
|
43
|
-
|
262
|
+
:default_codec!:
|
@@ -1,38 +1,42 @@
|
|
1
1
|
# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
|
2
2
|
|
3
3
|
require 'jar_dependencies'
|
4
|
-
require_jar('com.google.cloud', 'google-cloud-storage', '1.
|
5
|
-
require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.
|
6
|
-
require_jar('com.google.api', 'api-common', '1.
|
7
|
-
require_jar('com.google.api-client', 'google-api-client', '1.
|
8
|
-
require_jar('com.google.api', 'gax', '1.
|
9
|
-
require_jar('com.google.api', 'gax-httpjson', '0.
|
10
|
-
require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.
|
11
|
-
require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.
|
12
|
-
require_jar('com.google.apis', 'google-api-services-storage', 'v1-
|
13
|
-
require_jar('com.google.auth', 'google-auth-library-credentials', '0.
|
14
|
-
require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.
|
15
|
-
require_jar('com.google.cloud', 'google-cloud-core', '1.
|
16
|
-
require_jar('com.google.cloud', 'google-cloud-core-http', '1.
|
17
|
-
require_jar('com.google.code.findbugs', 'jsr305', '3.0.
|
4
|
+
require_jar('com.google.cloud', 'google-cloud-storage', '1.62.0')
|
5
|
+
require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.9.6')
|
6
|
+
require_jar('com.google.api', 'api-common', '1.7.0')
|
7
|
+
require_jar('com.google.api-client', 'google-api-client', '1.27.0')
|
8
|
+
require_jar('com.google.api', 'gax', '1.38.0')
|
9
|
+
require_jar('com.google.api', 'gax-httpjson', '0.55.0')
|
10
|
+
require_jar('com.google.api.grpc', 'proto-google-common-protos', '1.12.0')
|
11
|
+
require_jar('com.google.api.grpc', 'proto-google-iam-v1', '0.12.0')
|
12
|
+
require_jar('com.google.apis', 'google-api-services-storage', 'v1-rev20181109-1.27.0')
|
13
|
+
require_jar('com.google.auth', 'google-auth-library-credentials', '0.12.0')
|
14
|
+
require_jar('com.google.auth', 'google-auth-library-oauth2-http', '0.12.0')
|
15
|
+
require_jar('com.google.cloud', 'google-cloud-core', '1.62.0')
|
16
|
+
require_jar('com.google.cloud', 'google-cloud-core-http', '1.62.0')
|
17
|
+
require_jar('com.google.code.findbugs', 'jsr305', '3.0.2')
|
18
18
|
require_jar('com.google.code.gson', 'gson', '2.7')
|
19
19
|
require_jar('com.google.errorprone', 'error_prone_annotations', '2.2.0')
|
20
|
-
require_jar('com.google.guava', 'guava', '
|
20
|
+
require_jar('com.google.guava', 'guava', '26.0-android')
|
21
21
|
require_jar('com.google.guava', 'guava-jdk5', '17.0')
|
22
|
-
require_jar('com.google.http-client', 'google-http-client', '1.
|
23
|
-
require_jar('com.google.http-client', 'google-http-client-appengine', '1.
|
22
|
+
require_jar('com.google.http-client', 'google-http-client', '1.27.0')
|
23
|
+
require_jar('com.google.http-client', 'google-http-client-appengine', '1.27.0')
|
24
24
|
require_jar('com.google.http-client', 'google-http-client-jackson', '1.23.0')
|
25
|
-
require_jar('com.google.http-client', 'google-http-client-jackson2', '1.
|
26
|
-
require_jar('com.google.
|
27
|
-
require_jar('com.google.
|
28
|
-
require_jar('com.google.protobuf', 'protobuf-java
|
29
|
-
require_jar('
|
30
|
-
require_jar('commons-
|
31
|
-
require_jar('
|
32
|
-
require_jar('io.
|
33
|
-
require_jar('io.opencensus', 'opencensus-
|
25
|
+
require_jar('com.google.http-client', 'google-http-client-jackson2', '1.27.0')
|
26
|
+
require_jar('com.google.j2objc', 'j2objc-annotations', '1.1')
|
27
|
+
require_jar('com.google.oauth-client', 'google-oauth-client', '1.27.0')
|
28
|
+
require_jar('com.google.protobuf', 'protobuf-java', '3.6.1')
|
29
|
+
require_jar('com.google.protobuf', 'protobuf-java-util', '3.6.1')
|
30
|
+
require_jar('commons-codec', 'commons-codec', '1.10')
|
31
|
+
require_jar('commons-logging', 'commons-logging', '1.2')
|
32
|
+
require_jar('io.grpc', 'grpc-context', '1.14.0')
|
33
|
+
require_jar('io.opencensus', 'opencensus-api', '0.17.0')
|
34
|
+
require_jar('io.opencensus', 'opencensus-contrib-http-util', '0.17.0')
|
35
|
+
require_jar('javax.annotation', 'javax.annotation-api', '1.2')
|
34
36
|
require_jar('joda-time', 'joda-time', '2.9.2')
|
35
|
-
require_jar('org.apache.httpcomponents', 'httpclient', '4.
|
36
|
-
require_jar('org.apache.httpcomponents', 'httpcore', '4.
|
37
|
+
require_jar('org.apache.httpcomponents', 'httpclient', '4.5.5')
|
38
|
+
require_jar('org.apache.httpcomponents', 'httpcore', '4.4.9')
|
39
|
+
require_jar('org.checkerframework', 'checker-compat-qual', '2.5.2')
|
37
40
|
require_jar('org.codehaus.jackson', 'jackson-core-asl', '1.9.11')
|
41
|
+
require_jar('org.codehaus.mojo', 'animal-sniffer-annotations', '1.14')
|
38
42
|
require_jar('org.threeten', 'threetenbp', '1.3.3')
|
@@ -1,18 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
# Copyright 2018 Google LLC
|
4
|
-
#
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
-
# you may not use this file except in compliance with the License.
|
7
|
-
# You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
-
# See the License for the specific language governing permissions and
|
15
|
-
# limitations under the License.
|
16
3
|
require 'java'
|
17
4
|
require 'logstash-input-google_cloud_storage_jars.rb'
|
18
5
|
|
@@ -1,19 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
# Copyright 2018 Google LLC
|
4
|
-
#
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
-
# you may not use this file except in compliance with the License.
|
7
|
-
# You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
-
# See the License for the specific language governing permissions and
|
15
|
-
# limitations under the License.
|
16
|
-
|
17
3
|
module LogStash
|
18
4
|
module Inputs
|
19
5
|
module CloudStorage
|
@@ -1,18 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
# Copyright 2018 Google LLC
|
4
|
-
#
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
-
# you may not use this file except in compliance with the License.
|
7
|
-
# You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
-
# See the License for the specific language governing permissions and
|
15
|
-
# limitations under the License.
|
16
3
|
require 'thread'
|
17
4
|
require 'java'
|
18
5
|
require 'logstash-input-google_cloud_storage_jars.rb'
|
@@ -1,18 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
# Copyright 2018 Google LLC
|
4
|
-
#
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
-
# you may not use this file except in compliance with the License.
|
7
|
-
# You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# https://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
-
# See the License for the specific language governing permissions and
|
15
|
-
# limitations under the License.
|
16
3
|
require 'zlib'
|
17
4
|
|
18
5
|
module LogStash
|
@@ -41,7 +28,7 @@ module LogStash
|
|
41
28
|
def self.read_plain_lines(filename, &block)
|
42
29
|
line_num = 1
|
43
30
|
::File.open(filename).each do |line|
|
44
|
-
block.call(line
|
31
|
+
block.call(line, line_num)
|
45
32
|
line_num += 1
|
46
33
|
end
|
47
34
|
end
|
@@ -49,7 +36,7 @@ module LogStash
|
|
49
36
|
def self.read_gzip_lines(filename, &block)
|
50
37
|
line_num = 1
|
51
38
|
Zlib::GzipReader.open(filename).each_line do |line|
|
52
|
-
block.call(line
|
39
|
+
block.call(line, line_num)
|
53
40
|
line_num += 1
|
54
41
|
end
|
55
42
|
end
|