logstash-input-cloudwatch_logs_importer 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +12 -0
- data/README.md +42 -0
- data/VERSION +1 -0
- data/docs/index.asciidoc +117 -0
- data/lib/logstash-input-cloudwatch_logs_importer_jars.rb +5 -0
- data/lib/logstash/inputs/cloudwatch_logs_importer.rb +12 -0
- data/logstash-input-cloudwatch_logs_importer.gemspec +22 -0
- data/vendor/jar-dependencies/net/justmachinery/logstash-input-cloudwatch_logs_importer/1.0.0/logstash-input-cloudwatch_logs_importer-1.0.0.jar +0 -0
- metadata +106 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 605ab0b35b4ade16d6be30b69f0af3a79360285cd78a2683f5391ff45abb9081
|
4
|
+
data.tar.gz: 51a1a1f9ac98671fbbe24506486f2a37940b387973eae27c0ddc0a82761ed43e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 76fdc5dd2feb4e2216576f749fa4a5b7a4c5329f9173e74356c1e9d883bbf6be9db81788757706f84ede097f589e2c1c2e6617c1247000d1c8df0f88a9b70f8d
|
7
|
+
data.tar.gz: 89819bac50ff28101363f7e63c6969ae4382ad456c42a9c7cbe017325aae46f6e03c48c883269abdc5e13baa93e465d2dde9341d3d7a4a34ca851f9d2c9fdf5e
|
data/Gemfile
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
|
2
|
+
source 'https://rubygems.org'
|
3
|
+
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
7
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
8
|
+
|
9
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
10
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
11
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
12
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
# Logstash Cloudwatch Logs Importer Input
|
2
|
+
|
3
|
+
Plugin to import logs from Cloudwatch Logs.
|
4
|
+
|
5
|
+
If you encounter any problems, please file an issue! Contributions are also welcome.
|
6
|
+
|
7
|
+
See the [Logstash Java input plugin](https://www.elastic.co/guide/en/logstash/current/java-input-plugin.html) guide for instructions on compiling the project.
|
8
|
+
|
9
|
+
## How it works
|
10
|
+
Starting from its time of activation, this plugin will attempt to continuously import
|
11
|
+
both new and old logs.
|
12
|
+
|
13
|
+
Note that you will want to deduplicate your logs at some point in your Logstash pipeline, as some log events will be imported twice.
|
14
|
+
|
15
|
+
|
16
|
+
## Parameters
|
17
|
+
|
18
|
+
### Required
|
19
|
+
|
20
|
+
| Parameter | Input Type | Description |
|
21
|
+
|-----------|------------|---------|
|
22
|
+
| log_groups | array of strings | List of log group names to import logs from. You may need to delete data files in the plugin data directory to restart the import process if you add a new log group. |
|
23
|
+
| plugin_data_directory | string | Writable directory to store plugin data in |
|
24
|
+
|
25
|
+
|
26
|
+
### Optional
|
27
|
+
|
28
|
+
| Parameter | Input Type | Default | Description |
|
29
|
+
|-----------|------------|---------|---------|
|
30
|
+
| target_events_per_request | number | 5000 | Polling rate slows down if cloudwatch logs returns fewer than this many logs per request |
|
31
|
+
| max_polling_interval_milliseconds | number | 120000 (two minutes) | Longest interval allowed before checking for new events. |
|
32
|
+
| backwards_log_fetch_days | number | none | Maximum age of old logs to fetch. |
|
33
|
+
|
34
|
+
### AWS Credentials
|
35
|
+
If not supplied, these will come from the [default provider chain.](
|
36
|
+
https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default)
|
37
|
+
|
38
|
+
| Parameter | Input Type |
|
39
|
+
|-----------|------------|
|
40
|
+
| region | string |
|
41
|
+
| access_key | string |
|
42
|
+
| secret_key | string |
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.0.0
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
:plugin: example
|
2
|
+
:type: input
|
3
|
+
:default_codec: plain
|
4
|
+
// Update header with plugin name and default codec
|
5
|
+
|
6
|
+
///////////////////////////////////////////
|
7
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
8
|
+
///////////////////////////////////////////
|
9
|
+
:version: %VERSION%
|
10
|
+
:release_date: %RELEASE_DATE%
|
11
|
+
:changelog_url: %CHANGELOG_URL%
|
12
|
+
:include_path: ../../../../logstash/docs/include
|
13
|
+
///////////////////////////////////////////
|
14
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
15
|
+
///////////////////////////////////////////
|
16
|
+
|
17
|
+
[id="plugins-{type}s-{plugin}"]
|
18
|
+
|
19
|
+
=== Example input plugin
|
20
|
+
|
21
|
+
include::{include_path}/plugin_header.asciidoc[]
|
22
|
+
|
23
|
+
==== Description
|
24
|
+
|
25
|
+
Add plugin description here
|
26
|
+
|
27
|
+
// Format anchors and links to support generated ids for versioning
|
28
|
+
// Sample anchor: [id="plugins-{type}s-{plugin}-setting_name"]
|
29
|
+
// Sample link: <<plugins-{type}s-{plugin}-setting_name>>
|
30
|
+
|
31
|
+
[id="plugins-{type}s-{plugin}-options"]
|
32
|
+
==== Example Input Configuration Options
|
33
|
+
|
34
|
+
[cols="<,<,<",options="header",]
|
35
|
+
|=======================================================================
|
36
|
+
|Setting |Input type|Required
|
37
|
+
| <<plugins-{type}s-{plugin}-a_setting_name>> |<<boolean,boolean>>|No
|
38
|
+
| <<plugins-{type}s-{plugin}-another_setting_name>> |<<hash,hash>>|No
|
39
|
+
| <<plugins-{type}s-{plugin}-setting_name_3>> |<<string,string>>|No
|
40
|
+
| <<plugins-{type}s-{plugin}-setting_name_4>> |<<number,number>>|No
|
41
|
+
| <<plugins-{type}s-{plugin}-setting_name_5>> |<<array,array>>|No
|
42
|
+
| <<plugins-{type}s-{plugin}-setting_name_6>> |<<bytes,bytes>>|No
|
43
|
+
| <<plugins-{type}s-{plugin}-setting_name_7>> |<<path,path>>|No
|
44
|
+
| <<plugins-{type}s-{plugin}-setting_name_8>> |<<password,password>>|No
|
45
|
+
|=======================================================================
|
46
|
+
|
47
|
+
[id="plugins-{type}s-{plugin}-a_setting_name"]
|
48
|
+
===== `a_setting_name`
|
49
|
+
|
50
|
+
* Value type is <<boolean,boolean>>
|
51
|
+
* Default value is `true`
|
52
|
+
|
53
|
+
Add description here
|
54
|
+
|
55
|
+
[id="plugins-{type}s-{plugin}-another_setting_name"]
|
56
|
+
===== `another_setting_name`
|
57
|
+
|
58
|
+
* Value type is <<hash,hash>>
|
59
|
+
* Default value is `{}`
|
60
|
+
|
61
|
+
Add description here
|
62
|
+
|
63
|
+
[id="plugins-{type}s-{plugin}-setting_name_3"]
|
64
|
+
===== `setting_name_3`
|
65
|
+
|
66
|
+
* Value type is <<string,string>>
|
67
|
+
* Default value is `{}`
|
68
|
+
|
69
|
+
Add description here
|
70
|
+
|
71
|
+
[id="plugins-{type}s-{plugin}-setting_name_4"]
|
72
|
+
===== `setting_name_4`
|
73
|
+
|
74
|
+
* Value type is <<number,number>>
|
75
|
+
* Default value is `0`
|
76
|
+
|
77
|
+
Add description here
|
78
|
+
|
79
|
+
[id="plugins-{type}s-{plugin}-setting_name_5"]
|
80
|
+
===== `setting_name_5`
|
81
|
+
|
82
|
+
* Value type is <<array,array>>
|
83
|
+
* Default value is {}
|
84
|
+
|
85
|
+
Add description here
|
86
|
+
|
87
|
+
[id="plugins-{type}s-{plugin}-setting_name_6"]
|
88
|
+
===== `setting_name_6`
|
89
|
+
|
90
|
+
* Value type is <<bytes,bytes>>
|
91
|
+
* Default value is {}
|
92
|
+
|
93
|
+
Add description here
|
94
|
+
|
95
|
+
[id="plugins-{type}s-{plugin}-setting_name_7"]
|
96
|
+
===== `setting_name_7`
|
97
|
+
|
98
|
+
* Value type is <<path,path>>
|
99
|
+
* Default value is {}
|
100
|
+
|
101
|
+
Add description here
|
102
|
+
|
103
|
+
[id="plugins-{type}s-{plugin}-setting_name_8"]
|
104
|
+
===== `setting_name_8`
|
105
|
+
|
106
|
+
* Value type is <<password,password>>
|
107
|
+
* Default value is {}
|
108
|
+
|
109
|
+
Add description here
|
110
|
+
|
111
|
+
// The full list of Value Types is here:
|
112
|
+
// https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
|
113
|
+
|
114
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
115
|
+
include::{include_path}/{type}.asciidoc[]
|
116
|
+
|
117
|
+
:default_codec!:
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
|
2
|
+
# encoding: utf-8
|
3
|
+
require "logstash/inputs/base"
|
4
|
+
require "logstash/namespace"
|
5
|
+
require "logstash-input-cloudwatch_logs_importer_jars"
|
6
|
+
require "java"
|
7
|
+
|
8
|
+
class LogStash::Inputs::CloudwatchLogsImporter < LogStash::Inputs::Base
|
9
|
+
config_name "cloudwatch_logs_importer"
|
10
|
+
|
11
|
+
def self.javaClass() Java::net.justmachinery.CloudwatchLogsImporter.java_class; end
|
12
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
|
2
|
+
Gem::Specification.new do |s|
|
3
|
+
s.name = 'logstash-input-cloudwatch_logs_importer'
|
4
|
+
s.version = ::File.read('VERSION').split('\n').first
|
5
|
+
s.licenses = ['Apache-2.0']
|
6
|
+
s.summary = 'Imports log data from Cloudwatch Logs'
|
7
|
+
s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
|
8
|
+
s.authors = ['Elasticsearch']
|
9
|
+
s.email = ['info@elastic.co']
|
10
|
+
s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
|
11
|
+
s.require_paths = ['lib', 'vendor/jar-dependencies']
|
12
|
+
|
13
|
+
s.files = Dir["lib/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
14
|
+
|
15
|
+
# Special flag to let us know this is actually a logstash plugin
|
16
|
+
s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input', 'java_plugin' => 'true'}
|
17
|
+
|
18
|
+
# Gem dependencies
|
19
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
20
|
+
s.add_runtime_dependency 'jar-dependencies'
|
21
|
+
s.add_development_dependency 'logstash-devutils'
|
22
|
+
end
|
Binary file
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-input-cloudwatch_logs_importer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elasticsearch
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-11-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core-plugin-api
|
23
|
+
prerelease: false
|
24
|
+
type: :runtime
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
name: jar-dependencies
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '0'
|
53
|
+
name: logstash-devutils
|
54
|
+
prerelease: false
|
55
|
+
type: :development
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
62
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
63
|
+
gem is not a stand-alone program
|
64
|
+
email:
|
65
|
+
- info@elastic.co
|
66
|
+
executables: []
|
67
|
+
extensions: []
|
68
|
+
extra_rdoc_files: []
|
69
|
+
files:
|
70
|
+
- Gemfile
|
71
|
+
- README.md
|
72
|
+
- VERSION
|
73
|
+
- docs/index.asciidoc
|
74
|
+
- lib/logstash-input-cloudwatch_logs_importer_jars.rb
|
75
|
+
- lib/logstash/inputs/cloudwatch_logs_importer.rb
|
76
|
+
- logstash-input-cloudwatch_logs_importer.gemspec
|
77
|
+
- vendor/jar-dependencies/net/justmachinery/logstash-input-cloudwatch_logs_importer/1.0.0/logstash-input-cloudwatch_logs_importer-1.0.0.jar
|
78
|
+
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
79
|
+
licenses:
|
80
|
+
- Apache-2.0
|
81
|
+
metadata:
|
82
|
+
logstash_plugin: 'true'
|
83
|
+
logstash_group: input
|
84
|
+
java_plugin: 'true'
|
85
|
+
post_install_message:
|
86
|
+
rdoc_options: []
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
- vendor/jar-dependencies
|
90
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - ">="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
requirements: []
|
101
|
+
rubyforge_project:
|
102
|
+
rubygems_version: 2.7.9
|
103
|
+
signing_key:
|
104
|
+
specification_version: 4
|
105
|
+
summary: Imports log data from Cloudwatch Logs
|
106
|
+
test_files: []
|