logstash-input-qingstor 0.1.3 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -1
- data/README.md +5 -121
- data/README_zh_CN.md +116 -0
- data/lib/logstash/inputs/qingstor.rb +103 -177
- data/lib/logstash/inputs/qingstor/log_reader.rb +53 -0
- data/lib/logstash/inputs/qingstor/qingstor_validator.rb +32 -27
- data/lib/logstash/inputs/qingstor/sincedb.rb +36 -0
- data/lib/logstash/inputs/qingstor/uploader.rb +56 -0
- data/logstash-input-qingstor.gemspec +11 -9
- data/spec/logstash/inputs/qingstor/log_reader_spec.rb +48 -0
- data/spec/logstash/inputs/qingstor/sincedb_spec.rb +27 -0
- data/spec/logstash/inputs/qingstor/uploader_spec.rb +42 -0
- data/spec/logstash/inputs/qingstor_spec.rb +98 -0
- data/spec/{inputs → logstash/inputs}/qs_access_helper.rb +12 -14
- metadata +18 -10
- data/spec/inputs/qingstor_spec.rb +0 -66
- data/spec/inputs/qingstor_spec_validator_spec.rb +0 -37
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6bcfff3cc30d80ea28ba519311f4fa572ae10bdb
|
4
|
+
data.tar.gz: 0775ebfb4fe462564e9430b1553bf32a47279a85
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 15985ef6e93965b02752c7de12f158269e648b3447ec517abd9fa2224dcec0f03bedb8177e1552460a4d8e6234bb5d01bcc0b9c6b3c1db0a21fe48e3cee851eb
|
7
|
+
data.tar.gz: bbca219710ce7f3357bbbbfa6eb1b5c9ef5d1b185ca64d0224513942010b12d43a1a4bca7d866c9639fcbfd95a7025bd9027ee905c1dc4f52c99db056617b7df
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,124 +1,7 @@
|
|
1
|
-
# Logstash
|
1
|
+
# Logstash Output Plugin for QingStor
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
详细功能参考下面配置说明.
|
6
|
-
|
7
|
-
目前插件已经提交至rubygems.org, 使用以下命令安装:
|
8
|
-
```sh
|
9
|
-
bin/logstash-plugin install logstash-input-qingstor
|
10
|
-
```
|
11
|
-
手动安装本地代码, 安装方法参考下文.
|
12
|
-
|
13
|
-
## 1. 配置说明
|
14
|
-
|
15
|
-
#### 1.1 最小运行配置
|
16
|
-
- 使用'-f' 接受一个*.conf文件或者使用'-e'参数最小运行配置时, 至少需要以下三项
|
17
|
-
```sh
|
18
|
-
input {
|
19
|
-
qingstor {
|
20
|
-
access_key_id => 'your_access_key_id' #required
|
21
|
-
secret_access_key => 'your_secret_access_key' #required
|
22
|
-
bucket => 'bucket_name' #required
|
23
|
-
# region => "pek3a" #optional, default value "pek3a"
|
24
|
-
}
|
25
|
-
}
|
26
|
-
|
27
|
-
```
|
28
|
-
|
29
|
-
#### 1.2 其他可选参数说明
|
30
|
-
```sh
|
31
|
-
input {
|
32
|
-
qingstor {
|
33
|
-
......
|
34
|
-
# 指定下载文件的前缀.
|
35
|
-
# 默认nil,
|
36
|
-
prefix => 'aprefix'
|
37
|
-
|
38
|
-
# 本地保存临时文件的目录.
|
39
|
-
# 默认: 系统临时文件目录下的qingstor2logstash文件夹, 例如linux下 "/tmp/qingstor2logstash".
|
40
|
-
tmpdir => '/local/temporary/directory'
|
41
|
-
|
42
|
-
# 是否在处理之后, 删除远程bucket中的文件.
|
43
|
-
# 默认: false
|
44
|
-
delete_remote_files => true
|
45
|
-
|
46
|
-
# 重新配置qingstor的地址
|
47
|
-
# 默认: nil
|
48
|
-
host => "new.qingstor.net"
|
49
|
-
|
50
|
-
# 重新配置qingstor地址的端口号
|
51
|
-
# 默认: 443
|
52
|
-
port => 443
|
53
|
-
|
54
|
-
# 如果指定一个本地目录, 那么在处理完之后将文件备份至该位置.
|
55
|
-
# 默认: nil
|
56
|
-
local_dir => 'your/local/directory'
|
57
|
-
|
58
|
-
# 如果指定了该值, 那么在处理完之后将文件上传到Qingstor指定的bucket中.
|
59
|
-
# 默认: nil
|
60
|
-
backup_bucket => 'backupbucket'
|
61
|
-
|
62
|
-
# 配合上一项使用, 指定备份bucket所在的region.
|
63
|
-
# 默认: "pek3a", 可选枚举值: ["pek3a", "sh1a"]
|
64
|
-
backup_region => "sh1a"
|
65
|
-
|
66
|
-
# 备份文件的前缀
|
67
|
-
# 默认: nil
|
68
|
-
backup_prefix => "logstash/backup"
|
69
|
-
|
70
|
-
# 指定一个sincedb的保存位置, sincedb用于记录上一次抓取文件的时间
|
71
|
-
# 没有指定时默认在用户HOME目录下创建
|
72
|
-
# 默认: nil
|
73
|
-
sincedb_path => "~/qingstor/.sincedb"
|
74
|
-
|
75
|
-
# 每次抓取的时间间隔, 单位秒
|
76
|
-
# 默认: 10(s)
|
77
|
-
interval => 30
|
78
|
-
|
79
|
-
}
|
80
|
-
}
|
81
|
-
|
82
|
-
```
|
83
|
-
|
84
|
-
## 2. 安装插件
|
85
|
-
|
86
|
-
#### 2.1 直接运行本地的插件
|
87
|
-
|
88
|
-
- 编辑Logstash目录下的Genfile, 添加插件的路径, 例如
|
89
|
-
```ruby
|
90
|
-
gem "logstash-input-qingstor", :path => "/your/local/logstash-input-qingstor"
|
91
|
-
```
|
92
|
-
- 安装插件
|
93
|
-
```sh
|
94
|
-
bin/logstash-plugin install --no-verify
|
95
|
-
```
|
96
|
-
- 使用插件运行
|
97
|
-
```sh
|
98
|
-
bin/logstash -e "input {qingstor { access_key_id => 'your_access_key_id'
|
99
|
-
secret_access_key => 'your_secret_access_key'
|
100
|
-
bucket => 'bucket_name' }}'
|
101
|
-
```
|
102
|
-
此时你对插件所做的任意的代码上的修改都会直接生效.
|
103
|
-
|
104
|
-
#### 2.2 安装一个本地插件然后运行
|
105
|
-
|
106
|
-
这一步你需要生成一个插件的gem包, 然后通过logstash来安装到logstash的插件目录下
|
107
|
-
- 在logstash-input-qingstor项目目录下生成gem
|
108
|
-
```sh
|
109
|
-
gem build logstash-input-qingstor.gemspec
|
110
|
-
```
|
111
|
-
- 在Logstash的目录下使用logstash-plugin安装
|
112
|
-
```sh
|
113
|
-
bin/logstash-plugin install /your/local/plugin/logstash-input-qingstor.gem
|
114
|
-
```
|
115
|
-
- 安装完毕之后, 就可以使用Logstash运行开始测试了.
|
116
|
-
|
117
|
-
|
118
|
-
# Logstash Output Plugin for Qingstor
|
119
|
-
|
120
|
-
As an input plugin, it can download the log files from [Qingstor](https://www.qingcloud.com/products/storage#qingstor), and read them into logstash.
|
121
|
-
Qingstor is a remarkable object storage service provided by [Qingcloud](https://www.qingcloud.com/).
|
3
|
+
As an input plugin, it can download the log files from [QingStor](https://www.qingcloud.com/products/storage#qingstor), and read them into logstash.
|
4
|
+
QingStor is a remarkable object storage service provided by [QingCloud](https://www.qingcloud.com/).
|
122
5
|
|
123
6
|
For now, We've submitted this plugin to rubygems.org. Use the following command to install.
|
124
7
|
```sh
|
@@ -126,6 +9,7 @@ For now, We've submitted this plugin to rubygems.org. Use the following command
|
|
126
9
|
```
|
127
10
|
If want to install the local code, please refer to the following guide to install it manually.
|
128
11
|
|
12
|
+
Simplified Chinese version README can be found [here](/README_zh_CN.md)
|
129
13
|
## 1. Configuration Guide
|
130
14
|
|
131
15
|
#### 1.1 Run in minimal Configuration Settings
|
@@ -141,7 +25,7 @@ input {
|
|
141
25
|
|
142
26
|
```
|
143
27
|
|
144
|
-
More configuration details please refer to [
|
28
|
+
More configuration details please refer to [common options](/docs/index.asciidoc).
|
145
29
|
|
146
30
|
## 2. Running your unpublished Plugin in Logstash
|
147
31
|
|
data/README_zh_CN.md
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
# Logstash Input Plugin for QingStor
|
2
|
+
|
3
|
+
这是一个适配了[QingStor](https://www.qingcloud.com/products/storage#qingstor), 工作在logstash中的input插件. QingStor是[QingCloud](https://www.qingcloud.com/)推出的对象存储服务.
|
4
|
+
作为一个input插件, 它能下载存储在QingStor上的日志文件, 读入到logstash中进行进一步处理.
|
5
|
+
详细功能参考下面配置说明.
|
6
|
+
|
7
|
+
目前插件已经提交至rubygems.org, 使用以下命令安装:
|
8
|
+
```sh
|
9
|
+
bin/logstash-plugin install logstash-input-qingstor
|
10
|
+
```
|
11
|
+
手动安装本地代码, 安装方法参考下文.
|
12
|
+
|
13
|
+
## 1. 配置说明
|
14
|
+
|
15
|
+
#### 1.1 最小运行配置
|
16
|
+
- 使用'-f' 接受一个*.conf文件或者使用'-e'参数最小运行配置时, 至少需要以下三项
|
17
|
+
```sh
|
18
|
+
input {
|
19
|
+
qingstor {
|
20
|
+
access_key_id => 'your_access_key_id' #required
|
21
|
+
secret_access_key => 'your_secret_access_key' #required
|
22
|
+
bucket => 'bucket_name' #required
|
23
|
+
# region => "pek3a" #optional, default value "pek3a"
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
```
|
28
|
+
|
29
|
+
#### 1.2 其他可选参数说明
|
30
|
+
```sh
|
31
|
+
input {
|
32
|
+
qingstor {
|
33
|
+
......
|
34
|
+
# 指定下载文件的前缀.
|
35
|
+
# 默认nil,
|
36
|
+
prefix => 'aprefix'
|
37
|
+
|
38
|
+
# 本地保存临时文件的目录.
|
39
|
+
# 默认: 系统临时文件目录下的qingstor2logstash文件夹, 例如linux下 "/tmp/qingstor2logstash".
|
40
|
+
tmpdir => '/local/temporary/directory'
|
41
|
+
|
42
|
+
# 是否在处理之后, 删除远程bucket中的文件.
|
43
|
+
# 默认: false
|
44
|
+
delete_remote_files => true
|
45
|
+
|
46
|
+
# 重新配置QingStor的地址
|
47
|
+
# 默认: nil
|
48
|
+
host => "new.qingstor.net"
|
49
|
+
|
50
|
+
# 重新配置QingStor地址的端口号
|
51
|
+
# 默认: 443
|
52
|
+
port => 443
|
53
|
+
|
54
|
+
# 如果指定一个本地目录, 那么在处理完之后将文件备份至该位置.
|
55
|
+
# 默认: nil
|
56
|
+
local_dir => 'your/local/directory'
|
57
|
+
|
58
|
+
# 如果指定了该值, 那么在处理完之后将文件上传到QingStor指定的bucket中.
|
59
|
+
# 默认: nil
|
60
|
+
backup_bucket => 'backupbucket'
|
61
|
+
|
62
|
+
# 配合上一项使用, 指定备份bucket所在的region.
|
63
|
+
# 默认: "pek3a", 可选枚举值: ["pek3a", "sh1a"]
|
64
|
+
backup_region => "sh1a"
|
65
|
+
|
66
|
+
# 备份文件的前缀
|
67
|
+
# 默认: nil
|
68
|
+
backup_prefix => "logstash/backup"
|
69
|
+
|
70
|
+
# 指定一个sincedb的保存位置, sincedb用于记录上一次抓取文件的时间
|
71
|
+
# 没有指定时默认在用户HOME目录下创建
|
72
|
+
# 默认: nil
|
73
|
+
sincedb_path => "~/qingstor/.sincedb"
|
74
|
+
|
75
|
+
# 每次抓取的时间间隔, 单位秒
|
76
|
+
# 默认: 10(s)
|
77
|
+
interval => 30
|
78
|
+
|
79
|
+
}
|
80
|
+
}
|
81
|
+
|
82
|
+
```
|
83
|
+
|
84
|
+
## 2. 安装插件
|
85
|
+
|
86
|
+
#### 2.1 直接运行本地的插件
|
87
|
+
|
88
|
+
- 编辑Logstash目录下的Genfile, 添加插件的路径, 例如
|
89
|
+
```ruby
|
90
|
+
gem "logstash-input-qingstor", :path => "/your/local/logstash-input-qingstor"
|
91
|
+
```
|
92
|
+
- 安装插件
|
93
|
+
```sh
|
94
|
+
bin/logstash-plugin install --no-verify
|
95
|
+
```
|
96
|
+
- 使用插件运行
|
97
|
+
```sh
|
98
|
+
bin/logstash -e "input {qingstor { access_key_id => 'your_access_key_id'
|
99
|
+
secret_access_key => 'your_secret_access_key'
|
100
|
+
bucket => 'bucket_name' }}'
|
101
|
+
```
|
102
|
+
此时你对插件所做的任意的代码上的修改都会直接生效.
|
103
|
+
|
104
|
+
#### 2.2 安装一个本地插件然后运行
|
105
|
+
|
106
|
+
这一步你需要生成一个插件的gem包, 然后通过logstash来安装到logstash的插件目录下
|
107
|
+
- 在logstash-input-qingstor项目目录下生成gem
|
108
|
+
```sh
|
109
|
+
gem build logstash-input-qingstor.gemspec
|
110
|
+
```
|
111
|
+
- 在Logstash的目录下使用logstash-plugin安装
|
112
|
+
```sh
|
113
|
+
bin/logstash-plugin install /your/local/plugin/logstash-input-qingstor.gem
|
114
|
+
```
|
115
|
+
- 安装完毕之后, 就可以使用Logstash运行开始测试了.
|
116
|
+
|
@@ -1,16 +1,22 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
|
8
|
-
|
1
|
+
require 'logstash/inputs/base'
|
2
|
+
require 'logstash/namespace'
|
3
|
+
require 'stud/interval'
|
4
|
+
require 'qingstor/sdk'
|
5
|
+
require 'fileutils'
|
6
|
+
require 'tmpdir'
|
7
|
+
|
8
|
+
# Logstash Inputs Plugin for QingStor
|
9
9
|
class LogStash::Inputs::Qingstor < LogStash::Inputs::Base
|
10
|
-
require
|
11
|
-
|
10
|
+
require 'logstash/inputs/qingstor/qingstor_validator'
|
11
|
+
require 'logstash/inputs/qingstor/sincedb'
|
12
|
+
require 'logstash/inputs/qingstor/uploader'
|
13
|
+
require 'logstash/inputs/qingstor/log_reader'
|
14
|
+
|
15
|
+
include QingstorValidator
|
16
|
+
|
17
|
+
config_name 'qingstor'
|
12
18
|
|
13
|
-
default :codec,
|
19
|
+
default :codec, 'plain'
|
14
20
|
|
15
21
|
# The key id to access your QingStor
|
16
22
|
config :access_key_id, :validate => :string, :required => true
|
@@ -24,246 +30,166 @@ class LogStash::Inputs::Qingstor < LogStash::Inputs::Base
|
|
24
30
|
# It specifies the host port, please coordinate with config 'host'.
|
25
31
|
config :port, :validate => :number, :default => 443
|
26
32
|
|
27
|
-
# The name of the
|
33
|
+
# The name of the QingStor bucket
|
28
34
|
config :bucket, :validate => :string, :required => true
|
29
35
|
|
30
36
|
# The region of the QingStor bucket
|
31
|
-
config :region, :validate => :string, :default =>
|
37
|
+
config :region, :validate => :string, :default => 'pek3a'
|
32
38
|
|
33
39
|
# If specified, it would only download the files with the specified prefix.
|
34
40
|
config :prefix, :validate => :string, :default => nil
|
35
41
|
|
36
|
-
# Set the directory where logstash store the tmp files before
|
37
|
-
# sending it to logstash, default directory in linux /tmp/
|
38
|
-
config :tmpdir, :validate => :string,
|
42
|
+
# Set the directory where logstash store the tmp files before
|
43
|
+
# sending it to logstash, default directory in linux /tmp/qingstor2logstash
|
44
|
+
config :tmpdir, :validate => :string,
|
45
|
+
:default => File.join(Dir.tmpdir, 'qingstor2logstash')
|
39
46
|
|
40
47
|
# If this set to true, the remote file will be deleted after processed
|
41
48
|
config :delete_remote_files, :validate => :boolean, :default => false
|
42
|
-
|
43
|
-
# If this set to
|
49
|
+
|
50
|
+
# If this set to a valid path, the file will be backup under a local path,
|
44
51
|
# please make sure you can access to this dir.
|
45
|
-
config :backup_local_dir, :validate => :string, :default =>
|
52
|
+
config :backup_local_dir, :validate => :string, :default => nil
|
46
53
|
|
47
54
|
# If specified, the file will be upload to this bucket of the given region
|
48
55
|
config :backup_bucket, :validate => :string, :default => nil
|
49
56
|
|
50
|
-
# Specified the backup region in
|
51
|
-
config :backup_region, :validate =>
|
57
|
+
# Specified the backup region in QingStor.
|
58
|
+
config :backup_region, :validate => :string, :default => 'pek3a'
|
59
|
+
|
60
|
+
# This prefix, specially in QingStor, will add before the backup filename
|
61
|
+
config :backup_prefix, :validate => :string, :default => ''
|
62
|
+
|
63
|
+
# Use sincedb to record the last download time
|
64
|
+
config :sincedb_path, :validate => :string, :default => nil
|
52
65
|
|
53
|
-
# This prefix, specially in qingstor, will add before the backup filename
|
54
|
-
config :backup_prefix, :validate => :string, :default => ""
|
55
|
-
|
56
|
-
# Use sincedb to record the last download time
|
57
|
-
config :sincedb_path, :validate => :string, :default => nil
|
58
|
-
|
59
66
|
# Set how frequently messages should be sent.
|
60
67
|
# The default, `10`, means send a message every 10 seconds.
|
61
68
|
config :interval, :validate => :number, :default => 10
|
62
69
|
|
63
|
-
public
|
64
70
|
def register
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
71
|
+
check_settings
|
72
|
+
@logger.info('Registering QingStor plugin', :bucket => @bucket,
|
73
|
+
:region => @region)
|
74
|
+
@qs_bucket = qs_bucket(@bucket, @region)
|
75
|
+
bucket_valid?(@qs_bucket)
|
76
|
+
unless @backup_bucket.nil?
|
77
|
+
backup_bucket = qs_bucket(@backup_bucket, @backup_region)
|
78
|
+
prefix_valid?(@backup_prefix) unless @backup_prefix.nil?
|
79
|
+
@uploader = Uploader.new(backup_bucket, @backup_prefix, @logger)
|
72
80
|
end
|
73
|
-
|
74
|
-
|
75
|
-
@logger.info "Registering QingStor plugin", :bucket => @bucket, :region => @region
|
76
|
-
|
77
|
-
@qs_config = QingStor::SDK::Config.init @access_key_id, @secret_access_key
|
78
|
-
@qs_config.update({ host: @host, port: @port }) unless @host.nil?
|
79
|
-
@qs_service = QingStor::SDK::Service.new @qs_config
|
80
|
-
@qs_bucket = @qs_service.bucket @bucket, @region
|
81
|
-
|
82
|
-
QingstorValidator.bucket_valid?(@qs_bucket)
|
83
|
-
QingstorValidator.prefix_valid?(@backup_prefix) unless @backup_prefix.nil?
|
84
|
-
|
81
|
+
@log_reader = LogReader.new('/a/tmp/path')
|
85
82
|
end # def register
|
86
83
|
|
87
84
|
def run(queue)
|
88
|
-
@logger.info
|
85
|
+
@logger.info('starting processing')
|
89
86
|
@current_thread = Thread.current
|
90
87
|
Stud.interval(@interval) do
|
91
88
|
process_files(queue)
|
92
89
|
end
|
93
90
|
end # def run
|
94
91
|
|
95
|
-
|
96
92
|
def process_files(queue)
|
97
93
|
objects = list_new_files
|
98
94
|
|
99
|
-
# For each record in objects, it will be download and read
|
100
|
-
# into logstash. Then it will do deletion or backup job
|
95
|
+
# For each record in objects, it will be download and read
|
96
|
+
# into logstash. Then it will do deletion or backup job
|
101
97
|
# according to preset flags.
|
102
|
-
objects.each do |key,
|
98
|
+
objects.each do |key, _|
|
103
99
|
process_log(queue, key)
|
104
|
-
|
100
|
+
@uploader.upload_async(key, @tmp_file_path) unless @backup_bucket.nil?
|
105
101
|
backup_to_local_dir unless @backup_local_dir.nil?
|
106
|
-
@qs_bucket.delete_object
|
102
|
+
@qs_bucket.delete_object(key) if @delete_remote_files
|
107
103
|
end
|
108
104
|
|
109
105
|
# if fetched nothing, just return
|
110
106
|
return if objects.empty?
|
111
|
-
|
112
|
-
sincedb.write(Time.at(
|
107
|
+
tmp_time = objects.values.max
|
108
|
+
sincedb.write(Time.at(tmp_time))
|
109
|
+
end
|
110
|
+
|
111
|
+
def check_settings
|
112
|
+
unless directory_valid?(@tmpdir)
|
113
|
+
raise(LogStash::ConfigurationError, 'ERROR: no permissions to access'\
|
114
|
+
+ '#{@tmpdir}')
|
115
|
+
end
|
116
|
+
unless @backup_local_dir.nil? ||
|
117
|
+
directory_valid?(@backup_local_dir)
|
118
|
+
raise(LogStash::ConfigurationError, 'ERROR: no permissions to access'\
|
119
|
+
+ '#{@backup_local_dir}')
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def stop
|
124
|
+
Stud.stop!(@current_thread) unless @current_thread.nil?
|
125
|
+
end
|
126
|
+
|
127
|
+
def directory_valid?(path)
|
128
|
+
FileUtils.mkdir_p(path) unless Dir.exist?(path)
|
129
|
+
::File.writable?(path)
|
130
|
+
rescue
|
131
|
+
false
|
132
|
+
end
|
133
|
+
|
134
|
+
private
|
135
|
+
|
136
|
+
def qs_bucket(bucket, region)
|
137
|
+
config = QingStor::SDK::Config.init(@access_key_id, @secret_access_key)
|
138
|
+
config.update(:host => @host, :port => @port) unless @host.nil?
|
139
|
+
service = QingStor::SDK::Service.new(config)
|
140
|
+
service.bucket(bucket, region)
|
113
141
|
end
|
114
142
|
|
115
|
-
private
|
116
143
|
def list_new_files
|
117
144
|
objects = {}
|
118
|
-
@logger.info
|
119
|
-
res = @qs_bucket.list_objects(
|
145
|
+
@logger.info('starting fetching objects')
|
146
|
+
res = @qs_bucket.list_objects('prefix' => @prefix)
|
120
147
|
res[:keys].each do |log|
|
121
|
-
next unless
|
148
|
+
next unless @log_reader.valid_format?(log[:key])
|
122
149
|
next unless sincedb.newer?(log[:modified])
|
123
150
|
objects[log[:key]] = log[:modified]
|
124
|
-
@logger.info(
|
151
|
+
@logger.info('QingStor input: Adding to objects:', :key => log[:key])
|
125
152
|
end
|
126
|
-
|
153
|
+
objects
|
127
154
|
end
|
128
155
|
|
129
156
|
def process_log(queue, key)
|
130
|
-
# a global var, for the next possible upload and copy job
|
157
|
+
# a global var, for the next possible upload and copy job
|
131
158
|
@tmp_file_path = File.join(@tmpdir, File.basename(key))
|
132
|
-
|
133
159
|
File.open(@tmp_file_path, 'wb') do |logfile|
|
134
|
-
logfile.write
|
160
|
+
logfile.write(@qs_bucket.get_object(key)[:body])
|
135
161
|
end
|
136
162
|
process_local_log(queue, @tmp_file_path)
|
137
163
|
end
|
138
|
-
|
139
|
-
def backup_to_bucket(key)
|
140
|
-
properties = {'bucket-name' => @backup_bucket, 'zone' => @backup_region}
|
141
|
-
bucket = QingStor::SDK::Bucket.new @qs_config, properties
|
142
|
-
|
143
|
-
if bucket.head[:status_code] != 200
|
144
|
-
res = bucket.put
|
145
|
-
if res[:status_code] != 201
|
146
|
-
@logger.error("ERROR : cannot create the bucket ", res[:message])
|
147
|
-
raise LogStash::ConfigurationError, "cannot create the bucket"
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
md5_string = Digest::MD5.file(@tmp_file_path).to_s
|
152
|
-
|
153
|
-
new_key = if @backup_prefix.end_with?('/') || @backup_prefix.empty?
|
154
|
-
@backup_prefix + key
|
155
|
-
else
|
156
|
-
@backup_prefix + '/' + key
|
157
|
-
end
|
158
|
-
|
159
|
-
bucket.put_object new_key, {
|
160
|
-
'content_md5' => md5_string,
|
161
|
-
'body' => File.open(@tmp_file_path)
|
162
|
-
}
|
163
|
-
end
|
164
|
-
|
164
|
+
|
165
165
|
def backup_to_local_dir
|
166
166
|
FileUtils.mkdir_p @backup_local_dir unless File.exist? @backup_local_dir
|
167
167
|
FileUtils.cp @tmp_file_path, @backup_local_dir
|
168
168
|
end
|
169
|
-
|
169
|
+
|
170
170
|
def process_local_log(queue, filename)
|
171
|
-
|
171
|
+
@log_reader.filepath = filename
|
172
|
+
@log_reader.read_file do |line|
|
172
173
|
@codec.decode(line) do |event|
|
173
174
|
decorate(event)
|
174
175
|
queue << event
|
175
176
|
end
|
176
177
|
end
|
177
178
|
end
|
178
|
-
|
179
|
-
def read_file(filename, &block)
|
180
|
-
if gzip?(filename)
|
181
|
-
read_gzip_file(filename, block)
|
182
|
-
else
|
183
|
-
read_plain_file(filename, block)
|
184
|
-
end
|
185
|
-
end
|
186
|
-
|
187
|
-
def read_gzip_file(filename, block)
|
188
|
-
begin
|
189
|
-
Zlib::GzipReader.open(filename) do |decoder|
|
190
|
-
decoder.each_line { |line| block.call(line) }
|
191
|
-
end
|
192
|
-
rescue Zlib::Error, Zlib::GzipFile::Error => e
|
193
|
-
@logger.error("Gzip codec: Cannot uncompress the file", :filename => filename)
|
194
|
-
raise e
|
195
|
-
end
|
196
|
-
end
|
197
|
-
|
198
|
-
def read_plain_file(filename, block)
|
199
|
-
File.open(filename, 'rb') do |file|
|
200
|
-
file.each(&block)
|
201
|
-
end
|
202
|
-
end
|
203
179
|
|
204
|
-
def is_desired?(filename)
|
205
|
-
return logger?(filename) || gzip?(filename)
|
206
|
-
end
|
207
|
-
|
208
|
-
def logger?(filename)
|
209
|
-
return filename.end_with?('.log')
|
210
|
-
end
|
211
|
-
|
212
|
-
def gzip?(filename)
|
213
|
-
return filename.end_with?('.gz')
|
214
|
-
end
|
215
|
-
|
216
180
|
def sincedb
|
217
181
|
@sincedb ||= if @sincedb_path.nil?
|
218
|
-
|
219
|
-
|
182
|
+
@logger.info('Using default path for the sincedb',
|
183
|
+
:filename => sincedb_file)
|
184
|
+
SinceDB.new(sincedb_file)
|
220
185
|
else
|
221
|
-
|
222
|
-
|
186
|
+
@logger.info('Using the provided sincedb_path')
|
187
|
+
SinceDB.new(@sincedb_path)
|
223
188
|
end
|
224
|
-
end
|
225
|
-
|
226
|
-
def sincedb_file
|
227
|
-
File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
|
228
189
|
end
|
229
190
|
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
@sincedb_path = file
|
234
|
-
end
|
235
|
-
|
236
|
-
def newer?(date)
|
237
|
-
Time.at(date) > read
|
238
|
-
end
|
239
|
-
|
240
|
-
def read
|
241
|
-
if ::File.exist?(@sincedb_path)
|
242
|
-
content = ::File.read(@sincedb_path).chomp.strip
|
243
|
-
return content.empty? ? Time.new(0) : Time.parse(content)
|
244
|
-
else
|
245
|
-
return Time.new(0)
|
246
|
-
end
|
247
|
-
end
|
248
|
-
|
249
|
-
def write(since = nil)
|
250
|
-
since = Time.now() if since.nil?
|
251
|
-
::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
|
252
|
-
end
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
public
|
257
|
-
def stop
|
258
|
-
Stud.stop!(@current_thread) unless @current_thread.nil?
|
191
|
+
def sincedb_file
|
192
|
+
File.join(ENV['HOME'],
|
193
|
+
'.sincedb_' + Digest::MD5.hexdigest('#{@bucket}+#{@prefix}'))
|
259
194
|
end
|
260
|
-
|
261
|
-
def directory_valid?(path)
|
262
|
-
begin
|
263
|
-
FileUtils.mkdir_p(path) unless Dir.exist?(path)
|
264
|
-
::File.writable?(path)
|
265
|
-
rescue
|
266
|
-
false
|
267
|
-
end
|
268
|
-
end
|
269
195
|
end # class LogStash::Inputs::Qingstor
|