logstash-output-qingstor 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -117
- data/README_zh_CN.md +113 -0
- data/lib/logstash/outputs/qingstor.rb +1 -1
- data/logstash-output-qingstor.gemspec +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7edeee9f910f1ff39de1856a36166c30ad66052e
|
|
4
|
+
data.tar.gz: 31b21ceee52a6402f89fa782f368450e1f84ebd4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 088f7febcb4b81f11d956c52dc5aa100e01eb7cd4d50670952b9f24ac2b54f24447b17a74900697d23cea1b5b86b42e82972be48b4b92ada16e19b081c7e2e41
|
|
7
|
+
data.tar.gz: 4467d04227d805798d58f7806ae6f0f8a59cd7b81070126ecb9462104798474d0f508fd705dead75fa86c87607aa2a7c784224fd4270fb75c27ef3fdb77ffd2d
|
data/README.md
CHANGED
|
@@ -1,121 +1,6 @@
|
|
|
1
|
-
# Logstash Output Plugin for
|
|
1
|
+
# Logstash Output Plugin for QingStor
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
Qingstor是[Qingcloud](https://www.qingcloud.com/)推出的对象存储服务.
|
|
5
|
-
作为一个output插件, 可以将logstash输出的结果打包上传至Qingstor中.
|
|
6
|
-
详细功能参考下面配置说明.
|
|
7
|
-
|
|
8
|
-
目前插件已经提交至rubygems.org, 使用以下命令安装:
|
|
9
|
-
```sh
|
|
10
|
-
bin/logstash-plugin install logstash-output-qingstor
|
|
11
|
-
```
|
|
12
|
-
手动安装本地代码, 安装方法参考下文.
|
|
13
|
-
|
|
14
|
-
## 1. 配置说明
|
|
15
|
-
|
|
16
|
-
#### 1.1 最小运行配置
|
|
17
|
-
- 使用'-f'接受一个*.conf文件或者使用'-e'参数时, 最小运行配置至少需要以下三项
|
|
18
|
-
```sh
|
|
19
|
-
output {
|
|
20
|
-
qingstor {
|
|
21
|
-
access_key_id => 'your_access_key_id' #required
|
|
22
|
-
secret_access_key => 'your_secret_access_key' #required
|
|
23
|
-
bucket => 'bucket_name' #required
|
|
24
|
-
# region => "pek3a" #optional, default value "pek3a"
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
#### 1.2 其他可选参数说明
|
|
31
|
-
```sh
|
|
32
|
-
output {
|
|
33
|
-
qingstor {
|
|
34
|
-
......
|
|
35
|
-
# 前缀, 生成本地子目录, 并且添加到上传文件名的前半部分, 形成Qingstor的目录.
|
|
36
|
-
# 默认空,
|
|
37
|
-
prefix => 'aprefix'
|
|
38
|
-
|
|
39
|
-
# 本地保存临时文件的目录.
|
|
40
|
-
# 默认: 系统临时文件目录下的logstash2qingstor文件夹, 例如linux下 "/tmp/logstash2qingstor"
|
|
41
|
-
tmpdir => '/local/temporary/directory'
|
|
42
|
-
|
|
43
|
-
# 字符串数组, 添加到文件名中, 例如["a", "b", "c"], 文件名会形成如 xxx-a-b-c-xxx.xx
|
|
44
|
-
# 默认: 空
|
|
45
|
-
tags => ["tag1", "tag2", "tag3"]
|
|
46
|
-
|
|
47
|
-
# 上传文件的格式. 可选"gzip", 后缀为".gz".
|
|
48
|
-
# 默认: "none", 后缀".log".
|
|
49
|
-
encoding => "gzip"
|
|
50
|
-
|
|
51
|
-
# 文件上传的策略.
|
|
52
|
-
# 分别表示结合文件大小和时间的综合策略, 基于文件大小的策略, 以及基于时间的策略.
|
|
53
|
-
# 基于文件大小的策略表示: 当文件大小满足预设值之后, 将文件上传.
|
|
54
|
-
# 基于时间的策略表示: 每经过预设时间之后, 将文件上传.
|
|
55
|
-
# 默认: "size_and_time". 可选枚举值["size_and_time", "size", "time"].
|
|
56
|
-
rotation_strategy => "size_and_time"
|
|
57
|
-
|
|
58
|
-
# 配合"size_and_time", "size"的可选配置型, 单位byte
|
|
59
|
-
# 默认: 1024*1024*5 (byte) = 5 (MB)
|
|
60
|
-
size_file => 1024*1024*5
|
|
61
|
-
|
|
62
|
-
# 配合"size_and_time", "time"的可选配置型, 单位minute
|
|
63
|
-
# 默认: 15 (minutes)
|
|
64
|
-
time_file => 15
|
|
65
|
-
|
|
66
|
-
# 服务端文件加密, 支持AES256.
|
|
67
|
-
# 默认: "none". 可选枚举值: ["AES256", "none"]
|
|
68
|
-
server_side_encryption_algorithm => "AES256"
|
|
69
|
-
|
|
70
|
-
# 选用服务端文件加密时提供的秘钥, 秘钥要求32位/256bit
|
|
71
|
-
customer_key => "your_encryption_key"
|
|
72
|
-
|
|
73
|
-
# 宕机恢复, 启动logstash时, 自动上传目录下的遗留文件
|
|
74
|
-
# 默认: false
|
|
75
|
-
restore => true
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
## 2. 安装插件
|
|
83
|
-
|
|
84
|
-
#### 2.1 直接运行本地的插件
|
|
85
|
-
|
|
86
|
-
- 编辑Logstash目录下的Genfile, 添加插件的路径, 例如
|
|
87
|
-
```ruby
|
|
88
|
-
gem "logstash-output-qingstor", :path => "/your/local/logstash-output-qingstor"
|
|
89
|
-
```
|
|
90
|
-
- 安装插件
|
|
91
|
-
```sh
|
|
92
|
-
bin/logstash-plugin install --no-verify
|
|
93
|
-
```
|
|
94
|
-
- 使用插件运行
|
|
95
|
-
```sh
|
|
96
|
-
bin/logstash -e "output {qingstor {access_key_id => 'your_access_key_id'
|
|
97
|
-
secret_access_key => 'your_secret_access_key'
|
|
98
|
-
bucket => 'bucket_name' }}"
|
|
99
|
-
```
|
|
100
|
-
此时你对插件所做的任意的代码上的修改都会直接生效.
|
|
101
|
-
|
|
102
|
-
#### 2.2 安装一个本地插件然后运行
|
|
103
|
-
|
|
104
|
-
这一步你需要生成一个插件的gem包, 然后通过logstash来安装到logstash的插件目录下
|
|
105
|
-
- 在项目目录下生成gem
|
|
106
|
-
```sh
|
|
107
|
-
gem build logstash-output-qingstor.gemspec
|
|
108
|
-
```
|
|
109
|
-
- 在Logstash的目录下使用logstash-plugin安装
|
|
110
|
-
```sh
|
|
111
|
-
bin/logstash-plugin install /your/local/plugin/logstash-output-qingstor.gem
|
|
112
|
-
```
|
|
113
|
-
- 安装完毕之后, 就可以使用Logstash运行开始测试了.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
# Logstash Output Plugin for Qingstor
|
|
117
|
-
|
|
118
|
-
As an output plugin, it can collect the outputs from logstash, and store them in [Qingstor](https://www.qingcloud.com/products/storage#qingstor), which is a remarkable object storage service provided by [Qingcloud](https://www.qingcloud.com/).
|
|
3
|
+
As an output plugin, it can collect the outputs from logstash, and store them in [QingStor](https://www.qingcloud.com/products/storage#qingstor), which is a remarkable object storage service provided by [QingCloud](https://www.qingcloud.com/).
|
|
119
4
|
|
|
120
5
|
For now, We've submitted this plugin to rubygems.org. Use the following command to install.
|
|
121
6
|
```sh
|
|
@@ -123,6 +8,7 @@ For now, We've submitted this plugin to rubygems.org. Use the following command
|
|
|
123
8
|
```
|
|
124
9
|
If want to install the local code, please refer to the following guide to install it manually.
|
|
125
10
|
|
|
11
|
+
Simplified Chinese version README can be found [here](/README_zh_CN.md)
|
|
126
12
|
## 1. Configuration Guide
|
|
127
13
|
|
|
128
14
|
#### 1.1 Run in minimal Configuration Items
|
data/README_zh_CN.md
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# Logstash Output Plugin for QingStor
|
|
2
|
+
|
|
3
|
+
这是一个适配了[QingStor](https://www.qingcloud.com/products/storage#qingstor), 工作在logstash中的output插件.
|
|
4
|
+
QingStor是[QingCloud](https://www.qingcloud.com/)推出的对象存储服务.
|
|
5
|
+
作为一个output插件, 可以将logstash输出的结果打包上传至QingStor中.
|
|
6
|
+
详细功能参考下面配置说明.
|
|
7
|
+
|
|
8
|
+
目前插件已经提交至rubygems.org, 使用以下命令安装:
|
|
9
|
+
```sh
|
|
10
|
+
bin/logstash-plugin install logstash-output-qingstor
|
|
11
|
+
```
|
|
12
|
+
手动安装本地代码, 安装方法参考下文.
|
|
13
|
+
|
|
14
|
+
## 1. 配置说明
|
|
15
|
+
|
|
16
|
+
#### 1.1 最小运行配置
|
|
17
|
+
- 使用'-f'接受一个*.conf文件或者使用'-e'参数时, 最小运行配置至少需要以下三项
|
|
18
|
+
```sh
|
|
19
|
+
output {
|
|
20
|
+
qingstor {
|
|
21
|
+
access_key_id => 'your_access_key_id' #required
|
|
22
|
+
secret_access_key => 'your_secret_access_key' #required
|
|
23
|
+
bucket => 'bucket_name' #required
|
|
24
|
+
# region => "pek3a" #optional, default value "pek3a"
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
#### 1.2 其他可选参数说明
|
|
31
|
+
```sh
|
|
32
|
+
output {
|
|
33
|
+
qingstor {
|
|
34
|
+
......
|
|
35
|
+
# 前缀, 生成本地子目录, 并且添加到上传文件名的前半部分, 形成QingStor的目录.
|
|
36
|
+
# 默认空,
|
|
37
|
+
prefix => 'aprefix'
|
|
38
|
+
|
|
39
|
+
# 本地保存临时文件的目录.
|
|
40
|
+
# 默认: 系统临时文件目录下的logstash2qingstor文件夹, 例如linux下 "/tmp/logstash2qingstor"
|
|
41
|
+
tmpdir => '/local/temporary/directory'
|
|
42
|
+
|
|
43
|
+
# 字符串数组, 添加到文件名中, 例如["a", "b", "c"], 文件名会形成如 xxx-a-b-c-xxx.xx
|
|
44
|
+
# 默认: 空
|
|
45
|
+
tags => ["tag1", "tag2", "tag3"]
|
|
46
|
+
|
|
47
|
+
# 上传文件的格式. 可选"gzip", 后缀为".gz".
|
|
48
|
+
# 默认: "none", 后缀".log".
|
|
49
|
+
encoding => "gzip"
|
|
50
|
+
|
|
51
|
+
# 文件上传的策略.
|
|
52
|
+
# 分别表示结合文件大小和时间的综合策略, 基于文件大小的策略, 以及基于时间的策略.
|
|
53
|
+
# 基于文件大小的策略表示: 当文件大小满足预设值之后, 将文件上传.
|
|
54
|
+
# 基于时间的策略表示: 每经过预设时间之后, 将文件上传.
|
|
55
|
+
# 默认: "size_and_time". 可选枚举值["size_and_time", "size", "time"].
|
|
56
|
+
rotation_strategy => "size_and_time"
|
|
57
|
+
|
|
58
|
+
# 配合"size_and_time", "size"的可选配置型, 单位byte
|
|
59
|
+
# 默认: 1024*1024*5 (byte) = 5 (MB)
|
|
60
|
+
size_file => 1024*1024*5
|
|
61
|
+
|
|
62
|
+
# 配合"size_and_time", "time"的可选配置型, 单位minute
|
|
63
|
+
# 默认: 15 (minutes)
|
|
64
|
+
time_file => 15
|
|
65
|
+
|
|
66
|
+
# 服务端文件加密, 支持AES256.
|
|
67
|
+
# 默认: "none". 可选枚举值: ["AES256", "none"]
|
|
68
|
+
server_side_encryption_algorithm => "AES256"
|
|
69
|
+
|
|
70
|
+
# 选用服务端文件加密时提供的秘钥, 秘钥要求32位/256bit
|
|
71
|
+
customer_key => "your_encryption_key"
|
|
72
|
+
|
|
73
|
+
# 宕机恢复, 启动logstash时, 自动上传目录下的遗留文件
|
|
74
|
+
# 默认: false
|
|
75
|
+
restore => true
|
|
76
|
+
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## 2. 安装插件
|
|
83
|
+
|
|
84
|
+
#### 2.1 直接运行本地的插件
|
|
85
|
+
|
|
86
|
+
- 编辑Logstash目录下的Genfile, 添加插件的路径, 例如
|
|
87
|
+
```ruby
|
|
88
|
+
gem "logstash-output-qingstor", :path => "/your/local/logstash-output-qingstor"
|
|
89
|
+
```
|
|
90
|
+
- 安装插件
|
|
91
|
+
```sh
|
|
92
|
+
bin/logstash-plugin install --no-verify
|
|
93
|
+
```
|
|
94
|
+
- 使用插件运行
|
|
95
|
+
```sh
|
|
96
|
+
bin/logstash -e "output {qingstor {access_key_id => 'your_access_key_id'
|
|
97
|
+
secret_access_key => 'your_secret_access_key'
|
|
98
|
+
bucket => 'bucket_name' }}"
|
|
99
|
+
```
|
|
100
|
+
此时你对插件所做的任意的代码上的修改都会直接生效.
|
|
101
|
+
|
|
102
|
+
#### 2.2 安装一个本地插件然后运行
|
|
103
|
+
|
|
104
|
+
这一步你需要生成一个插件的gem包, 然后通过logstash来安装到logstash的插件目录下
|
|
105
|
+
- 在项目目录下生成gem
|
|
106
|
+
```sh
|
|
107
|
+
gem build logstash-output-qingstor.gemspec
|
|
108
|
+
```
|
|
109
|
+
- 在Logstash的目录下使用logstash-plugin安装
|
|
110
|
+
```sh
|
|
111
|
+
bin/logstash-plugin install /your/local/plugin/logstash-output-qingstor.gem
|
|
112
|
+
```
|
|
113
|
+
- 安装完毕之后, 就可以使用Logstash运行开始测试了.
|
|
@@ -127,7 +127,7 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
|
|
|
127
127
|
prefix_written_to << prefix_key
|
|
128
128
|
|
|
129
129
|
begin
|
|
130
|
-
@file_repository.get_file(prefix_key) { |file| file.write(encoded) }
|
|
130
|
+
@file_repository.get_file(prefix_key) { |file| file.write(encoded + "\n") }
|
|
131
131
|
rescue Errno::ENOSPC => e
|
|
132
132
|
@logger.error("QingStor: Nospace left in temporary directory", :tmpdir => @tmpdir)
|
|
133
133
|
raise e
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-output-qingstor'
|
|
3
|
-
s.version = '0.1.
|
|
3
|
+
s.version = '0.1.3'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = 'logstash output plugin for qingstor'
|
|
6
6
|
s.description = 'Collect the outputs of logstash and store into Qingstor'
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-output-qingstor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Evan Zhao
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-05-22 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -108,6 +108,7 @@ files:
|
|
|
108
108
|
- Gemfile
|
|
109
109
|
- LICENSE
|
|
110
110
|
- README.md
|
|
111
|
+
- README_zh_CN.md
|
|
111
112
|
- lib/logstash/outputs/qingstor.rb
|
|
112
113
|
- lib/logstash/outputs/qingstor/file_repository.rb
|
|
113
114
|
- lib/logstash/outputs/qingstor/qingstor_validator.rb
|