fluent-plugin-rackspace-cloud-files 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +11 -0
- data/.travis.yml +1 -1
- data/README.rdoc +1 -0
- data/VERSION +1 -1
- data/fluent-plugin-rackspace-cloud-files.gemspec +1 -1
- data/lib/fluent/plugin/out_rackspace_cloud_files.rb +102 -91
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e8adc0ac521e3f0645f1e790a383bdb5b25cff93
|
4
|
+
data.tar.gz: ad67f03f2cd45d669c5bca4a6cd6a9ea8e288b39
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 57a75f07a66991d52d32e8f814ff4c9f9d4a9d2779cf2598d5c4d09f9fad1ab7178b982cfa46e410358950b010036fc73dff3d5461aa5b34f67f0cb8286cf3a9
|
7
|
+
data.tar.gz: 1e3f74b993b87be01e675ff60c5863f299f60f0252b2a8fac4d35cc369eb1f934924edb3b8c053caaec905e215a5fc0c5eac88ae0d744e40eeaf2d8e4ee76d6c
|
data/.rubocop.yml
ADDED
data/.travis.yml
CHANGED
data/README.rdoc
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
|
5
5
|
{<img src="https://travis-ci.org/brint/fluent-plugin-rackspace-cloud-files.svg" alt="Build Status" />}[https://travis-ci.org/brint/fluent-plugin-rackspace-cloud-files]
|
6
6
|
{<img src="https://badge.fury.io/rb/fluent-plugin-rackspace-cloud-files.svg" alt="Gem Version" />}[http://badge.fury.io/rb/fluent-plugin-rackspace-cloud-files]
|
7
|
+
{<img src="https://gemnasium.com/brint/fluent-plugin-rackspace-cloud-files.svg" alt="Dependency Status" />}[https://gemnasium.com/brint/fluent-plugin-rackspace-cloud-files]
|
7
8
|
|
8
9
|
This has only been tested on fluentd version 0.12.12.
|
9
10
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.4
|
@@ -26,5 +26,5 @@ Gem::Specification.new do |gem|
|
|
26
26
|
'>= 0.3.0'
|
27
27
|
gem.add_development_dependency 'rake', '~> 10.1', '>= 10.1.0'
|
28
28
|
gem.add_development_dependency 'flexmock', '~> 1.2', '>= 1.2.0'
|
29
|
-
gem.add_development_dependency 'rubocop', '~> 0.33
|
29
|
+
gem.add_development_dependency 'rubocop', '~> 0.33', '>= 0.33'
|
30
30
|
end
|
@@ -1,8 +1,7 @@
|
|
1
|
-
module Fluent
|
2
|
-
|
1
|
+
module Fluent #:nodoc: all
|
3
2
|
require 'fluent/mixin/config_placeholders'
|
4
3
|
|
5
|
-
class RackspaceCloudFilesOutput < TimeSlicedOutput
|
4
|
+
class RackspaceCloudFilesOutput < Fluent::TimeSlicedOutput
|
6
5
|
Fluent::Plugin.register_output('rackspace_cloud_files', self)
|
7
6
|
|
8
7
|
def initialize
|
@@ -11,11 +10,10 @@ module Fluent
|
|
11
10
|
require 'zlib'
|
12
11
|
require 'time'
|
13
12
|
require 'tempfile'
|
14
|
-
require 'open3'
|
15
13
|
end
|
16
14
|
|
17
|
-
config_param :path, :string, :
|
18
|
-
config_param :time_format, :string, :
|
15
|
+
config_param :path, :string, default: ''
|
16
|
+
config_param :time_format, :string, default: nil
|
19
17
|
|
20
18
|
include SetTagKeyMixin
|
21
19
|
config_set_default :include_tag_key, false
|
@@ -23,18 +21,21 @@ module Fluent
|
|
23
21
|
include SetTimeKeyMixin
|
24
22
|
config_set_default :include_time_key, false
|
25
23
|
|
26
|
-
config_param :rackspace_auth_url, :string,
|
24
|
+
config_param :rackspace_auth_url, :string,
|
25
|
+
default: 'https://identity.api.rackspacecloud.com/v2.0'
|
27
26
|
config_param :rackspace_username, :string
|
28
27
|
config_param :rackspace_api_key, :string
|
29
28
|
config_param :rackspace_container, :string
|
30
29
|
config_param :rackspace_region, :string
|
31
30
|
|
32
|
-
config_param :object_key_format, :string,
|
33
|
-
|
34
|
-
config_param :
|
35
|
-
config_param :
|
36
|
-
config_param :
|
37
|
-
config_param :
|
31
|
+
config_param :object_key_format, :string,
|
32
|
+
default: '%{path}%{time_slice}_%{index}.%{file_extension}'
|
33
|
+
config_param :store_as, :string, default: 'gzip'
|
34
|
+
config_param :auto_create_container, :bool, default: true
|
35
|
+
config_param :check_apikey_on_start, :bool, default: true
|
36
|
+
config_param :proxy_uri, :string, default: nil
|
37
|
+
config_param :ssl_verify, :bool, default: true
|
38
|
+
config_param :format_json, :bool, default: false
|
38
39
|
|
39
40
|
# attr_reader :container
|
40
41
|
|
@@ -46,58 +47,30 @@ module Fluent
|
|
46
47
|
|
47
48
|
def configure(conf)
|
48
49
|
super
|
49
|
-
|
50
|
-
if format_json = conf['format_json']
|
51
|
-
@format_json = true
|
52
|
-
else
|
53
|
-
@format_json = false
|
54
|
-
end
|
55
|
-
|
56
|
-
@ext, @mime_type = case @store_as
|
57
|
-
when 'gzip' then ['gz', 'application/x-gzip']
|
58
|
-
when 'json' then ['json', 'application/json']
|
59
|
-
else ['txt', 'text/plain']
|
60
|
-
end
|
61
|
-
|
50
|
+
@ext, @mime_type = storage_method
|
62
51
|
@timef = TimeFormatter.new(@time_format, @localtime)
|
63
|
-
|
64
|
-
if @localtime
|
65
|
-
@path_slicer = Proc.new {|path|
|
66
|
-
Time.now.strftime(path)
|
67
|
-
}
|
68
|
-
else
|
69
|
-
@path_slicer = Proc.new {|path|
|
70
|
-
Time.now.utc.strftime(path)
|
71
|
-
}
|
72
|
-
end
|
52
|
+
@path_slicer = time_slicer
|
73
53
|
end
|
74
54
|
|
75
55
|
def start
|
76
56
|
super
|
77
|
-
|
78
57
|
Excon.defaults[:ssl_verify_peer] = @ssl_verify
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
:rackspace_region => @rackspace_region
|
58
|
+
@storage = Fog::Storage.new provider: 'Rackspace',
|
59
|
+
rackspace_auth_url: @rackspace_auth_url,
|
60
|
+
rackspace_username: @rackspace_username,
|
61
|
+
rackspace_api_key: @rackspace_api_key,
|
62
|
+
rackspace_region: @rackspace_region
|
85
63
|
|
86
64
|
check_container
|
87
65
|
end
|
88
66
|
|
89
67
|
def format(tag, time, record)
|
90
|
-
if @include_time_key || !@format_json
|
91
|
-
time_str = @timef.format(time)
|
92
|
-
end
|
68
|
+
time_str = @timef.format(time) if @include_time_key || !@format_json
|
93
69
|
|
94
|
-
# copied from each mixin because current TimeSlicedOutput can't support
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
if @include_time_key
|
99
|
-
record[@time_key] = time_str
|
100
|
-
end
|
70
|
+
# copied from each mixin because current TimeSlicedOutput can't support
|
71
|
+
# mixins.
|
72
|
+
record[@tag_key] = tag if @include_tag_key
|
73
|
+
record[@time_key] = time_str if @include_time_key
|
101
74
|
|
102
75
|
if @format_json
|
103
76
|
Yajl.dump(record) + "\n"
|
@@ -108,65 +81,103 @@ module Fluent
|
|
108
81
|
|
109
82
|
def write(chunk)
|
110
83
|
i = 0
|
84
|
+
previous_path = nil
|
111
85
|
|
112
86
|
begin
|
113
87
|
path = @path_slicer.call(@path)
|
114
88
|
values_for_swift_object_key = {
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
}
|
120
|
-
swift_path = @object_key_format.gsub(%r(%{[^}]+})) { |expr|
|
121
|
-
values_for_swift_object_key[expr[2...expr.size-1]]
|
89
|
+
'path' => path,
|
90
|
+
'time_slice' => chunk.key,
|
91
|
+
'file_extension' => @ext,
|
92
|
+
'index' => i
|
122
93
|
}
|
94
|
+
swift_path = @object_key_format.gsub(%r(%{[^}]+})) do |expr|
|
95
|
+
values_for_swift_object_key[expr[2...expr.size - 1]]
|
96
|
+
end
|
97
|
+
if (i > 0) && (swift_path == previous_path)
|
98
|
+
fail 'duplicated path is generated. use %{index} in '\
|
99
|
+
"object_key_format: path = #{swift_path}"
|
100
|
+
end
|
123
101
|
i += 1
|
102
|
+
previous_path = swift_path
|
124
103
|
end while check_object_exists(@rackspace_container, swift_path)
|
104
|
+
write_temp_file(chunk, swift_path)
|
105
|
+
end
|
125
106
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
107
|
+
private
|
108
|
+
|
109
|
+
# Configure methods
|
110
|
+
def storage_method
|
111
|
+
case @store_as
|
112
|
+
when 'gzip' then return ['gz', 'application/x-gzip']
|
113
|
+
when 'json' then return ['json', 'application/json']
|
114
|
+
else return ['txt', 'text/plain']
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def time_slicer
|
119
|
+
if @localtime
|
120
|
+
return proc do |path|
|
121
|
+
Time.now.strftime(path)
|
135
122
|
end
|
136
|
-
|
137
|
-
|
123
|
+
else
|
124
|
+
return proc do |path|
|
125
|
+
Time.now.utc.strftime(path)
|
138
126
|
end
|
139
|
-
$log.info "Put Log to Rackspace Cloud Files. container=#{@rackspace_container} object=#{swift_path}"
|
140
|
-
ensure
|
141
|
-
tmp.close(true) rescue nil
|
142
|
-
w.close rescue nil
|
143
|
-
w.unlink rescue nil
|
144
127
|
end
|
145
128
|
end
|
146
129
|
|
147
|
-
|
148
|
-
|
130
|
+
# Start methods
|
149
131
|
def check_container
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
132
|
+
@storage.get_container(@rackspace_container)
|
133
|
+
rescue Fog::Storage::Rackspace::NotFound
|
134
|
+
if @auto_create_container
|
135
|
+
log.info 'Creating container #{@rackspace_container} in region '\
|
136
|
+
"#{@rackspace_region}"
|
137
|
+
@storage.put_container(@rackspace_container)
|
138
|
+
else
|
139
|
+
raise 'The specified container does not exist: container = '\
|
140
|
+
"#{rackspace_container}"
|
159
141
|
end
|
160
142
|
end
|
161
143
|
|
144
|
+
# Write methods
|
162
145
|
def check_object_exists(container, object)
|
146
|
+
@storage.head_object(container, object)
|
147
|
+
return true
|
148
|
+
rescue Fog::Storage::Rackspace::NotFound
|
149
|
+
return false
|
150
|
+
end
|
151
|
+
|
152
|
+
def write_temp_file(chunk, swift_path)
|
153
|
+
tmp = Tempfile.new('rackspace-cloud-files-')
|
163
154
|
begin
|
164
|
-
@
|
165
|
-
|
166
|
-
|
155
|
+
if @store_as == 'gzip'
|
156
|
+
w = write_gzip_temp_file(chunk, tmp)
|
157
|
+
else
|
158
|
+
chunk.write_to(tmp)
|
159
|
+
tmp.close
|
160
|
+
end
|
161
|
+
write_temp_file_to_cloud_files(tmp, swift_path)
|
162
|
+
ensure
|
163
|
+
tmp.close(true)
|
164
|
+
w.close(true)
|
167
165
|
end
|
168
|
-
return true
|
169
166
|
end
|
170
167
|
|
168
|
+
def write_gzip_temp_file(chunk, tmp)
|
169
|
+
w = Zlib::GzipWriter.new(tmp)
|
170
|
+
chunk.write_to(w)
|
171
|
+
w.close
|
172
|
+
end
|
173
|
+
|
174
|
+
def write_temp_file_to_cloud_files(tmp, swift_path)
|
175
|
+
File.open(tmp.path) do |file|
|
176
|
+
@storage.put_object(@rackspace_container, swift_path, file,
|
177
|
+
content_type: @mime_type)
|
178
|
+
end
|
179
|
+
log.info 'Put Log to Rackspace Cloud Files. container='\
|
180
|
+
"#{@rackspace_container} object=#{swift_path}"
|
181
|
+
end
|
171
182
|
end
|
172
183
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rackspace-cloud-files
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brint O'Hearn
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -136,20 +136,20 @@ dependencies:
|
|
136
136
|
requirements:
|
137
137
|
- - "~>"
|
138
138
|
- !ruby/object:Gem::Version
|
139
|
-
version: 0.33
|
139
|
+
version: '0.33'
|
140
140
|
- - ">="
|
141
141
|
- !ruby/object:Gem::Version
|
142
|
-
version: 0.33
|
142
|
+
version: '0.33'
|
143
143
|
type: :development
|
144
144
|
prerelease: false
|
145
145
|
version_requirements: !ruby/object:Gem::Requirement
|
146
146
|
requirements:
|
147
147
|
- - "~>"
|
148
148
|
- !ruby/object:Gem::Version
|
149
|
-
version: 0.33
|
149
|
+
version: '0.33'
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: 0.33
|
152
|
+
version: '0.33'
|
153
153
|
description: Rackspace Cloud Files output plugin for Fluent event collector
|
154
154
|
email: brintly@gmail.com
|
155
155
|
executables: []
|
@@ -157,6 +157,7 @@ extensions: []
|
|
157
157
|
extra_rdoc_files: []
|
158
158
|
files:
|
159
159
|
- ".gitignore"
|
160
|
+
- ".rubocop.yml"
|
160
161
|
- ".travis.yml"
|
161
162
|
- Gemfile
|
162
163
|
- LICENSE.md
|