fluent-plugin-sqs 0.2.3 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
data/AUTHORS CHANGED
@@ -1 +1 @@
1
- Yudai Odagiri <ixixizko _at_ gmail.com>
1
+ Yuri Odagiri <ixixizko _at_ gmail.com>
data/README.rdoc CHANGED
@@ -9,6 +9,10 @@
9
9
 
10
10
  Store fluent-event as queue message to amazon SQS.
11
11
 
12
+ === SQSInput
13
+
14
+ Read events from from amazon SQS.
15
+
12
16
  == Configuratin
13
17
 
14
18
  === SQSOutput
@@ -40,13 +44,38 @@ Store fluent-event as queue message to amazon SQS.
40
44
 
41
45
  </match>
42
46
 
43
- == TODO
47
+ === SQSInput
48
+
49
+ <source>
50
+
51
+ type sqs
52
+
53
+ # following attibutes are required
54
+
55
+ aws_key_id {your_aws_key_id}
56
+ aws_sec_key {your_aws_secret_key}
57
+ sqs_url {queue_url}
58
+
59
+ # following attibutes are optional
60
+  
61
+ sqs_endpoint {endpointURL}
62
+
63
+ ### endpoint list ###
64
+ # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
65
+ # Asia Pacific (Singapore) : sqs.ap-southeast-1.amazonaws.com
66
+ # US-East (Virginia) : sqs.us-east-1.amazonaws.com
67
+ # US-West (Oregon) : sqs.us-west-2.amazonaws.com
68
+ # US-West (N.California) : sqs.us-west-1.amazonaws.com
69
+ # EU-West (Ireland) : sqs.eu-west-1.amazonaws.com
70
+ # South America (São Paulo) : sns.sa-east-1.amazonaws.com
71
+
72
+ receive_interval {receive_interval_seconds}
44
73
 
45
- === input plugin
74
+ </source>
46
75
 
47
76
  == Tool
48
77
 
49
78
  == Copyright
50
79
 
51
- Copyright:: Copyright (c) 2011- Yudai Odagiri
80
+ Copyright:: Copyright (c) 2011- Yuri Odagiri
52
81
  License:: Apache License, Version 2.0
data/Rakefile CHANGED
@@ -7,15 +7,15 @@ begin
7
7
  Jeweler::Tasks.new do |gemspec|
8
8
  gemspec.name = "fluent-plugin-sqs"
9
9
  gemspec.summary = "Amazon SQS output plugin for Fluent event collector"
10
- gemspec.author = "Yudai Odagiri"
10
+ gemspec.author = "Yuri Odagiri"
11
11
  gemspec.email = "ixixizko@gmail.com"
12
12
  gemspec.homepage = "https://github.com/ixixi/fluent-plugin-sqs"
13
13
  gemspec.has_rdoc = false
14
14
  gemspec.require_paths = ["lib"]
15
15
  gemspec.add_dependency "fluentd", "~> 0.10.0"
16
16
  gemspec.add_dependency "aws-sdk", "~> 1.3.2"
17
- gemspec.test_files = Dir["test/**/*.rb"]
18
- gemspec.files = Dir["lib/**/*", "test/**/*.rb"] + %w[VERSION AUTHORS Rakefile]
17
+ gemspec.test_files = Dir["test/**/*.rb", "spec/**/*.rb"]
18
+ gemspec.files = Dir["lib/**/*", "test/**/*.rb", "spec/**/*.rb"] + %w[VERSION AUTHORS Rakefile]
19
19
  gemspec.executables = []
20
20
  end
21
21
  Jeweler::GemcutterTasks.new
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.3
1
+ 1.1.0
@@ -0,0 +1,67 @@
1
+ module Fluent
2
+
3
+ require 'aws-sdk'
4
+
5
+ class SQSInput < Input
6
+ Plugin.register_input('sqs', self)
7
+
8
+ def initialize
9
+ super
10
+ end
11
+
12
+ config_param :aws_key_id, :string
13
+ config_param :aws_sec_key, :string
14
+ config_param :tag, :string
15
+ config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
16
+ config_param :sqs_url, :string
17
+ config_param :receive_interval, :time, :default => 1
18
+
19
+ def configure(conf)
20
+ super
21
+
22
+ end
23
+
24
+ def start
25
+ super
26
+
27
+ AWS.config(
28
+ :access_key_id => @aws_key_id,
29
+ :secret_access_key => @aws_sec_key
30
+ )
31
+
32
+ @queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
33
+
34
+ @finished = false
35
+ @thread = Thread.new(&method(:run_periodic))
36
+ end
37
+
38
+ def shutdown
39
+ super
40
+
41
+ @finished = true
42
+ @thread.join
43
+ end
44
+
45
+ def run_periodic
46
+ until @finished
47
+ begin
48
+ sleep @receive_interval
49
+ @queue.receive_message do |message|
50
+ record = {}
51
+ record[:body] = message.body.to_s
52
+ record[:handle] = message.handle.to_s
53
+ record[:id] = message.id.to_s
54
+ record[:md5] = message.md5.to_s
55
+ record[:url] = message.queue.url.to_s
56
+ record[:sender_id] = message.sender_id.to_s
57
+
58
+ Engine.emit(@tag, Time.now, record)
59
+ end
60
+ rescue
61
+ $log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
62
+ $log.warn_backtrace $!.backtrace
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -28,10 +28,10 @@ module Fluent
28
28
 
29
29
  AWS.config(
30
30
  :access_key_id => @aws_key_id,
31
- :secret_access_key => @aws_sec_key,
32
- :sqs_endpoint => @sqs_endpoint )
31
+ :secret_access_key => @aws_sec_key)
33
32
 
34
- @sqs = AWS::SQS.new
33
+ @sqs = AWS::SQS.new(
34
+ :sqs_endpoint => @sqs_endpoint)
35
35
  @queue = @sqs.queues.create(@queue_name)
36
36
 
37
37
  end
@@ -0,0 +1,96 @@
1
+ require 'spec_helper'
2
+
3
+ describe do
4
+ let(:driver) {
5
+ AWS.stub!
6
+ Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
7
+ }
8
+ let(:instance) {driver.instance}
9
+
10
+ describe 'config' do
11
+ let(:config) {
12
+ %[
13
+ aws_key_id AWS_KEY_ID
14
+ aws_sec_key AWS_SEC_KEY
15
+ tag TAG
16
+ sqs_url SQS_URL
17
+ ]
18
+ }
19
+
20
+ context do
21
+ subject {instance.aws_key_id}
22
+ it{should == 'AWS_KEY_ID'}
23
+ end
24
+
25
+ context do
26
+ subject {instance.aws_sec_key}
27
+ it{should == 'AWS_SEC_KEY'}
28
+ end
29
+
30
+ context do
31
+ subject {instance.tag}
32
+ it{should == 'TAG'}
33
+ end
34
+
35
+ context do
36
+ subject {instance.sqs_url}
37
+ it{should == 'SQS_URL'}
38
+ end
39
+
40
+ context do
41
+ subject {instance.receive_interval}
42
+ it{should == 1}
43
+ end
44
+ end
45
+
46
+ describe 'emit' do
47
+ let(:message) do
48
+ { :body => 'body',
49
+ :handle => 'handle',
50
+ :id => 'id',
51
+ :md5 => 'md5',
52
+ :url => 'url',
53
+ :sender_id => 'sender_id'
54
+ }
55
+ end
56
+ let(:emmits) {
57
+ stub(Time).now {0}
58
+
59
+ class AWS::SQS::Queue
60
+ def receive_message
61
+ yield OpenStruct.new(
62
+ { :body => 'body',
63
+ :handle => 'handle',
64
+ :id => 'id',
65
+ :md5 => 'md5',
66
+ :queue => OpenStruct.new(:url => 'url'),
67
+ :sender_id => 'sender_id',
68
+ :sent_at => 0
69
+ })
70
+ end
71
+ end
72
+ d = driver
73
+ d.run do
74
+ sleep 2
75
+ end
76
+
77
+ d.emits
78
+ }
79
+
80
+ context do
81
+ let(:config) {
82
+ %[
83
+ aws_key_id AWS_KEY_ID
84
+ aws_sec_key AWS_SEC_KEY
85
+ tag TAG
86
+ sqs_url SQS_URL
87
+ ]
88
+ }
89
+
90
+ subject {emmits.first}
91
+ it{should == ['TAG', 0, message]}
92
+ end
93
+
94
+ end
95
+
96
+ end
@@ -0,0 +1,25 @@
1
+ # This file was generated by the `rspec --init` command. Conventionally, all
2
+ # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
+ # Require this file using `require "spec_helper"` to ensure that it is only
4
+ # loaded once.
5
+ #
6
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
7
+ RSpec.configure do |config|
8
+ config.treat_symbols_as_metadata_keys_with_true_values = true
9
+ config.run_all_when_everything_filtered = true
10
+ config.filter_run :focus
11
+
12
+ # Run specs in random order to surface order dependencies. If you find an
13
+ # order dependency and want to debug it, you can fix the order by providing
14
+ # the seed, which is printed after each run.
15
+ # --seed 1234
16
+ config.order = 'random'
17
+
18
+ require 'fluent/load'
19
+ require 'fluent/test'
20
+
21
+ require 'fluent/plugin/in_sqs'
22
+ require 'ostruct'
23
+ require 'pry'
24
+ require 'rr'
25
+ end
metadata CHANGED
@@ -1,79 +1,199 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-sqs
3
- version: !ruby/object:Gem::Version
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.1.0
4
5
  prerelease:
5
- version: 0.2.3
6
6
  platform: ruby
7
- authors:
8
- - Yudai Odagiri
7
+ authors:
8
+ - Yuri Odagiri
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
-
13
- date: 2013-06-15 00:00:00 Z
14
- dependencies:
15
- - !ruby/object:Gem::Dependency
16
- name: fluentd
12
+ date: 2014-01-29 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: jeweler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
17
23
  prerelease: false
18
- requirement: &id001 !ruby/object:Gem::Requirement
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: fluent-plugin-sqs
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: bundler
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: rake
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: rspec
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ - !ruby/object:Gem::Dependency
95
+ name: rr
96
+ requirement: !ruby/object:Gem::Requirement
19
97
  none: false
20
- requirements:
98
+ requirements:
99
+ - - ! '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :development
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ! '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ - !ruby/object:Gem::Dependency
111
+ name: pry
112
+ requirement: !ruby/object:Gem::Requirement
113
+ none: false
114
+ requirements:
115
+ - - ! '>='
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ none: false
122
+ requirements:
123
+ - - ! '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ - !ruby/object:Gem::Dependency
127
+ name: fluentd
128
+ requirement: !ruby/object:Gem::Requirement
129
+ none: false
130
+ requirements:
21
131
  - - ~>
22
- - !ruby/object:Gem::Version
132
+ - !ruby/object:Gem::Version
23
133
  version: 0.10.0
24
134
  type: :runtime
25
- version_requirements: *id001
26
- - !ruby/object:Gem::Dependency
27
- name: aws-sdk
28
135
  prerelease: false
29
- requirement: &id002 !ruby/object:Gem::Requirement
136
+ version_requirements: !ruby/object:Gem::Requirement
30
137
  none: false
31
- requirements:
138
+ requirements:
32
139
  - - ~>
33
- - !ruby/object:Gem::Version
140
+ - !ruby/object:Gem::Version
141
+ version: 0.10.0
142
+ - !ruby/object:Gem::Dependency
143
+ name: aws-sdk
144
+ requirement: !ruby/object:Gem::Requirement
145
+ none: false
146
+ requirements:
147
+ - - ~>
148
+ - !ruby/object:Gem::Version
34
149
  version: 1.3.2
35
150
  type: :runtime
36
- version_requirements: *id002
151
+ prerelease: false
152
+ version_requirements: !ruby/object:Gem::Requirement
153
+ none: false
154
+ requirements:
155
+ - - ~>
156
+ - !ruby/object:Gem::Version
157
+ version: 1.3.2
37
158
  description:
38
159
  email: ixixizko@gmail.com
39
160
  executables: []
40
-
41
161
  extensions: []
42
-
43
- extra_rdoc_files:
162
+ extra_rdoc_files:
44
163
  - README.rdoc
45
- files:
164
+ files:
46
165
  - AUTHORS
47
166
  - Rakefile
48
167
  - VERSION
168
+ - lib/fluent/plugin/in_sqs.rb
49
169
  - lib/fluent/plugin/out_sqs.rb
170
+ - spec/lib/fluent/plugin/in_sqs_spec.rb
171
+ - spec/spec_helper.rb
50
172
  - README.rdoc
51
173
  homepage: https://github.com/ixixi/fluent-plugin-sqs
52
174
  licenses: []
53
-
54
175
  post_install_message:
55
176
  rdoc_options: []
56
-
57
- require_paths:
177
+ require_paths:
58
178
  - lib
59
- required_ruby_version: !ruby/object:Gem::Requirement
179
+ required_ruby_version: !ruby/object:Gem::Requirement
60
180
  none: false
61
- requirements:
62
- - - ">="
63
- - !ruby/object:Gem::Version
64
- version: "0"
65
- required_rubygems_version: !ruby/object:Gem::Requirement
181
+ requirements:
182
+ - - ! '>='
183
+ - !ruby/object:Gem::Version
184
+ version: '0'
185
+ required_rubygems_version: !ruby/object:Gem::Requirement
66
186
  none: false
67
- requirements:
68
- - - ">="
69
- - !ruby/object:Gem::Version
70
- version: "0"
187
+ requirements:
188
+ - - ! '>='
189
+ - !ruby/object:Gem::Version
190
+ version: '0'
71
191
  requirements: []
72
-
73
192
  rubyforge_project:
74
193
  rubygems_version: 1.8.23
75
194
  signing_key:
76
195
  specification_version: 3
77
196
  summary: Amazon SQS output plugin for Fluent event collector
78
- test_files: []
79
-
197
+ test_files:
198
+ - spec/lib/fluent/plugin/in_sqs_spec.rb
199
+ - spec/spec_helper.rb