logstash_sqs 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/logstash_sqs/awssqs.rb +10 -5
- data/lib/logstash_sqs/consumer.rb +26 -21
- data/lib/logstash_sqs/version.rb +1 -1
- data/lib/logstash_sqs.rb +1 -0
- metadata +112 -108
- checksums.yaml +0 -15
data/lib/logstash_sqs/awssqs.rb
CHANGED
@@ -21,10 +21,15 @@ class AWSSQS
|
|
21
21
|
|
22
22
|
|
23
23
|
def add_message(message)
|
24
|
-
|
25
|
-
|
26
|
-
@
|
27
|
-
|
28
|
-
|
24
|
+
begin
|
25
|
+
@buffer << message
|
26
|
+
if (@buffer.size >= @buffer_size)
|
27
|
+
@queue.batch_send(*@buffer)
|
28
|
+
@buffer.clear
|
29
|
+
end
|
30
|
+
rescue AWS::SQS::Errors::BatchSendError => exception
|
31
|
+
DaemonSpawn::Base.log_file << "Could not send the message(s)"
|
32
|
+
DaemonSpawn::Base.log_file << exception.failures
|
33
|
+
end
|
29
34
|
end
|
30
35
|
end
|
@@ -9,8 +9,7 @@ class Consumer < EventMachine::FileTail
|
|
9
9
|
|
10
10
|
def receive_data(data)
|
11
11
|
@buffer.extract(data).each do |message|
|
12
|
-
|
13
|
-
@aws_sqs_client.add_message(entry)
|
12
|
+
@aws_sqs_client.add_message(marshall(message))
|
14
13
|
end
|
15
14
|
end
|
16
15
|
|
@@ -19,28 +18,34 @@ class Consumer < EventMachine::FileTail
|
|
19
18
|
def marshall(message)
|
20
19
|
|
21
20
|
case @format
|
22
|
-
|
23
|
-
|
24
|
-
|
21
|
+
when 'json' then message
|
22
|
+
when 'syslog' then format_syslog(message)
|
23
|
+
when 'apache' then format_apache(message)
|
24
|
+
else message
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def format_syslog(entry)
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
29
|
+
#TODO - rewrite this function!
|
30
|
+
month_map = {'Jan' => 1, 'Fev' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7,
|
31
|
+
'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => '11', 'Dec' => 12}
|
32
|
+
month_name, day, hhmmss, host, *message = entry.split
|
33
|
+
year = Time.now.year
|
34
|
+
month = month_map[month_name]
|
35
|
+
hours, minutes, seconds = hhmmss.split(':')
|
36
|
+
timestamp = Time.local(year,month,day,hours,minutes,seconds).utc.iso8601
|
37
|
+
"{\"@timestamp\": \"#{timestamp}\",\"@type\":\"syslog\", \"@message\":\"#{message.join(' ')}\",\"@host\":\"#{host}\"}"
|
38
|
+
end
|
39
|
+
|
40
|
+
def format_apache(entry)
|
41
|
+
|
42
|
+
#"%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\""
|
43
|
+
pattern = /^(\S*) (\S*) (\S*) (\S*) (\[[^\]]+\]) (\S*) (\S*) (\S*) (\S*) (\S*) (\S*) (.*)$/
|
44
|
+
if (matcher = pattern.match(entry))
|
45
|
+
|
46
|
+
return "{\"@virtual_host\": \"#{matcher[1]}\",\"@remote_host\": \"#{matcher[2]}\",\"@i\": \"#{matcher[3]}\",\"@userid\": \"#{matcher[4]}\",\"@time\": \"#{matcher[5]}\",\"@method\": \"#{matcher[6].gsub('"','')}\",\"@req_path\": \"#{matcher[7]}\",\"@http_version\": \"#{matcher[8].gsub('"','')}\",\"@status\": \"#{matcher[9]}\",\"@size\": \"#{matcher[10]}\",\"@referer\": \"#{matcher[11]}\",\"@user_agent\": \"#{matcher[12]}\"}"
|
47
|
+
|
48
|
+
end
|
49
|
+
|
45
50
|
end
|
46
51
|
end
|
data/lib/logstash_sqs/version.rb
CHANGED
data/lib/logstash_sqs.rb
CHANGED
metadata
CHANGED
@@ -1,121 +1,118 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash_sqs
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 0
|
8
|
+
- 5
|
9
|
+
version: 0.0.5
|
5
10
|
platform: ruby
|
6
|
-
authors:
|
11
|
+
authors:
|
7
12
|
- Anderson Nogueira
|
8
13
|
autorequire:
|
9
14
|
bindir: bin
|
10
15
|
cert_chain: []
|
11
|
-
|
12
|
-
|
13
|
-
|
16
|
+
|
17
|
+
date: 2013-09-30 00:00:00 -03:00
|
18
|
+
default_executable:
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
14
21
|
name: bundler
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.3'
|
20
|
-
type: :development
|
21
22
|
prerelease: false
|
22
|
-
|
23
|
-
requirements:
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
24
25
|
- - ~>
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
|
27
|
-
-
|
28
|
-
|
29
|
-
|
30
|
-
requirements:
|
31
|
-
- - ! '>='
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 1
|
29
|
+
- 3
|
30
|
+
version: "1.3"
|
34
31
|
type: :development
|
32
|
+
version_requirements: *id001
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: rake
|
35
35
|
prerelease: false
|
36
|
-
|
37
|
-
requirements:
|
38
|
-
- -
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
|
41
|
-
-
|
36
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
segments:
|
41
|
+
- 0
|
42
|
+
version: "0"
|
43
|
+
type: :development
|
44
|
+
version_requirements: *id002
|
45
|
+
- !ruby/object:Gem::Dependency
|
42
46
|
name: aws-sdk
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ! '>='
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '0'
|
48
|
-
type: :runtime
|
49
47
|
prerelease: false
|
50
|
-
|
51
|
-
requirements:
|
52
|
-
- -
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
|
55
|
-
-
|
56
|
-
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - ! '>='
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
48
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
segments:
|
53
|
+
- 0
|
54
|
+
version: "0"
|
62
55
|
type: :runtime
|
56
|
+
version_requirements: *id003
|
57
|
+
- !ruby/object:Gem::Dependency
|
58
|
+
name: logstasher
|
63
59
|
prerelease: false
|
64
|
-
|
65
|
-
requirements:
|
66
|
-
- -
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
|
69
|
-
-
|
70
|
-
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ! '>='
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
60
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
segments:
|
65
|
+
- 0
|
66
|
+
version: "0"
|
76
67
|
type: :runtime
|
68
|
+
version_requirements: *id004
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: eventmachine
|
77
71
|
prerelease: false
|
78
|
-
|
79
|
-
requirements:
|
80
|
-
- -
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
|
83
|
-
-
|
84
|
-
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ! '>='
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
72
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
segments:
|
77
|
+
- 0
|
78
|
+
version: "0"
|
90
79
|
type: :runtime
|
80
|
+
version_requirements: *id005
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: eventmachine-tail
|
91
83
|
prerelease: false
|
92
|
-
|
93
|
-
requirements:
|
94
|
-
- -
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
|
97
|
-
-
|
98
|
-
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - ! '>='
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
84
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
segments:
|
89
|
+
- 0
|
90
|
+
version: "0"
|
104
91
|
type: :runtime
|
92
|
+
version_requirements: *id006
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: daemon-spawn
|
105
95
|
prerelease: false
|
106
|
-
|
107
|
-
requirements:
|
108
|
-
- -
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
|
96
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
segments:
|
101
|
+
- 0
|
102
|
+
version: "0"
|
103
|
+
type: :runtime
|
104
|
+
version_requirements: *id007
|
111
105
|
description: Logstash SQS Client
|
112
|
-
email:
|
106
|
+
email:
|
113
107
|
- anderson@quimbik.com
|
114
|
-
executables:
|
108
|
+
executables:
|
109
|
+
- logstashsqs
|
115
110
|
- logstashsqs
|
116
111
|
extensions: []
|
112
|
+
|
117
113
|
extra_rdoc_files: []
|
118
|
-
|
114
|
+
|
115
|
+
files:
|
119
116
|
- .gitignore
|
120
117
|
- Gemfile
|
121
118
|
- LICENSE.txt
|
@@ -128,28 +125,35 @@ files:
|
|
128
125
|
- lib/logstash_sqs/settings_provider.rb
|
129
126
|
- lib/logstash_sqs/version.rb
|
130
127
|
- logstash_sqs.gemspec
|
131
|
-
|
132
|
-
|
128
|
+
has_rdoc: true
|
129
|
+
homepage: ""
|
130
|
+
licenses:
|
133
131
|
- MIT
|
134
|
-
metadata: {}
|
135
132
|
post_install_message:
|
136
133
|
rdoc_options: []
|
137
|
-
|
134
|
+
|
135
|
+
require_paths:
|
138
136
|
- lib
|
139
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
140
|
-
requirements:
|
141
|
-
- -
|
142
|
-
- !ruby/object:Gem::Version
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
137
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
138
|
+
requirements:
|
139
|
+
- - ">="
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
segments:
|
142
|
+
- 0
|
143
|
+
version: "0"
|
144
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
145
|
+
requirements:
|
146
|
+
- - ">="
|
147
|
+
- !ruby/object:Gem::Version
|
148
|
+
segments:
|
149
|
+
- 0
|
150
|
+
version: "0"
|
149
151
|
requirements: []
|
152
|
+
|
150
153
|
rubyforge_project:
|
151
|
-
rubygems_version:
|
154
|
+
rubygems_version: 1.3.6
|
152
155
|
signing_key:
|
153
|
-
specification_version:
|
156
|
+
specification_version: 3
|
154
157
|
summary: Sends logstasher logs to logstash through Amazon SQS
|
155
158
|
test_files: []
|
159
|
+
|
checksums.yaml
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
---
|
2
|
-
!binary "U0hBMQ==":
|
3
|
-
metadata.gz: !binary |-
|
4
|
-
OWFmNzRjMjJhMjg4NTk5N2VhMTBmYmFmMzcyN2YwZDZkZWEwZDAyNg==
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
ZjhmYzc5MTE3ZjYwZDAyNmQxM2E0YTI3NWMyY2I3ZGJjMTk2OWQ0MQ==
|
7
|
-
!binary "U0hBNTEy":
|
8
|
-
metadata.gz: !binary |-
|
9
|
-
ZDE2NWFjMWJmNGU4YmE2ODM3NWYwYTUxNjI4MjJjMWVmZTMzZmM0ZWI4ZDEw
|
10
|
-
ODVlODQ4MGJjNzY0ZDQ4NjEyODI3ZTk2MTUyNjU4NTExYzI0ODJmNzViNWZm
|
11
|
-
ZTQ3ZDAwZDU2YTNlZGM3Yjg1NjE2YzM1YTg4ZDM2MDhkMzhjNTc=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
NTViYWFkMmZkZDM1YzllMTU4YmZkMWRiMmYyZjM1MTVhOWUwOTY3YWEwNWNh
|
14
|
-
Yzg2MTVmZTQzN2E4MTFhNGNhZGZkMWFiNzVjYjg0ZWZmMDlkMGY1M2Y2YWVi
|
15
|
-
NTJkNDMwZGIzNTU5OGJmYjMzZTkwNTlkZjNmOTdiM2ViMzgwZDk=
|