logstash_sqs 0.0.4 → 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. data/lib/logstash_sqs/awssqs.rb +10 -5
  2. data/lib/logstash_sqs/consumer.rb +26 -21
  3. data/lib/logstash_sqs/version.rb +1 -1
  4. data/lib/logstash_sqs.rb +1 -0
  5. metadata +112 -108
  6. checksums.yaml +0 -15
data/lib/logstash_sqs/awssqs.rb CHANGED
@@ -21,10 +21,15 @@ class AWSSQS
21
21
 
22
22
 
23
23
  def add_message(message)
24
- @buffer << message
25
- if (@buffer.size >= @buffer_size)
26
- @queue.batch_send(*@buffer)
27
- @buffer.clear
28
- end
24
+ begin
25
+ @buffer << message
26
+ if (@buffer.size >= @buffer_size)
27
+ @queue.batch_send(*@buffer)
28
+ @buffer.clear
29
+ end
30
+ rescue AWS::SQS::Errors::BatchSendError => exception
31
+ DaemonSpawn::Base.log_file << "Could not send the message(s)"
32
+ DaemonSpawn::Base.log_file << exception.failures
33
+ end
29
34
  end
30
35
  end
data/lib/logstash_sqs/consumer.rb CHANGED
@@ -9,8 +9,7 @@ class Consumer < EventMachine::FileTail
9
9
 
10
10
  def receive_data(data)
11
11
  @buffer.extract(data).each do |message|
12
- entry = marshall(message)
13
- @aws_sqs_client.add_message(entry)
12
+ @aws_sqs_client.add_message(marshall(message))
14
13
  end
15
14
  end
16
15
 
@@ -19,28 +18,34 @@ class Consumer < EventMachine::FileTail
19
18
  def marshall(message)
20
19
 
21
20
  case @format
22
- when 'json' then message
23
- when 'syslog' then format_syslog(message)
24
- else message
21
+ when 'json' then message
22
+ when 'syslog' then format_syslog(message)
23
+ when 'apache' then format_apache(message)
24
+ else message
25
25
  end
26
26
  end
27
27
 
28
28
  def format_syslog(entry)
29
- month_map = {'Jan' => 1, 'Fev' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7,
30
- 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => '11', 'Dec' => 12}
31
- "Sep 5 06:51:42"
32
-
33
- tokens = entry.split
34
-
35
- year = Time.now.year
36
- month = month_map[tokens[0]]
37
- day = tokens[1]
38
- hh_mm_ss = tokens[2].split(':')
39
- time = Time.local(year,month,day,hh_mm_ss[0],hh_mm_ss[1],hh_mm_ss[2])
40
- timestamp = time.utc.iso8601
41
- host = tokens[3]
42
- type = "syslog"
43
- message = tokens[4..-1].join(' ')
44
- "{\"@timestamp\": \"#{timestamp}\",\"@type\":\"syslog\", \"@message\":\"#{message}\",\"@host\":\"#{host}\"}"
29
+ #TODO - rewrite this function!
30
+ month_map = {'Jan' => 1, 'Fev' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7,
31
+ 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => '11', 'Dec' => 12}
32
+ month_name, day, hhmmss, host, *message = entry.split
33
+ year = Time.now.year
34
+ month = month_map[month_name]
35
+ hours, minutes, seconds = hhmmss.split(':')
36
+ timestamp = Time.local(year,month,day,hours,minutes,seconds).utc.iso8601
37
+ "{\"@timestamp\": \"#{timestamp}\",\"@type\":\"syslog\", \"@message\":\"#{message.join(' ')}\",\"@host\":\"#{host}\"}"
38
+ end
39
+
40
+ def format_apache(entry)
41
+
42
+ #"%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\""
43
+ pattern = /^(\S*) (\S*) (\S*) (\S*) (\[[^\]]+\]) (\S*) (\S*) (\S*) (\S*) (\S*) (\S*) (.*)$/
44
+ if (matcher = pattern.match(entry))
45
+
46
+ return "{\"@virtual_host\": \"#{matcher[1]}\",\"@remote_host\": \"#{matcher[2]}\",\"@i\": \"#{matcher[3]}\",\"@userid\": \"#{matcher[4]}\",\"@time\": \"#{matcher[5]}\",\"@method\": \"#{matcher[6].gsub('"','')}\",\"@req_path\": \"#{matcher[7]}\",\"@http_version\": \"#{matcher[8].gsub('"','')}\",\"@status\": \"#{matcher[9]}\",\"@size\": \"#{matcher[10]}\",\"@referer\": \"#{matcher[11]}\",\"@user_agent\": \"#{matcher[12]}\"}"
47
+
48
+ end
49
+
45
50
  end
46
51
  end
data/lib/logstash_sqs/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module LogstashSQS
2
- VERSION = "0.0.4"
2
+ VERSION = "0.0.5"
3
3
  end
data/lib/logstash_sqs.rb CHANGED
@@ -3,6 +3,7 @@ require "eventmachine"
3
3
  require "eventmachine-tail"
4
4
  require "aws-sdk"
5
5
  require "yaml"
6
+ require 'time'
6
7
  require "logstash_sqs/version"
7
8
  require "logstash_sqs/settings_provider"
8
9
  require "logstash_sqs/awssqs"
metadata CHANGED
@@ -1,121 +1,118 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: logstash_sqs
3
- version: !ruby/object:Gem::Version
4
- version: 0.0.4
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 0
8
+ - 5
9
+ version: 0.0.5
5
10
  platform: ruby
6
- authors:
11
+ authors:
7
12
  - Anderson Nogueira
8
13
  autorequire:
9
14
  bindir: bin
10
15
  cert_chain: []
11
- date: 2013-09-06 00:00:00.000000000 Z
12
- dependencies:
13
- - !ruby/object:Gem::Dependency
16
+
17
+ date: 2013-09-30 00:00:00 -03:00
18
+ default_executable:
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
14
21
  name: bundler
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.3'
20
- type: :development
21
22
  prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ requirements:
24
25
  - - ~>
25
- - !ruby/object:Gem::Version
26
- version: '1.3'
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - ! '>='
32
- - !ruby/object:Gem::Version
33
- version: '0'
26
+ - !ruby/object:Gem::Version
27
+ segments:
28
+ - 1
29
+ - 3
30
+ version: "1.3"
34
31
  type: :development
32
+ version_requirements: *id001
33
+ - !ruby/object:Gem::Dependency
34
+ name: rake
35
35
  prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
38
- - - ! '>='
39
- - !ruby/object:Gem::Version
40
- version: '0'
41
- - !ruby/object:Gem::Dependency
36
+ requirement: &id002 !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ segments:
41
+ - 0
42
+ version: "0"
43
+ type: :development
44
+ version_requirements: *id002
45
+ - !ruby/object:Gem::Dependency
42
46
  name: aws-sdk
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ! '>='
46
- - !ruby/object:Gem::Version
47
- version: '0'
48
- type: :runtime
49
47
  prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ! '>='
53
- - !ruby/object:Gem::Version
54
- version: '0'
55
- - !ruby/object:Gem::Dependency
56
- name: logstasher
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - ! '>='
60
- - !ruby/object:Gem::Version
61
- version: '0'
48
+ requirement: &id003 !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ segments:
53
+ - 0
54
+ version: "0"
62
55
  type: :runtime
56
+ version_requirements: *id003
57
+ - !ruby/object:Gem::Dependency
58
+ name: logstasher
63
59
  prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - ! '>='
67
- - !ruby/object:Gem::Version
68
- version: '0'
69
- - !ruby/object:Gem::Dependency
70
- name: eventmachine
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - ! '>='
74
- - !ruby/object:Gem::Version
75
- version: '0'
60
+ requirement: &id004 !ruby/object:Gem::Requirement
61
+ requirements:
62
+ - - ">="
63
+ - !ruby/object:Gem::Version
64
+ segments:
65
+ - 0
66
+ version: "0"
76
67
  type: :runtime
68
+ version_requirements: *id004
69
+ - !ruby/object:Gem::Dependency
70
+ name: eventmachine
77
71
  prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - ! '>='
81
- - !ruby/object:Gem::Version
82
- version: '0'
83
- - !ruby/object:Gem::Dependency
84
- name: eventmachine-tail
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - ! '>='
88
- - !ruby/object:Gem::Version
89
- version: '0'
72
+ requirement: &id005 !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ segments:
77
+ - 0
78
+ version: "0"
90
79
  type: :runtime
80
+ version_requirements: *id005
81
+ - !ruby/object:Gem::Dependency
82
+ name: eventmachine-tail
91
83
  prerelease: false
92
- version_requirements: !ruby/object:Gem::Requirement
93
- requirements:
94
- - - ! '>='
95
- - !ruby/object:Gem::Version
96
- version: '0'
97
- - !ruby/object:Gem::Dependency
98
- name: daemon-spawn
99
- requirement: !ruby/object:Gem::Requirement
100
- requirements:
101
- - - ! '>='
102
- - !ruby/object:Gem::Version
103
- version: '0'
84
+ requirement: &id006 !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ segments:
89
+ - 0
90
+ version: "0"
104
91
  type: :runtime
92
+ version_requirements: *id006
93
+ - !ruby/object:Gem::Dependency
94
+ name: daemon-spawn
105
95
  prerelease: false
106
- version_requirements: !ruby/object:Gem::Requirement
107
- requirements:
108
- - - ! '>='
109
- - !ruby/object:Gem::Version
110
- version: '0'
96
+ requirement: &id007 !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - ">="
99
+ - !ruby/object:Gem::Version
100
+ segments:
101
+ - 0
102
+ version: "0"
103
+ type: :runtime
104
+ version_requirements: *id007
111
105
  description: Logstash SQS Client
112
- email:
106
+ email:
113
107
  - anderson@quimbik.com
114
- executables:
108
+ executables:
109
+ - logstashsqs
115
110
  - logstashsqs
116
111
  extensions: []
112
+
117
113
  extra_rdoc_files: []
118
- files:
114
+
115
+ files:
119
116
  - .gitignore
120
117
  - Gemfile
121
118
  - LICENSE.txt
@@ -128,28 +125,35 @@ files:
128
125
  - lib/logstash_sqs/settings_provider.rb
129
126
  - lib/logstash_sqs/version.rb
130
127
  - logstash_sqs.gemspec
131
- homepage: ''
132
- licenses:
128
+ has_rdoc: true
129
+ homepage: ""
130
+ licenses:
133
131
  - MIT
134
- metadata: {}
135
132
  post_install_message:
136
133
  rdoc_options: []
137
- require_paths:
134
+
135
+ require_paths:
138
136
  - lib
139
- required_ruby_version: !ruby/object:Gem::Requirement
140
- requirements:
141
- - - ! '>='
142
- - !ruby/object:Gem::Version
143
- version: '0'
144
- required_rubygems_version: !ruby/object:Gem::Requirement
145
- requirements:
146
- - - ! '>='
147
- - !ruby/object:Gem::Version
148
- version: '0'
137
+ required_ruby_version: !ruby/object:Gem::Requirement
138
+ requirements:
139
+ - - ">="
140
+ - !ruby/object:Gem::Version
141
+ segments:
142
+ - 0
143
+ version: "0"
144
+ required_rubygems_version: !ruby/object:Gem::Requirement
145
+ requirements:
146
+ - - ">="
147
+ - !ruby/object:Gem::Version
148
+ segments:
149
+ - 0
150
+ version: "0"
149
151
  requirements: []
152
+
150
153
  rubyforge_project:
151
- rubygems_version: 2.0.5
154
+ rubygems_version: 1.3.6
152
155
  signing_key:
153
- specification_version: 4
156
+ specification_version: 3
154
157
  summary: Sends logstasher logs to logstash through Amazon SQS
155
158
  test_files: []
159
+
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- OWFmNzRjMjJhMjg4NTk5N2VhMTBmYmFmMzcyN2YwZDZkZWEwZDAyNg==
5
- data.tar.gz: !binary |-
6
- ZjhmYzc5MTE3ZjYwZDAyNmQxM2E0YTI3NWMyY2I3ZGJjMTk2OWQ0MQ==
7
- !binary "U0hBNTEy":
8
- metadata.gz: !binary |-
9
- ZDE2NWFjMWJmNGU4YmE2ODM3NWYwYTUxNjI4MjJjMWVmZTMzZmM0ZWI4ZDEw
10
- ODVlODQ4MGJjNzY0ZDQ4NjEyODI3ZTk2MTUyNjU4NTExYzI0ODJmNzViNWZm
11
- ZTQ3ZDAwZDU2YTNlZGM3Yjg1NjE2YzM1YTg4ZDM2MDhkMzhjNTc=
12
- data.tar.gz: !binary |-
13
- NTViYWFkMmZkZDM1YzllMTU4YmZkMWRiMmYyZjM1MTVhOWUwOTY3YWEwNWNh
14
- Yzg2MTVmZTQzN2E4MTFhNGNhZGZkMWFiNzVjYjg0ZWZmMDlkMGY1M2Y2YWVi
15
- NTJkNDMwZGIzNTU5OGJmYjMzZTkwNTlkZjNmOTdiM2ViMzgwZDk=