logstash_sqs 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. data/lib/logstash_sqs/awssqs.rb +10 -5
  2. data/lib/logstash_sqs/consumer.rb +26 -21
  3. data/lib/logstash_sqs/version.rb +1 -1
  4. data/lib/logstash_sqs.rb +1 -0
  5. metadata +112 -108
  6. checksums.yaml +0 -15
data/lib/logstash_sqs/awssqs.rb CHANGED
@@ -21,10 +21,15 @@ class AWSSQS
21
21
 
22
22
 
23
23
  def add_message(message)
24
- @buffer << message
25
- if (@buffer.size >= @buffer_size)
26
- @queue.batch_send(*@buffer)
27
- @buffer.clear
28
- end
24
+ begin
25
+ @buffer << message
26
+ if (@buffer.size >= @buffer_size)
27
+ @queue.batch_send(*@buffer)
28
+ @buffer.clear
29
+ end
30
+ rescue AWS::SQS::Errors::BatchSendError => exception
31
+ DaemonSpawn::Base.log_file << "Could not send the message(s)"
32
+ DaemonSpawn::Base.log_file << exception.failures
33
+ end
29
34
  end
30
35
  end
data/lib/logstash_sqs/consumer.rb CHANGED
@@ -9,8 +9,7 @@ class Consumer < EventMachine::FileTail
9
9
 
10
10
  def receive_data(data)
11
11
  @buffer.extract(data).each do |message|
12
- entry = marshall(message)
13
- @aws_sqs_client.add_message(entry)
12
+ @aws_sqs_client.add_message(marshall(message))
14
13
  end
15
14
  end
16
15
 
@@ -19,28 +18,34 @@ class Consumer < EventMachine::FileTail
19
18
  def marshall(message)
20
19
 
21
20
  case @format
22
- when 'json' then message
23
- when 'syslog' then format_syslog(message)
24
- else message
21
+ when 'json' then message
22
+ when 'syslog' then format_syslog(message)
23
+ when 'apache' then format_apache(message)
24
+ else message
25
25
  end
26
26
  end
27
27
 
28
28
  def format_syslog(entry)
29
- month_map = {'Jan' => 1, 'Fev' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7,
30
- 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => '11', 'Dec' => 12}
31
- "Sep 5 06:51:42"
32
-
33
- tokens = entry.split
34
-
35
- year = Time.now.year
36
- month = month_map[tokens[0]]
37
- day = tokens[1]
38
- hh_mm_ss = tokens[2].split(':')
39
- time = Time.local(year,month,day,hh_mm_ss[0],hh_mm_ss[1],hh_mm_ss[2])
40
- timestamp = time.utc.iso8601
41
- host = tokens[3]
42
- type = "syslog"
43
- message = tokens[4..-1].join(' ')
44
- "{\"@timestamp\": \"#{timestamp}\",\"@type\":\"syslog\", \"@message\":\"#{message}\",\"@host\":\"#{host}\"}"
29
+ #TODO - rewrite this function!
30
+ month_map = {'Jan' => 1, 'Fev' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7,
31
+ 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => '11', 'Dec' => 12}
32
+ month_name, day, hhmmss, host, *message = entry.split
33
+ year = Time.now.year
34
+ month = month_map[month_name]
35
+ hours, minutes, seconds = hhmmss.split(':')
36
+ timestamp = Time.local(year,month,day,hours,minutes,seconds).utc.iso8601
37
+ "{\"@timestamp\": \"#{timestamp}\",\"@type\":\"syslog\", \"@message\":\"#{message.join(' ')}\",\"@host\":\"#{host}\"}"
38
+ end
39
+
40
+ def format_apache(entry)
41
+
42
+ #"%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\""
43
+ pattern = /^(\S*) (\S*) (\S*) (\S*) (\[[^\]]+\]) (\S*) (\S*) (\S*) (\S*) (\S*) (\S*) (.*)$/
44
+ if (matcher = pattern.match(entry))
45
+
46
+ return "{\"@virtual_host\": \"#{matcher[1]}\",\"@remote_host\": \"#{matcher[2]}\",\"@i\": \"#{matcher[3]}\",\"@userid\": \"#{matcher[4]}\",\"@time\": \"#{matcher[5]}\",\"@method\": \"#{matcher[6].gsub('"','')}\",\"@req_path\": \"#{matcher[7]}\",\"@http_version\": \"#{matcher[8].gsub('"','')}\",\"@status\": \"#{matcher[9]}\",\"@size\": \"#{matcher[10]}\",\"@referer\": \"#{matcher[11]}\",\"@user_agent\": \"#{matcher[12]}\"}"
47
+
48
+ end
49
+
45
50
  end
46
51
  end
data/lib/logstash_sqs/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module LogstashSQS
2
- VERSION = "0.0.4"
2
+ VERSION = "0.0.5"
3
3
  end
data/lib/logstash_sqs.rb CHANGED
@@ -3,6 +3,7 @@ require "eventmachine"
3
3
  require "eventmachine-tail"
4
4
  require "aws-sdk"
5
5
  require "yaml"
6
+ require 'time'
6
7
  require "logstash_sqs/version"
7
8
  require "logstash_sqs/settings_provider"
8
9
  require "logstash_sqs/awssqs"
metadata CHANGED
@@ -1,121 +1,118 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: logstash_sqs
3
- version: !ruby/object:Gem::Version
4
- version: 0.0.4
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 0
8
+ - 5
9
+ version: 0.0.5
5
10
  platform: ruby
6
- authors:
11
+ authors:
7
12
  - Anderson Nogueira
8
13
  autorequire:
9
14
  bindir: bin
10
15
  cert_chain: []
11
- date: 2013-09-06 00:00:00.000000000 Z
12
- dependencies:
13
- - !ruby/object:Gem::Dependency
16
+
17
+ date: 2013-09-30 00:00:00 -03:00
18
+ default_executable:
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
14
21
  name: bundler
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.3'
20
- type: :development
21
22
  prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ requirements:
24
25
  - - ~>
25
- - !ruby/object:Gem::Version
26
- version: '1.3'
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - ! '>='
32
- - !ruby/object:Gem::Version
33
- version: '0'
26
+ - !ruby/object:Gem::Version
27
+ segments:
28
+ - 1
29
+ - 3
30
+ version: "1.3"
34
31
  type: :development
32
+ version_requirements: *id001
33
+ - !ruby/object:Gem::Dependency
34
+ name: rake
35
35
  prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
38
- - - ! '>='
39
- - !ruby/object:Gem::Version
40
- version: '0'
41
- - !ruby/object:Gem::Dependency
36
+ requirement: &id002 !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ segments:
41
+ - 0
42
+ version: "0"
43
+ type: :development
44
+ version_requirements: *id002
45
+ - !ruby/object:Gem::Dependency
42
46
  name: aws-sdk
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ! '>='
46
- - !ruby/object:Gem::Version
47
- version: '0'
48
- type: :runtime
49
47
  prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ! '>='
53
- - !ruby/object:Gem::Version
54
- version: '0'
55
- - !ruby/object:Gem::Dependency
56
- name: logstasher
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - ! '>='
60
- - !ruby/object:Gem::Version
61
- version: '0'
48
+ requirement: &id003 !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ segments:
53
+ - 0
54
+ version: "0"
62
55
  type: :runtime
56
+ version_requirements: *id003
57
+ - !ruby/object:Gem::Dependency
58
+ name: logstasher
63
59
  prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - ! '>='
67
- - !ruby/object:Gem::Version
68
- version: '0'
69
- - !ruby/object:Gem::Dependency
70
- name: eventmachine
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - ! '>='
74
- - !ruby/object:Gem::Version
75
- version: '0'
60
+ requirement: &id004 !ruby/object:Gem::Requirement
61
+ requirements:
62
+ - - ">="
63
+ - !ruby/object:Gem::Version
64
+ segments:
65
+ - 0
66
+ version: "0"
76
67
  type: :runtime
68
+ version_requirements: *id004
69
+ - !ruby/object:Gem::Dependency
70
+ name: eventmachine
77
71
  prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - ! '>='
81
- - !ruby/object:Gem::Version
82
- version: '0'
83
- - !ruby/object:Gem::Dependency
84
- name: eventmachine-tail
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - ! '>='
88
- - !ruby/object:Gem::Version
89
- version: '0'
72
+ requirement: &id005 !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ segments:
77
+ - 0
78
+ version: "0"
90
79
  type: :runtime
80
+ version_requirements: *id005
81
+ - !ruby/object:Gem::Dependency
82
+ name: eventmachine-tail
91
83
  prerelease: false
92
- version_requirements: !ruby/object:Gem::Requirement
93
- requirements:
94
- - - ! '>='
95
- - !ruby/object:Gem::Version
96
- version: '0'
97
- - !ruby/object:Gem::Dependency
98
- name: daemon-spawn
99
- requirement: !ruby/object:Gem::Requirement
100
- requirements:
101
- - - ! '>='
102
- - !ruby/object:Gem::Version
103
- version: '0'
84
+ requirement: &id006 !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ segments:
89
+ - 0
90
+ version: "0"
104
91
  type: :runtime
92
+ version_requirements: *id006
93
+ - !ruby/object:Gem::Dependency
94
+ name: daemon-spawn
105
95
  prerelease: false
106
- version_requirements: !ruby/object:Gem::Requirement
107
- requirements:
108
- - - ! '>='
109
- - !ruby/object:Gem::Version
110
- version: '0'
96
+ requirement: &id007 !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - ">="
99
+ - !ruby/object:Gem::Version
100
+ segments:
101
+ - 0
102
+ version: "0"
103
+ type: :runtime
104
+ version_requirements: *id007
111
105
  description: Logstash SQS Client
112
- email:
106
+ email:
113
107
  - anderson@quimbik.com
114
- executables:
108
+ executables:
109
+ - logstashsqs
115
110
  - logstashsqs
116
111
  extensions: []
112
+
117
113
  extra_rdoc_files: []
118
- files:
114
+
115
+ files:
119
116
  - .gitignore
120
117
  - Gemfile
121
118
  - LICENSE.txt
@@ -128,28 +125,35 @@ files:
128
125
  - lib/logstash_sqs/settings_provider.rb
129
126
  - lib/logstash_sqs/version.rb
130
127
  - logstash_sqs.gemspec
131
- homepage: ''
132
- licenses:
128
+ has_rdoc: true
129
+ homepage: ""
130
+ licenses:
133
131
  - MIT
134
- metadata: {}
135
132
  post_install_message:
136
133
  rdoc_options: []
137
- require_paths:
134
+
135
+ require_paths:
138
136
  - lib
139
- required_ruby_version: !ruby/object:Gem::Requirement
140
- requirements:
141
- - - ! '>='
142
- - !ruby/object:Gem::Version
143
- version: '0'
144
- required_rubygems_version: !ruby/object:Gem::Requirement
145
- requirements:
146
- - - ! '>='
147
- - !ruby/object:Gem::Version
148
- version: '0'
137
+ required_ruby_version: !ruby/object:Gem::Requirement
138
+ requirements:
139
+ - - ">="
140
+ - !ruby/object:Gem::Version
141
+ segments:
142
+ - 0
143
+ version: "0"
144
+ required_rubygems_version: !ruby/object:Gem::Requirement
145
+ requirements:
146
+ - - ">="
147
+ - !ruby/object:Gem::Version
148
+ segments:
149
+ - 0
150
+ version: "0"
149
151
  requirements: []
152
+
150
153
  rubyforge_project:
151
- rubygems_version: 2.0.5
154
+ rubygems_version: 1.3.6
152
155
  signing_key:
153
- specification_version: 4
156
+ specification_version: 3
154
157
  summary: Sends logstasher logs to logstash through Amazon SQS
155
158
  test_files: []
159
+
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- OWFmNzRjMjJhMjg4NTk5N2VhMTBmYmFmMzcyN2YwZDZkZWEwZDAyNg==
5
- data.tar.gz: !binary |-
6
- ZjhmYzc5MTE3ZjYwZDAyNmQxM2E0YTI3NWMyY2I3ZGJjMTk2OWQ0MQ==
7
- !binary "U0hBNTEy":
8
- metadata.gz: !binary |-
9
- ZDE2NWFjMWJmNGU4YmE2ODM3NWYwYTUxNjI4MjJjMWVmZTMzZmM0ZWI4ZDEw
10
- ODVlODQ4MGJjNzY0ZDQ4NjEyODI3ZTk2MTUyNjU4NTExYzI0ODJmNzViNWZm
11
- ZTQ3ZDAwZDU2YTNlZGM3Yjg1NjE2YzM1YTg4ZDM2MDhkMzhjNTc=
12
- data.tar.gz: !binary |-
13
- NTViYWFkMmZkZDM1YzllMTU4YmZkMWRiMmYyZjM1MTVhOWUwOTY3YWEwNWNh
14
- Yzg2MTVmZTQzN2E4MTFhNGNhZGZkMWFiNzVjYjg0ZWZmMDlkMGY1M2Y2YWVi
15
- NTJkNDMwZGIzNTU5OGJmYjMzZTkwNTlkZjNmOTdiM2ViMzgwZDk=