vayacondios-server 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/.gitignore +61 -0
  2. data/.travis.yml +11 -0
  3. data/.yardopts +10 -0
  4. data/CHANGELOG.md +0 -0
  5. data/Gemfile +4 -0
  6. data/Guardfile +41 -0
  7. data/LICENSE.md +99 -0
  8. data/Procfile +2 -0
  9. data/README.md +183 -0
  10. data/Rakefile +6 -0
  11. data/app/http_shim.rb +67 -0
  12. data/bin/vcd.sh +27 -0
  13. data/config/http_shim.rb +43 -0
  14. data/config/vayacondios.example.yaml +4 -0
  15. data/config/vayacondios.yaml +4 -0
  16. data/lib/tasks/publish.rake +23 -0
  17. data/lib/tasks/spec.rake +9 -0
  18. data/lib/tasks/yard.rake +2 -0
  19. data/lib/vayacondios/client/configliere.rb +38 -0
  20. data/lib/vayacondios/client/http_client.rb +49 -0
  21. data/lib/vayacondios/client/notifier.rb +84 -0
  22. data/lib/vayacondios/server/handlers/config_handler.rb +35 -0
  23. data/lib/vayacondios/server/handlers/event_handler.rb +30 -0
  24. data/lib/vayacondios/server/model/config_document.rb +94 -0
  25. data/lib/vayacondios/server/model/document.rb +25 -0
  26. data/lib/vayacondios/server/model/event_document.rb +94 -0
  27. data/lib/vayacondios/version.rb +3 -0
  28. data/lib/vayacondios-client.rb +20 -0
  29. data/lib/vayacondios-server.rb +18 -0
  30. data/scripts/hadoop_monitor/configurable.rb +74 -0
  31. data/scripts/hadoop_monitor/hadoop_client.rb +249 -0
  32. data/scripts/hadoop_monitor/hadoop_monitor.rb +91 -0
  33. data/scripts/hadoop_monitor/hadoopable.rb +65 -0
  34. data/scripts/hadoop_monitor/machine_monitor.rb +115 -0
  35. data/scripts/s3_cataloger/buckets +33 -0
  36. data/scripts/s3_cataloger/foreach_bucket +88 -0
  37. data/scripts/s3_cataloger/parse_ls.py +391 -0
  38. data/spec/client/notifier_spec.rb +120 -0
  39. data/spec/server/config_spec.rb +55 -0
  40. data/spec/server/event_spec.rb +44 -0
  41. data/spec/server/server_spec.rb +20 -0
  42. data/spec/spec_helper.rb +10 -0
  43. data/spec/support/mongo_cleaner.rb +26 -0
  44. data/vayacondios-client.gemspec +26 -0
  45. data/vayacondios-server.gemspec +30 -0
  46. metadata +216 -0
@@ -0,0 +1,120 @@
1
+ require 'vayacondios-client'
2
+
3
+ require 'spec_helper'
4
+
5
+ class FakeModel
6
+ include Vayacondios::Notifications
7
+ end
8
+
9
+ describe FakeModel do
10
+ context 'including', Vayacondios::Notifications do
11
+
12
+ it 'defines an instance method notify()' do
13
+ subject.should respond_to(:notify)
14
+ end
15
+
16
+ it 'adds a configurable attribute :notifier with default' do
17
+ subject.notifier.should be_instance_of(Vayacondios.default_notifier.class)
18
+ end
19
+
20
+ end
21
+ end
22
+
23
+ describe Vayacondios::Notifier do
24
+
25
+ shared_examples_for described_class do
26
+ it{ should respond_to(:notify) }
27
+ end
28
+
29
+ context '.prepare' do
30
+ context 'given a Hash-like object' do
31
+ let(:hashlike) { double :hashlike, :to_hash => {} }
32
+ it 'returns a Hash' do
33
+ subject.prepare(hashlike).should be_instance_of(Hash)
34
+ end
35
+ end
36
+
37
+ context 'given a bad argument' do
38
+ let(:bad_arg) { 'shazam' }
39
+ it 'raises an ArgumentError' do
40
+ expect{ subject.prepare(bad_arg) }.to raise_error(ArgumentError, /Cannot notify.*#{bad_arg}.*require a hash-like object/)
41
+ end
42
+ end
43
+ end
44
+
45
+ end
46
+
47
+ describe Vayacondios::HttpNotifier do
48
+ it_behaves_like Vayacondios::Notifier
49
+
50
+ its(:client){ should be_instance_of(Vayacondios::HttpClient) }
51
+
52
+ context '#notify' do
53
+ let(:test_client) { double :client }
54
+ let(:topic) { 'weeeeeeeeeee' }
55
+ let(:cargo) { Hash.new }
56
+
57
+ before do
58
+ subject.stub(:client).and_return(test_client)
59
+ subject.stub(:prepare).and_return(cargo)
60
+ end
61
+
62
+ it 'notifies its client correctly' do
63
+ test_client.should_receive(:insert).with(cargo, :event, topic)
64
+ subject.notify(topic, cargo)
65
+ end
66
+ end
67
+ end
68
+
69
+ describe Vayacondios::LogNotifier do
70
+ it_behaves_like Vayacondios::Notifier
71
+
72
+ its(:client){ should be_instance_of(Logger) }
73
+
74
+ context '#notify' do
75
+ let(:test_client) { double :client }
76
+ let(:topic) { 'weeeeeeeeeee' }
77
+ let(:cargo) { Hash.new }
78
+
79
+ before do
80
+ subject.stub(:client).and_return(test_client)
81
+ subject.stub(:prepare).and_return(cargo)
82
+ end
83
+
84
+ it 'notifies its client correctly' do
85
+ test_client.should_receive(:info).with(/Notification.*#{topic}/)
86
+ subject.notify(topic, cargo)
87
+ end
88
+ end
89
+
90
+ end
91
+
92
+ describe Vayacondios::NotifierFactory do
93
+ context '.receive' do
94
+ context 'given :http' do
95
+ it 'builds a HttpNotifier' do
96
+ described_class.receive(type: 'http').should be_instance_of(Vayacondios::HttpNotifier)
97
+ end
98
+ end
99
+
100
+ context 'given :log' do
101
+ it 'builds a LogNotifier' do
102
+ described_class.receive(type: 'log').should be_instance_of(Vayacondios::LogNotifier)
103
+ end
104
+ end
105
+
106
+ context 'given a bad argument' do
107
+ it 'raises an ArgumentError' do
108
+ expect{ described_class.receive(type: 'bad') }.to raise_error(ArgumentError, /not a valid build option/)
109
+ end
110
+ end
111
+ end
112
+ end
113
+
114
+ describe Vayacondios do
115
+
116
+ it 'has a class method notify()' do
117
+ described_class.should respond_to(:notify)
118
+ end
119
+
120
+ end
@@ -0,0 +1,55 @@
1
+ require 'spec_helper'
2
+
3
+ require 'multi_json'
4
+
5
+ require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
+
7
+ describe HttpShim do
8
+ include Goliath::TestHelper
9
+
10
+ let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.response}" } }
11
+
12
+ context 'Configuration management' do
13
+ it 'stores configuration' do
14
+ with_api(HttpShim) do |api|
15
+ post_request({
16
+ :path => '/v1/infochimps/config/power/level',
17
+ :body => {:level=>"awesome"}
18
+ }, err) do |c|
19
+ c.response_header.status.should == 200
20
+ MultiJson.load(c.response).should eql ({
21
+ "topic" => "power",
22
+ "status" => "success",
23
+ "cargo" => {
24
+ "level" => "awesome"
25
+ }
26
+ })
27
+ end
28
+ end
29
+ end
30
+
31
+ it 'retrieves configuration' do
32
+ with_api(HttpShim) do |api|
33
+ post_request({
34
+ :path => '/v1/infochimps/config/power/level',
35
+ :body => {:level=>"awesome"}
36
+ }, err) do |c|
37
+ c.response_header.status.should == 200
38
+ MultiJson.load(c.response).should eql ({
39
+ "topic" => "power",
40
+ "status" => "success",
41
+ "cargo" => {
42
+ "level" => "awesome"
43
+ }
44
+ })
45
+ end
46
+ end
47
+ with_api(HttpShim) do |api|
48
+ get_request({:path => '/v1/infochimps/config/power/level'}, err) do |c|
49
+ c.response_header.status.should == 200
50
+ MultiJson.load(c.response).should eql({"level" => "awesome"})
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,44 @@
1
+ require 'spec_helper'
2
+
3
+ require 'multi_json'
4
+
5
+ require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
+
7
+ describe HttpShim do
8
+ include Goliath::TestHelper
9
+
10
+ let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.response}" } }
11
+
12
+ context 'Event tracking' do
13
+ it 'stores events' do
14
+ with_api(HttpShim) do |api|
15
+ post_request({
16
+ :path => '/v1/infochimps/event/power/level',
17
+ :body => {:level=>"awesome"}
18
+ }, err) do |c|
19
+ c.response_header.status.should == 200
20
+ MultiJson.load(c.response).should eql ({"level" => "awesome"})
21
+ end
22
+ end
23
+ end
24
+
25
+ it 'retrieves events' do
26
+ current_time = Time.now
27
+ with_api(HttpShim) do |api|
28
+ post_request({
29
+ :path => '/v1/infochimps/event/power/level',
30
+ :body => {:level=>"awesome", :_timestamp => current_time}
31
+ }, err) do |c|
32
+ c.response_header.status.should == 200
33
+ MultiJson.load(c.response).should eql ({"level" => "awesome", "_timestamp" => current_time.to_s})
34
+ end
35
+ end
36
+ with_api(HttpShim) do |api|
37
+ get_request({:path => '/v1/infochimps/event/power/level'}, err) do |c|
38
+ c.response_header.status.should == 200
39
+ MultiJson.load(c.response).should eql ({"level" => "awesome", "_timestamp" => current_time.to_s})
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+
3
+ require 'multi_json'
4
+
5
+ require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
+
7
+ describe HttpShim do
8
+ include Goliath::TestHelper
9
+
10
+ let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.response}" } }
11
+
12
+ it "responds to requests" do
13
+ with_api(HttpShim) do |api|
14
+ get_request({}, err) do |c|
15
+ c.response_header.status.should == 400
16
+ MultiJson.load(c.response).should eql({"error" => "Bad Request"})
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,10 @@
1
+ require 'bundler'
2
+
3
+ Bundler.setup
4
+ Bundler.require
5
+
6
+ Dir["spec/support/**/*.rb"].each {|f| require File.join(File.dirname(__FILE__), '..', f) }
7
+
8
+ require 'goliath/test_helper'
9
+
10
+ Goliath.env = :test
@@ -0,0 +1,26 @@
1
+ require 'mongo'
2
+ require 'configliere'
3
+
4
+ Settings.define :app_name, :default => 'vayacondios', :description => 'Name to key on for tracer stats, statsd metrics, etc.'
5
+ Settings.define 'mongo.host', :default => 'localhost', :description => 'Mongo hostname'
6
+ Settings.define 'mongo.port', :default => '27017', :description => 'Mongo port'
7
+
8
+ Settings.read(File.join File.dirname(__FILE__), '..', '..', 'config', 'vayacondios.yaml')
9
+ Settings.resolve!
10
+
11
+ def clean_mongo
12
+ conn = Mongo::Connection.new(Settings[:mongo][:host],Settings[:mongo][:port])
13
+ mongo = conn.db(Settings[:mongo][:database])
14
+ mongo.collections.select {|c| c.name !~ /system/ }.each { |c| c.drop }
15
+ conn.close
16
+ end
17
+
18
+ RSpec.configure do |config|
19
+ config.before :each do
20
+ clean_mongo
21
+ end
22
+
23
+ config.after :all do
24
+ clean_mongo
25
+ end
26
+ end
@@ -0,0 +1,26 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ $:.push File.expand_path('../lib', __FILE__)
4
+ require 'vayacondios/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = 'vayacondios-client'
8
+ gem.version = Vayacondios::VERSION
9
+ gem.authors = ['Philip (flip) Kromer', 'Travis Dempsey', 'Huston Hoburg', 'Logan Lowell']
10
+ gem.homepage = 'https://github.com/infochimps-labs/vayacondios'
11
+ gem.summary = 'Data goes in. The right thing happens'
12
+ gem.description = "Simple enough to use in a shell script, performant enough to use everywhere. Dios mío! Record that metric, ese!"
13
+
14
+ gem.files = `git ls-files -- lib | grep client`.split("\n")
15
+ gem.test_files = `git ls-files -- spec | grep client`.split("\n")
16
+ gem.require_paths = ['lib']
17
+
18
+ gem.add_dependency('configliere', '>= 0.4.16')
19
+ gem.add_dependency('multi_json', '~> 1.1')
20
+ # Gorillib versioning is borked
21
+ gem.add_dependency('gorillib', '0.4.0pre')
22
+
23
+ gem.add_development_dependency('rake')
24
+ gem.add_development_dependency('yard', '>= 0.7')
25
+ gem.add_development_dependency('rspec', '>= 2.8')
26
+ end
@@ -0,0 +1,30 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ $:.push File.expand_path('../lib', __FILE__)
4
+ require 'vayacondios/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = 'vayacondios-server'
8
+ gem.version = Vayacondios::VERSION
9
+ gem.authors = ['Philip (flip) Kromer', 'Travis Dempsey', 'Huston Hoburg', 'Logan Lowell']
10
+ gem.homepage = 'https://github.com/infochimps-labs/vayacondios'
11
+ gem.summary = 'Data goes in. The right thing happens'
12
+ gem.description = "Simple enough to use in a shell script, performant enough to use everywhere. Dios mío! Record that metric, ese!"
13
+
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.executables = []
16
+ gem.test_files = gem.files.grep(/^spec/)
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_dependency('configliere', '>= 0.4.13')
20
+ gem.add_dependency('gorillib', '0.4.0pre')
21
+ gem.add_dependency('eventmachine', '~> 1.0.0.beta.4')
22
+ gem.add_dependency('goliath', '~> 1.0')
23
+ gem.add_dependency('em-http-request', '~> 1.0')
24
+ gem.add_dependency('em-mongo', '~> 0.4.3')
25
+ gem.add_dependency('bson_ext', '~> 1.6')
26
+ gem.add_dependency('foreman')
27
+
28
+ gem.add_development_dependency('rake')
29
+ gem.add_development_dependency('mongo')
30
+ end
metadata ADDED
@@ -0,0 +1,216 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: vayacondios-server
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.4
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Philip (flip) Kromer
9
+ - Travis Dempsey
10
+ - Huston Hoburg
11
+ - Logan Lowell
12
+ autorequire:
13
+ bindir: bin
14
+ cert_chain: []
15
+ date: 2012-08-28 00:00:00.000000000Z
16
+ dependencies:
17
+ - !ruby/object:Gem::Dependency
18
+ name: configliere
19
+ requirement: &2161806380 !ruby/object:Gem::Requirement
20
+ none: false
21
+ requirements:
22
+ - - ! '>='
23
+ - !ruby/object:Gem::Version
24
+ version: 0.4.13
25
+ type: :runtime
26
+ prerelease: false
27
+ version_requirements: *2161806380
28
+ - !ruby/object:Gem::Dependency
29
+ name: gorillib
30
+ requirement: &2161805460 !ruby/object:Gem::Requirement
31
+ none: false
32
+ requirements:
33
+ - - =
34
+ - !ruby/object:Gem::Version
35
+ version: 0.4.0pre
36
+ type: :runtime
37
+ prerelease: false
38
+ version_requirements: *2161805460
39
+ - !ruby/object:Gem::Dependency
40
+ name: eventmachine
41
+ requirement: &2161804800 !ruby/object:Gem::Requirement
42
+ none: false
43
+ requirements:
44
+ - - ~>
45
+ - !ruby/object:Gem::Version
46
+ version: 1.0.0.beta.4
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: *2161804800
50
+ - !ruby/object:Gem::Dependency
51
+ name: goliath
52
+ requirement: &2161803800 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ~>
56
+ - !ruby/object:Gem::Version
57
+ version: '1.0'
58
+ type: :runtime
59
+ prerelease: false
60
+ version_requirements: *2161803800
61
+ - !ruby/object:Gem::Dependency
62
+ name: em-http-request
63
+ requirement: &2161802360 !ruby/object:Gem::Requirement
64
+ none: false
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: '1.0'
69
+ type: :runtime
70
+ prerelease: false
71
+ version_requirements: *2161802360
72
+ - !ruby/object:Gem::Dependency
73
+ name: em-mongo
74
+ requirement: &2161800840 !ruby/object:Gem::Requirement
75
+ none: false
76
+ requirements:
77
+ - - ~>
78
+ - !ruby/object:Gem::Version
79
+ version: 0.4.3
80
+ type: :runtime
81
+ prerelease: false
82
+ version_requirements: *2161800840
83
+ - !ruby/object:Gem::Dependency
84
+ name: bson_ext
85
+ requirement: &2161786380 !ruby/object:Gem::Requirement
86
+ none: false
87
+ requirements:
88
+ - - ~>
89
+ - !ruby/object:Gem::Version
90
+ version: '1.6'
91
+ type: :runtime
92
+ prerelease: false
93
+ version_requirements: *2161786380
94
+ - !ruby/object:Gem::Dependency
95
+ name: foreman
96
+ requirement: &2161784640 !ruby/object:Gem::Requirement
97
+ none: false
98
+ requirements:
99
+ - - ! '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :runtime
103
+ prerelease: false
104
+ version_requirements: *2161784640
105
+ - !ruby/object:Gem::Dependency
106
+ name: rake
107
+ requirement: &2161782960 !ruby/object:Gem::Requirement
108
+ none: false
109
+ requirements:
110
+ - - ! '>='
111
+ - !ruby/object:Gem::Version
112
+ version: '0'
113
+ type: :development
114
+ prerelease: false
115
+ version_requirements: *2161782960
116
+ - !ruby/object:Gem::Dependency
117
+ name: mongo
118
+ requirement: &2161781540 !ruby/object:Gem::Requirement
119
+ none: false
120
+ requirements:
121
+ - - ! '>='
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
124
+ type: :development
125
+ prerelease: false
126
+ version_requirements: *2161781540
127
+ description: Simple enough to use in a shell script, performant enough to use everywhere.
128
+ Dios mío! Record that metric, ese!
129
+ email:
130
+ executables: []
131
+ extensions: []
132
+ extra_rdoc_files: []
133
+ files:
134
+ - .gitignore
135
+ - .travis.yml
136
+ - .yardopts
137
+ - CHANGELOG.md
138
+ - Gemfile
139
+ - Guardfile
140
+ - LICENSE.md
141
+ - Procfile
142
+ - README.md
143
+ - Rakefile
144
+ - app/http_shim.rb
145
+ - bin/vcd.sh
146
+ - config/http_shim.rb
147
+ - config/vayacondios.example.yaml
148
+ - config/vayacondios.yaml
149
+ - lib/tasks/publish.rake
150
+ - lib/tasks/spec.rake
151
+ - lib/tasks/yard.rake
152
+ - lib/vayacondios-client.rb
153
+ - lib/vayacondios-server.rb
154
+ - lib/vayacondios/client/configliere.rb
155
+ - lib/vayacondios/client/http_client.rb
156
+ - lib/vayacondios/client/notifier.rb
157
+ - lib/vayacondios/server/handlers/config_handler.rb
158
+ - lib/vayacondios/server/handlers/event_handler.rb
159
+ - lib/vayacondios/server/model/config_document.rb
160
+ - lib/vayacondios/server/model/document.rb
161
+ - lib/vayacondios/server/model/event_document.rb
162
+ - lib/vayacondios/version.rb
163
+ - scripts/hadoop_monitor/configurable.rb
164
+ - scripts/hadoop_monitor/hadoop_client.rb
165
+ - scripts/hadoop_monitor/hadoop_monitor.rb
166
+ - scripts/hadoop_monitor/hadoopable.rb
167
+ - scripts/hadoop_monitor/machine_monitor.rb
168
+ - scripts/s3_cataloger/buckets
169
+ - scripts/s3_cataloger/foreach_bucket
170
+ - scripts/s3_cataloger/parse_ls.py
171
+ - spec/client/notifier_spec.rb
172
+ - spec/server/config_spec.rb
173
+ - spec/server/event_spec.rb
174
+ - spec/server/server_spec.rb
175
+ - spec/spec_helper.rb
176
+ - spec/support/mongo_cleaner.rb
177
+ - vayacondios-client.gemspec
178
+ - vayacondios-server.gemspec
179
+ homepage: https://github.com/infochimps-labs/vayacondios
180
+ licenses: []
181
+ post_install_message:
182
+ rdoc_options: []
183
+ require_paths:
184
+ - lib
185
+ required_ruby_version: !ruby/object:Gem::Requirement
186
+ none: false
187
+ requirements:
188
+ - - ! '>='
189
+ - !ruby/object:Gem::Version
190
+ version: '0'
191
+ segments:
192
+ - 0
193
+ hash: -326515220208641293
194
+ required_rubygems_version: !ruby/object:Gem::Requirement
195
+ none: false
196
+ requirements:
197
+ - - ! '>='
198
+ - !ruby/object:Gem::Version
199
+ version: '0'
200
+ segments:
201
+ - 0
202
+ hash: -326515220208641293
203
+ requirements: []
204
+ rubyforge_project:
205
+ rubygems_version: 1.8.15
206
+ signing_key:
207
+ specification_version: 3
208
+ summary: Data goes in. The right thing happens
209
+ test_files:
210
+ - spec/client/notifier_spec.rb
211
+ - spec/server/config_spec.rb
212
+ - spec/server/event_spec.rb
213
+ - spec/server/server_spec.rb
214
+ - spec/spec_helper.rb
215
+ - spec/support/mongo_cleaner.rb
216
+ has_rdoc: