vayacondios-server 0.2.11 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (142) hide show
  1. data/.gitignore +3 -1
  2. data/.travis.yml +2 -0
  3. data/Gemfile +15 -9
  4. data/LICENSE.md +2 -6
  5. data/Procfile +1 -1
  6. data/README.md +656 -111
  7. data/Rakefile +89 -6
  8. data/bin/vcd +10 -0
  9. data/bin/vcd-server +8 -0
  10. data/config/database.yml +6 -0
  11. data/config/spec.example.yml +18 -0
  12. data/config/vayacondios.example.yml +15 -0
  13. data/config/vcd-server.rb +37 -0
  14. data/examples/configuration.rb +56 -0
  15. data/examples/event_stream.rb +19 -0
  16. data/examples/simple.rb +61 -0
  17. data/features/event.feature +319 -0
  18. data/features/events.feature +208 -0
  19. data/features/stash.feature +840 -0
  20. data/features/stashes.feature +492 -0
  21. data/features/step_definitions/stash_steps.rb +113 -0
  22. data/features/stream.feature +30 -0
  23. data/features/support/em.rb +14 -0
  24. data/features/support/env.rb +13 -0
  25. data/lib/vayacondios/configuration.rb +63 -0
  26. data/lib/vayacondios/server/api.rb +126 -0
  27. data/lib/vayacondios/server/api_options.rb +56 -0
  28. data/lib/vayacondios/server/configuration.rb +23 -0
  29. data/lib/vayacondios/server/driver.rb +71 -0
  30. data/lib/vayacondios/server/drivers/mongo.rb +126 -0
  31. data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
  32. data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
  33. data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
  34. data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
  35. data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
  36. data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
  37. data/lib/vayacondios/server/models/document.rb +87 -0
  38. data/lib/vayacondios/server/models/event.rb +198 -0
  39. data/lib/vayacondios/server/models/stash.rb +100 -0
  40. data/lib/vayacondios/server.rb +35 -0
  41. data/lib/vayacondios-server.rb +19 -13
  42. data/lib/vayacondios.rb +22 -0
  43. data/pom.xml +124 -4
  44. data/spec/configuration_spec.rb +41 -0
  45. data/spec/server/api_options_spec.rb +32 -0
  46. data/spec/server/api_spec.rb +279 -0
  47. data/spec/server/configuration_spec.rb +27 -0
  48. data/spec/server/drivers/mongo_spec.rb +107 -0
  49. data/spec/server/handlers/event_handler_spec.rb +62 -0
  50. data/spec/server/handlers/events_handler_spec.rb +51 -0
  51. data/spec/server/handlers/stash_handler_spec.rb +68 -0
  52. data/spec/server/handlers/stashes_handler_spec.rb +50 -0
  53. data/spec/server/handlers/stream_handler_spec.rb +5 -0
  54. data/spec/server/models/document_spec.rb +9 -0
  55. data/spec/server/models/event_spec.rb +185 -0
  56. data/spec/server/models/stash_spec.rb +95 -0
  57. data/spec/spec_helper.rb +23 -3
  58. data/spec/support/database_helper.rb +42 -0
  59. data/spec/support/log_helper.rb +19 -0
  60. data/spec/support/shared_context_for_events.rb +22 -0
  61. data/spec/support/shared_context_for_stashes.rb +24 -0
  62. data/spec/support/shared_examples_for_handlers.rb +32 -0
  63. data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
  64. data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
  65. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
  66. data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
  67. data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
  68. data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
  69. data/vayacondios-server.gemspec +9 -9
  70. metadata +127 -122
  71. checksums.yaml +0 -15
  72. data/.rspec +0 -2
  73. data/.yardopts +0 -10
  74. data/Guardfile +0 -41
  75. data/app/http_shim.rb +0 -71
  76. data/bin/vcd.sh +0 -27
  77. data/config/http_shim.rb +0 -43
  78. data/config/vayacondios.example.yaml +0 -7
  79. data/config/vayacondios.yaml +0 -7
  80. data/examples/java/ItemSetTest.java +0 -76
  81. data/lib/tasks/publish.rake +0 -23
  82. data/lib/tasks/spec.rake +0 -11
  83. data/lib/tasks/yard.rake +0 -2
  84. data/lib/vayacondios/client/config.rb +0 -7
  85. data/lib/vayacondios/client/configliere.rb +0 -38
  86. data/lib/vayacondios/client/cube_client.rb +0 -39
  87. data/lib/vayacondios/client/http_client.rb +0 -49
  88. data/lib/vayacondios/client/itemset.rb +0 -130
  89. data/lib/vayacondios/client/legacy_switch.rb +0 -43
  90. data/lib/vayacondios/client/notifier.rb +0 -123
  91. data/lib/vayacondios/client/zabbix_client.rb +0 -148
  92. data/lib/vayacondios/legacy_switch.rb +0 -43
  93. data/lib/vayacondios/server/errors/bad_request.rb +0 -6
  94. data/lib/vayacondios/server/errors/not_found.rb +0 -6
  95. data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
  96. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
  97. data/lib/vayacondios/server/legacy_switch.rb +0 -43
  98. data/lib/vayacondios/server/model/config_document.rb +0 -89
  99. data/lib/vayacondios/server/model/document.rb +0 -25
  100. data/lib/vayacondios/server/model/event_document.rb +0 -94
  101. data/lib/vayacondios/server/model/itemset_document.rb +0 -126
  102. data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
  103. data/lib/vayacondios/server/rack/jsonize.rb +0 -43
  104. data/lib/vayacondios/server/rack/params.rb +0 -50
  105. data/lib/vayacondios/server/rack/path.rb +0 -23
  106. data/lib/vayacondios/server/rack/path_validation.rb +0 -22
  107. data/lib/vayacondios/version.rb +0 -3
  108. data/lib/vayacondios-client.rb +0 -22
  109. data/scripts/hadoop_monitor/configurable.rb +0 -66
  110. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
  111. data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
  112. data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
  113. data/scripts/hadoop_monitor/hadoopable.rb +0 -65
  114. data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
  115. data/scripts/s3_cataloger/buckets +0 -33
  116. data/scripts/s3_cataloger/foreach_bucket +0 -88
  117. data/scripts/s3_cataloger/parse_ls.py +0 -391
  118. data/spec/client/itemset_legacy_spec.rb +0 -55
  119. data/spec/client/itemset_spec.rb +0 -60
  120. data/spec/client/notifier_spec.rb +0 -120
  121. data/spec/server/config_spec.rb +0 -113
  122. data/spec/server/event_spec.rb +0 -103
  123. data/spec/server/itemset_legacy_spec.rb +0 -320
  124. data/spec/server/itemset_spec.rb +0 -317
  125. data/spec/server/rack/extract_methods_spec.rb +0 -60
  126. data/spec/server/rack/path_spec.rb +0 -36
  127. data/spec/server/rack/path_validation_spec.rb +0 -22
  128. data/spec/server/server_spec.rb +0 -20
  129. data/spec/support/mongo_cleaner.rb +0 -32
  130. data/src/main/java/ItemSetTest.java +0 -76
  131. data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
  132. data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
  133. data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
  134. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
  135. data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
  136. data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
  137. data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
  138. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
  139. data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
  140. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
  141. data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
  142. data/vayacondios-client.gemspec +0 -25
@@ -1,60 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require 'multi_json'
4
-
5
- require_relative '../../lib/vayacondios/client/itemset'
6
-
7
- describe Vayacondios::Client::ItemSet do
8
- context "when initialized" do
9
- it "can be passed a host, port, organization, topic and id" do
10
- Vayacondios::Client::ItemSet.new("foohost", 9999, "fooorg", "footopic", "fooid")
11
- end
12
- end
13
-
14
- context "after instantiation" do
15
- itemset = Vayacondios::Client::ItemSet.new("foohost", 9999, "fooorg", "footopic", "fooid")
16
- ary = ["foo", "bar", "baz"]
17
-
18
- # testing internals here to avoid shimming up HTTP libraries.
19
-
20
- it "generates a put request without a patch header when asked to create" do
21
- Vayacondios.force_legacy_mode false
22
-
23
- req = itemset.instance_eval{_req(:create, ary)}
24
-
25
- req.method.should eql('PUT')
26
- req.body.should eql(MultiJson.encode(contents: ary))
27
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
28
- req.each_header.to_a.should_not include(["x_method", "PATCH"])
29
- end
30
-
31
- it "generates a put request with a patch header when asked to update" do
32
- Vayacondios.force_legacy_mode false
33
-
34
- req = itemset.instance_eval{_req(:update, ary)}
35
-
36
- req.method.should eql('PUT')
37
- req.body.should eql(MultiJson.encode(contents: ary))
38
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
39
- req.each_header.to_a.should include(["x-method", "PATCH"])
40
- end
41
-
42
- it "generates a get request when asked to fetch" do
43
- req = itemset.instance_eval{_req(:fetch)}
44
-
45
- req.method.should eql('GET')
46
- req.body.should be_nil
47
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
48
- end
49
-
50
- it "generates a delete request when asked to remove" do
51
- Vayacondios.force_legacy_mode false
52
-
53
- req = itemset.instance_eval{_req(:remove, ary)}
54
-
55
- req.method.should eql('DELETE')
56
- req.body.should eql(MultiJson.encode(contents: ary))
57
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
58
- end
59
- end
60
- end
@@ -1,120 +0,0 @@
1
- require 'vayacondios-client'
2
-
3
- require 'spec_helper'
4
-
5
- class FakeModel
6
- include Vayacondios::Notifications
7
- end
8
-
9
- describe FakeModel do
10
- context 'including', Vayacondios::Notifications do
11
-
12
- it 'defines an instance method notify()' do
13
- subject.should respond_to(:notify)
14
- end
15
-
16
- it 'adds a configurable attribute :notifier with default' do
17
- subject.notifier.should be_instance_of(Vayacondios.default_notifier.class)
18
- end
19
-
20
- end
21
- end
22
-
23
- describe Vayacondios::Notifier do
24
-
25
- shared_examples_for described_class do
26
- it{ should respond_to(:notify) }
27
- end
28
-
29
- context '.prepare' do
30
- context 'given a Hash-like object' do
31
- let(:hashlike) { double :hashlike, :to_hash => {} }
32
- it 'returns a Hash' do
33
- subject.prepare(hashlike).should be_instance_of(Hash)
34
- end
35
- end
36
-
37
- context 'given a bad argument' do
38
- let(:bad_arg) { 'shazam' }
39
- it 'raises an ArgumentError' do
40
- expect{ subject.prepare(bad_arg) }.to raise_error(ArgumentError, /Cannot notify.*#{bad_arg}.*require a hash-like object/)
41
- end
42
- end
43
- end
44
-
45
- end
46
-
47
- describe Vayacondios::HttpNotifier do
48
- it_behaves_like Vayacondios::Notifier
49
-
50
- its(:client){ should be_instance_of(Vayacondios::HttpClient) }
51
-
52
- context '#notify' do
53
- let(:test_client) { double :client }
54
- let(:topic) { 'weeeeeeeeeee' }
55
- let(:cargo) { Hash.new }
56
-
57
- before do
58
- subject.stub(:client).and_return(test_client)
59
- subject.stub(:prepare).and_return(cargo)
60
- end
61
-
62
- it 'notifies its client correctly' do
63
- test_client.should_receive(:insert).with(cargo, :event, topic)
64
- subject.notify(topic, cargo)
65
- end
66
- end
67
- end
68
-
69
- describe Vayacondios::LogNotifier do
70
- it_behaves_like Vayacondios::Notifier
71
-
72
- its(:client){ should be_instance_of(Logger) }
73
-
74
- context '#notify' do
75
- let(:test_client) { double :client }
76
- let(:topic) { 'weeeeeeeeeee' }
77
- let(:cargo) { Hash.new }
78
-
79
- before do
80
- subject.stub(:client).and_return(test_client)
81
- subject.stub(:prepare).and_return(cargo)
82
- end
83
-
84
- it 'notifies its client correctly' do
85
- test_client.should_receive(:info).with(/Notification.*#{topic}/)
86
- subject.notify(topic, cargo)
87
- end
88
- end
89
-
90
- end
91
-
92
- describe Vayacondios::NotifierFactory do
93
- context '.receive' do
94
- context 'given :http' do
95
- it 'builds a HttpNotifier' do
96
- described_class.receive(type: 'http').should be_instance_of(Vayacondios::HttpNotifier)
97
- end
98
- end
99
-
100
- context 'given :log' do
101
- it 'builds a LogNotifier' do
102
- described_class.receive(type: 'log').should be_instance_of(Vayacondios::LogNotifier)
103
- end
104
- end
105
-
106
- context 'given a bad argument' do
107
- it 'raises an ArgumentError' do
108
- expect{ described_class.receive(type: 'bad') }.to raise_error(ArgumentError, /not a valid build option/)
109
- end
110
- end
111
- end
112
- end
113
-
114
- describe Vayacondios do
115
-
116
- it 'has a class method notify()' do
117
- described_class.should respond_to(:notify)
118
- end
119
-
120
- end
@@ -1,113 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require 'multi_json'
4
-
5
- require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
-
7
- describe HttpShim do
8
- include Goliath::TestHelper
9
-
10
- let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.error}" } }
11
-
12
- context 'Configuration management' do
13
- it 'requires a topic' do
14
- with_api(HttpShim) do |api|
15
- put_request({
16
- :path => '/v1/infochimps/config/',
17
- :body => MultiJson.dump({:level=>"awesome"}),
18
- :head => { :content_type => 'application/json' }
19
- }, err) do |c|
20
- c.response_header.status.should == 400
21
- end
22
- end
23
- end
24
-
25
- it 'requires an id' do
26
- with_api(HttpShim) do |api|
27
- put_request({
28
- :path => '/v1/infochimps/config/power',
29
- :body => MultiJson.dump({:level=>"awesome"}),
30
- :head => { :content_type => 'application/json' }
31
- }, err) do |c|
32
- c.response_header.status.should == 400
33
- end
34
- end
35
- end
36
-
37
- it 'stores configuration' do
38
- with_api(HttpShim) do |api|
39
- put_request({
40
- :path => '/v1/infochimps/config/power/level',
41
- :body => MultiJson.dump({:level=>"awesome"}),
42
- :head => { :content_type => 'application/json' }
43
- }, err) do |c|
44
- c.response_header.status.should == 200
45
- end
46
-
47
- get_mongo_db do |db|
48
- db.collection("infochimps.config").find_one({:_id => "power"}).should eql({"_id" => "power", "level" => "awesome"})
49
- end
50
- end
51
- end
52
-
53
- it 'rejects deep IDs' do
54
- with_api(HttpShim) do |api|
55
- put_request({
56
- :path => '/v1/infochimps/config/power/level/is/invalid',
57
- :body => MultiJson.dump({:level=>"awesome"}),
58
- :head => { :content_type => 'application/json' }
59
- }, err) do |c|
60
- c.response_header.status.should == 400
61
- end
62
-
63
- get_mongo_db do |db|
64
- db.collection("infochimps.config").find_one({:_id => "power"}).should be_nil
65
- end
66
- end
67
- end
68
-
69
- it 'retrieves configuration' do
70
- with_api(HttpShim) do |api|
71
- put_request({
72
- :path => '/v1/infochimps/config/power/level',
73
- :body => MultiJson.dump({:level=>"awesome"}),
74
- :head => { :content_type => 'application/json' }
75
- }, err)
76
- end
77
- with_api(HttpShim) do |api|
78
- get_request({:path => '/v1/infochimps/config/power/level'}, err) do |c|
79
- c.response_header.status.should == 200
80
- MultiJson.load(c.response).should eql({"level" => "awesome"})
81
- end
82
- end
83
- end
84
-
85
- it 'merge deep configuration' do
86
- with_api(HttpShim) do |api|
87
- put_request({
88
- :path => '/v1/infochimps/config/merge/test',
89
- :body => MultiJson.dump({ :foo => { :bar => 3 } }),
90
- :head => { :content_type => 'application/json' }
91
- }, err)
92
- end
93
- with_api(HttpShim) do |api|
94
- put_request({
95
- :path => '/v1/infochimps/config/merge/test',
96
- :body => MultiJson.dump({ :foo => { :baz => 7 } }),
97
- :head => { :content_type => 'application/json' }
98
- }, err)
99
- end
100
- with_api(HttpShim) do |api|
101
- get_request({:path => '/v1/infochimps/config/merge/test'}, err) do |c|
102
- c.response_header.status.should == 200
103
- MultiJson.load(c.response).should eql({
104
- "foo" => {
105
- "bar" => 3,
106
- "baz" => 7
107
- }
108
- })
109
- end
110
- end
111
- end
112
- end
113
- end
@@ -1,103 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require 'multi_json'
4
-
5
- require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
-
7
- describe HttpShim do
8
- include Goliath::TestHelper
9
-
10
- let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.response}" } }
11
-
12
- context 'Event tracking' do
13
- it 'requires a topic' do
14
- with_api(HttpShim) do |api|
15
- put_request({
16
- :path => '/v1/infochimps/event/',
17
- :body => MultiJson.dump({:level=>"awesome"}),
18
- :head => { :content_type => 'application/json' }
19
- }, err) do |c|
20
- c.response_header.status.should == 400
21
- end
22
- end
23
- end
24
-
25
- it 'does not require an id' do
26
- with_api(HttpShim) do |api|
27
- put_request({
28
- :path => '/v1/infochimps/event/power',
29
- :body => MultiJson.dump({:level=>"awesome"}),
30
- :head => { :content_type => 'application/json' }
31
- }, err) do |c|
32
- c.response_header.status.should == 200
33
- end
34
- end
35
- end
36
-
37
- it 'will accept an id' do
38
- with_api(HttpShim) do |api|
39
- put_request({
40
- :path => '/v1/infochimps/event/power/level',
41
- :body => MultiJson.dump({:level=>"awesome"}),
42
- :head => { :content_type => 'application/json' }
43
- }, err) do |c|
44
- c.response_header.status.should == 200
45
- end
46
- end
47
- end
48
-
49
- it 'rejects deep IDs' do
50
- with_api(HttpShim) do |api|
51
- put_request({
52
- :path => '/v1/infochimps/event/power/level/is/invalid',
53
- :body => MultiJson.dump({:level=>"awesome"}),
54
- :head => { :content_type => 'application/json' }
55
- }, err) do |c|
56
- c.response_header.status.should == 400
57
- end
58
-
59
- get_mongo_db do |db|
60
- db.collection("infochimps.config").find_one({:_id => "power"}).should be_nil
61
- end
62
- end
63
- end
64
-
65
- it 'stores events' do
66
- with_api(HttpShim) do |api|
67
- put_request({
68
- :path => '/v1/infochimps/event/power/level',
69
- :body => MultiJson.dump({:level=>"awesome"}),
70
- :head => { :content_type => 'application/json' }
71
- }, err) do |c|
72
- c.response_header.status.should == 200
73
- end
74
-
75
- get_mongo_db do |db|
76
- doc = db.collection("infochimps.power.events").find_one({:_id => "level"})
77
- doc["_id"].should eql "level"
78
- doc["d"].should eql ({"level" => "awesome"})
79
- doc["t"].should be_within(1).of Time.now
80
- end
81
- end
82
- end
83
-
84
- it 'retrieves events' do
85
- current_time = Time.now
86
- with_api(HttpShim) do |api|
87
- put_request({
88
- :path => '/v1/infochimps/event/power/level',
89
- :body => MultiJson.dump({:level=>"awesome", :_timestamp => current_time}),
90
- :head => { :content_type => 'application/json' }
91
- }, err) do |c|
92
- c.response_header.status.should == 200
93
- end
94
- end
95
- with_api(HttpShim) do |api|
96
- get_request({:path => '/v1/infochimps/event/power/level'}, err) do |c|
97
- c.response_header.status.should == 200
98
- MultiJson.load(c.response).should eql ({"level" => "awesome", "_timestamp" => current_time.to_s})
99
- end
100
- end
101
- end
102
- end
103
- end