vayacondios-server 0.2.11 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. data/.gitignore +3 -1
  2. data/.travis.yml +2 -0
  3. data/Gemfile +15 -9
  4. data/LICENSE.md +2 -6
  5. data/Procfile +1 -1
  6. data/README.md +656 -111
  7. data/Rakefile +89 -6
  8. data/bin/vcd +10 -0
  9. data/bin/vcd-server +8 -0
  10. data/config/database.yml +6 -0
  11. data/config/spec.example.yml +18 -0
  12. data/config/vayacondios.example.yml +15 -0
  13. data/config/vcd-server.rb +37 -0
  14. data/examples/configuration.rb +56 -0
  15. data/examples/event_stream.rb +19 -0
  16. data/examples/simple.rb +61 -0
  17. data/features/event.feature +319 -0
  18. data/features/events.feature +208 -0
  19. data/features/stash.feature +840 -0
  20. data/features/stashes.feature +492 -0
  21. data/features/step_definitions/stash_steps.rb +113 -0
  22. data/features/stream.feature +30 -0
  23. data/features/support/em.rb +14 -0
  24. data/features/support/env.rb +13 -0
  25. data/lib/vayacondios/configuration.rb +63 -0
  26. data/lib/vayacondios/server/api.rb +126 -0
  27. data/lib/vayacondios/server/api_options.rb +56 -0
  28. data/lib/vayacondios/server/configuration.rb +23 -0
  29. data/lib/vayacondios/server/driver.rb +71 -0
  30. data/lib/vayacondios/server/drivers/mongo.rb +126 -0
  31. data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
  32. data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
  33. data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
  34. data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
  35. data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
  36. data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
  37. data/lib/vayacondios/server/models/document.rb +87 -0
  38. data/lib/vayacondios/server/models/event.rb +198 -0
  39. data/lib/vayacondios/server/models/stash.rb +100 -0
  40. data/lib/vayacondios/server.rb +35 -0
  41. data/lib/vayacondios-server.rb +19 -13
  42. data/lib/vayacondios.rb +22 -0
  43. data/pom.xml +124 -4
  44. data/spec/configuration_spec.rb +41 -0
  45. data/spec/server/api_options_spec.rb +32 -0
  46. data/spec/server/api_spec.rb +279 -0
  47. data/spec/server/configuration_spec.rb +27 -0
  48. data/spec/server/drivers/mongo_spec.rb +107 -0
  49. data/spec/server/handlers/event_handler_spec.rb +62 -0
  50. data/spec/server/handlers/events_handler_spec.rb +51 -0
  51. data/spec/server/handlers/stash_handler_spec.rb +68 -0
  52. data/spec/server/handlers/stashes_handler_spec.rb +50 -0
  53. data/spec/server/handlers/stream_handler_spec.rb +5 -0
  54. data/spec/server/models/document_spec.rb +9 -0
  55. data/spec/server/models/event_spec.rb +185 -0
  56. data/spec/server/models/stash_spec.rb +95 -0
  57. data/spec/spec_helper.rb +23 -3
  58. data/spec/support/database_helper.rb +42 -0
  59. data/spec/support/log_helper.rb +19 -0
  60. data/spec/support/shared_context_for_events.rb +22 -0
  61. data/spec/support/shared_context_for_stashes.rb +24 -0
  62. data/spec/support/shared_examples_for_handlers.rb +32 -0
  63. data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
  64. data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
  65. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
  66. data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
  67. data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
  68. data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
  69. data/vayacondios-server.gemspec +9 -9
  70. metadata +127 -122
  71. checksums.yaml +0 -15
  72. data/.rspec +0 -2
  73. data/.yardopts +0 -10
  74. data/Guardfile +0 -41
  75. data/app/http_shim.rb +0 -71
  76. data/bin/vcd.sh +0 -27
  77. data/config/http_shim.rb +0 -43
  78. data/config/vayacondios.example.yaml +0 -7
  79. data/config/vayacondios.yaml +0 -7
  80. data/examples/java/ItemSetTest.java +0 -76
  81. data/lib/tasks/publish.rake +0 -23
  82. data/lib/tasks/spec.rake +0 -11
  83. data/lib/tasks/yard.rake +0 -2
  84. data/lib/vayacondios/client/config.rb +0 -7
  85. data/lib/vayacondios/client/configliere.rb +0 -38
  86. data/lib/vayacondios/client/cube_client.rb +0 -39
  87. data/lib/vayacondios/client/http_client.rb +0 -49
  88. data/lib/vayacondios/client/itemset.rb +0 -130
  89. data/lib/vayacondios/client/legacy_switch.rb +0 -43
  90. data/lib/vayacondios/client/notifier.rb +0 -123
  91. data/lib/vayacondios/client/zabbix_client.rb +0 -148
  92. data/lib/vayacondios/legacy_switch.rb +0 -43
  93. data/lib/vayacondios/server/errors/bad_request.rb +0 -6
  94. data/lib/vayacondios/server/errors/not_found.rb +0 -6
  95. data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
  96. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
  97. data/lib/vayacondios/server/legacy_switch.rb +0 -43
  98. data/lib/vayacondios/server/model/config_document.rb +0 -89
  99. data/lib/vayacondios/server/model/document.rb +0 -25
  100. data/lib/vayacondios/server/model/event_document.rb +0 -94
  101. data/lib/vayacondios/server/model/itemset_document.rb +0 -126
  102. data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
  103. data/lib/vayacondios/server/rack/jsonize.rb +0 -43
  104. data/lib/vayacondios/server/rack/params.rb +0 -50
  105. data/lib/vayacondios/server/rack/path.rb +0 -23
  106. data/lib/vayacondios/server/rack/path_validation.rb +0 -22
  107. data/lib/vayacondios/version.rb +0 -3
  108. data/lib/vayacondios-client.rb +0 -22
  109. data/scripts/hadoop_monitor/configurable.rb +0 -66
  110. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
  111. data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
  112. data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
  113. data/scripts/hadoop_monitor/hadoopable.rb +0 -65
  114. data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
  115. data/scripts/s3_cataloger/buckets +0 -33
  116. data/scripts/s3_cataloger/foreach_bucket +0 -88
  117. data/scripts/s3_cataloger/parse_ls.py +0 -391
  118. data/spec/client/itemset_legacy_spec.rb +0 -55
  119. data/spec/client/itemset_spec.rb +0 -60
  120. data/spec/client/notifier_spec.rb +0 -120
  121. data/spec/server/config_spec.rb +0 -113
  122. data/spec/server/event_spec.rb +0 -103
  123. data/spec/server/itemset_legacy_spec.rb +0 -320
  124. data/spec/server/itemset_spec.rb +0 -317
  125. data/spec/server/rack/extract_methods_spec.rb +0 -60
  126. data/spec/server/rack/path_spec.rb +0 -36
  127. data/spec/server/rack/path_validation_spec.rb +0 -22
  128. data/spec/server/server_spec.rb +0 -20
  129. data/spec/support/mongo_cleaner.rb +0 -32
  130. data/src/main/java/ItemSetTest.java +0 -76
  131. data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
  132. data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
  133. data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
  134. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
  135. data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
  136. data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
  137. data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
  138. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
  139. data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
  140. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
  141. data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
  142. data/vayacondios-client.gemspec +0 -25
@@ -1,60 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require 'multi_json'
4
-
5
- require_relative '../../lib/vayacondios/client/itemset'
6
-
7
- describe Vayacondios::Client::ItemSet do
8
- context "when initialized" do
9
- it "can be passed a host, port, organization, topic and id" do
10
- Vayacondios::Client::ItemSet.new("foohost", 9999, "fooorg", "footopic", "fooid")
11
- end
12
- end
13
-
14
- context "after instantiation" do
15
- itemset = Vayacondios::Client::ItemSet.new("foohost", 9999, "fooorg", "footopic", "fooid")
16
- ary = ["foo", "bar", "baz"]
17
-
18
- # testing internals here to avoid shimming up HTTP libraries.
19
-
20
- it "generates a put request without a patch header when asked to create" do
21
- Vayacondios.force_legacy_mode false
22
-
23
- req = itemset.instance_eval{_req(:create, ary)}
24
-
25
- req.method.should eql('PUT')
26
- req.body.should eql(MultiJson.encode(contents: ary))
27
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
28
- req.each_header.to_a.should_not include(["x_method", "PATCH"])
29
- end
30
-
31
- it "generates a put request with a patch header when asked to update" do
32
- Vayacondios.force_legacy_mode false
33
-
34
- req = itemset.instance_eval{_req(:update, ary)}
35
-
36
- req.method.should eql('PUT')
37
- req.body.should eql(MultiJson.encode(contents: ary))
38
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
39
- req.each_header.to_a.should include(["x-method", "PATCH"])
40
- end
41
-
42
- it "generates a get request when asked to fetch" do
43
- req = itemset.instance_eval{_req(:fetch)}
44
-
45
- req.method.should eql('GET')
46
- req.body.should be_nil
47
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
48
- end
49
-
50
- it "generates a delete request when asked to remove" do
51
- Vayacondios.force_legacy_mode false
52
-
53
- req = itemset.instance_eval{_req(:remove, ary)}
54
-
55
- req.method.should eql('DELETE')
56
- req.body.should eql(MultiJson.encode(contents: ary))
57
- req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
58
- end
59
- end
60
- end
@@ -1,120 +0,0 @@
1
- require 'vayacondios-client'
2
-
3
- require 'spec_helper'
4
-
5
- class FakeModel
6
- include Vayacondios::Notifications
7
- end
8
-
9
- describe FakeModel do
10
- context 'including', Vayacondios::Notifications do
11
-
12
- it 'defines an instance method notify()' do
13
- subject.should respond_to(:notify)
14
- end
15
-
16
- it 'adds a configurable attribute :notifier with default' do
17
- subject.notifier.should be_instance_of(Vayacondios.default_notifier.class)
18
- end
19
-
20
- end
21
- end
22
-
23
- describe Vayacondios::Notifier do
24
-
25
- shared_examples_for described_class do
26
- it{ should respond_to(:notify) }
27
- end
28
-
29
- context '.prepare' do
30
- context 'given a Hash-like object' do
31
- let(:hashlike) { double :hashlike, :to_hash => {} }
32
- it 'returns a Hash' do
33
- subject.prepare(hashlike).should be_instance_of(Hash)
34
- end
35
- end
36
-
37
- context 'given a bad argument' do
38
- let(:bad_arg) { 'shazam' }
39
- it 'raises an ArgumentError' do
40
- expect{ subject.prepare(bad_arg) }.to raise_error(ArgumentError, /Cannot notify.*#{bad_arg}.*require a hash-like object/)
41
- end
42
- end
43
- end
44
-
45
- end
46
-
47
- describe Vayacondios::HttpNotifier do
48
- it_behaves_like Vayacondios::Notifier
49
-
50
- its(:client){ should be_instance_of(Vayacondios::HttpClient) }
51
-
52
- context '#notify' do
53
- let(:test_client) { double :client }
54
- let(:topic) { 'weeeeeeeeeee' }
55
- let(:cargo) { Hash.new }
56
-
57
- before do
58
- subject.stub(:client).and_return(test_client)
59
- subject.stub(:prepare).and_return(cargo)
60
- end
61
-
62
- it 'notifies its client correctly' do
63
- test_client.should_receive(:insert).with(cargo, :event, topic)
64
- subject.notify(topic, cargo)
65
- end
66
- end
67
- end
68
-
69
- describe Vayacondios::LogNotifier do
70
- it_behaves_like Vayacondios::Notifier
71
-
72
- its(:client){ should be_instance_of(Logger) }
73
-
74
- context '#notify' do
75
- let(:test_client) { double :client }
76
- let(:topic) { 'weeeeeeeeeee' }
77
- let(:cargo) { Hash.new }
78
-
79
- before do
80
- subject.stub(:client).and_return(test_client)
81
- subject.stub(:prepare).and_return(cargo)
82
- end
83
-
84
- it 'notifies its client correctly' do
85
- test_client.should_receive(:info).with(/Notification.*#{topic}/)
86
- subject.notify(topic, cargo)
87
- end
88
- end
89
-
90
- end
91
-
92
- describe Vayacondios::NotifierFactory do
93
- context '.receive' do
94
- context 'given :http' do
95
- it 'builds a HttpNotifier' do
96
- described_class.receive(type: 'http').should be_instance_of(Vayacondios::HttpNotifier)
97
- end
98
- end
99
-
100
- context 'given :log' do
101
- it 'builds a LogNotifier' do
102
- described_class.receive(type: 'log').should be_instance_of(Vayacondios::LogNotifier)
103
- end
104
- end
105
-
106
- context 'given a bad argument' do
107
- it 'raises an ArgumentError' do
108
- expect{ described_class.receive(type: 'bad') }.to raise_error(ArgumentError, /not a valid build option/)
109
- end
110
- end
111
- end
112
- end
113
-
114
- describe Vayacondios do
115
-
116
- it 'has a class method notify()' do
117
- described_class.should respond_to(:notify)
118
- end
119
-
120
- end
@@ -1,113 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require 'multi_json'
4
-
5
- require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
-
7
- describe HttpShim do
8
- include Goliath::TestHelper
9
-
10
- let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.error}" } }
11
-
12
- context 'Configuration management' do
13
- it 'requires a topic' do
14
- with_api(HttpShim) do |api|
15
- put_request({
16
- :path => '/v1/infochimps/config/',
17
- :body => MultiJson.dump({:level=>"awesome"}),
18
- :head => { :content_type => 'application/json' }
19
- }, err) do |c|
20
- c.response_header.status.should == 400
21
- end
22
- end
23
- end
24
-
25
- it 'requires an id' do
26
- with_api(HttpShim) do |api|
27
- put_request({
28
- :path => '/v1/infochimps/config/power',
29
- :body => MultiJson.dump({:level=>"awesome"}),
30
- :head => { :content_type => 'application/json' }
31
- }, err) do |c|
32
- c.response_header.status.should == 400
33
- end
34
- end
35
- end
36
-
37
- it 'stores configuration' do
38
- with_api(HttpShim) do |api|
39
- put_request({
40
- :path => '/v1/infochimps/config/power/level',
41
- :body => MultiJson.dump({:level=>"awesome"}),
42
- :head => { :content_type => 'application/json' }
43
- }, err) do |c|
44
- c.response_header.status.should == 200
45
- end
46
-
47
- get_mongo_db do |db|
48
- db.collection("infochimps.config").find_one({:_id => "power"}).should eql({"_id" => "power", "level" => "awesome"})
49
- end
50
- end
51
- end
52
-
53
- it 'rejects deep IDs' do
54
- with_api(HttpShim) do |api|
55
- put_request({
56
- :path => '/v1/infochimps/config/power/level/is/invalid',
57
- :body => MultiJson.dump({:level=>"awesome"}),
58
- :head => { :content_type => 'application/json' }
59
- }, err) do |c|
60
- c.response_header.status.should == 400
61
- end
62
-
63
- get_mongo_db do |db|
64
- db.collection("infochimps.config").find_one({:_id => "power"}).should be_nil
65
- end
66
- end
67
- end
68
-
69
- it 'retrieves configuration' do
70
- with_api(HttpShim) do |api|
71
- put_request({
72
- :path => '/v1/infochimps/config/power/level',
73
- :body => MultiJson.dump({:level=>"awesome"}),
74
- :head => { :content_type => 'application/json' }
75
- }, err)
76
- end
77
- with_api(HttpShim) do |api|
78
- get_request({:path => '/v1/infochimps/config/power/level'}, err) do |c|
79
- c.response_header.status.should == 200
80
- MultiJson.load(c.response).should eql({"level" => "awesome"})
81
- end
82
- end
83
- end
84
-
85
- it 'merge deep configuration' do
86
- with_api(HttpShim) do |api|
87
- put_request({
88
- :path => '/v1/infochimps/config/merge/test',
89
- :body => MultiJson.dump({ :foo => { :bar => 3 } }),
90
- :head => { :content_type => 'application/json' }
91
- }, err)
92
- end
93
- with_api(HttpShim) do |api|
94
- put_request({
95
- :path => '/v1/infochimps/config/merge/test',
96
- :body => MultiJson.dump({ :foo => { :baz => 7 } }),
97
- :head => { :content_type => 'application/json' }
98
- }, err)
99
- end
100
- with_api(HttpShim) do |api|
101
- get_request({:path => '/v1/infochimps/config/merge/test'}, err) do |c|
102
- c.response_header.status.should == 200
103
- MultiJson.load(c.response).should eql({
104
- "foo" => {
105
- "bar" => 3,
106
- "baz" => 7
107
- }
108
- })
109
- end
110
- end
111
- end
112
- end
113
- end
@@ -1,103 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require 'multi_json'
4
-
5
- require File.join(File.dirname(__FILE__), '../../', 'app/http_shim')
6
-
7
- describe HttpShim do
8
- include Goliath::TestHelper
9
-
10
- let(:err) { Proc.new{ |c| fail "HTTP Request Failed #{c.response}" } }
11
-
12
- context 'Event tracking' do
13
- it 'requires a topic' do
14
- with_api(HttpShim) do |api|
15
- put_request({
16
- :path => '/v1/infochimps/event/',
17
- :body => MultiJson.dump({:level=>"awesome"}),
18
- :head => { :content_type => 'application/json' }
19
- }, err) do |c|
20
- c.response_header.status.should == 400
21
- end
22
- end
23
- end
24
-
25
- it 'does not require an id' do
26
- with_api(HttpShim) do |api|
27
- put_request({
28
- :path => '/v1/infochimps/event/power',
29
- :body => MultiJson.dump({:level=>"awesome"}),
30
- :head => { :content_type => 'application/json' }
31
- }, err) do |c|
32
- c.response_header.status.should == 200
33
- end
34
- end
35
- end
36
-
37
- it 'will accept an id' do
38
- with_api(HttpShim) do |api|
39
- put_request({
40
- :path => '/v1/infochimps/event/power/level',
41
- :body => MultiJson.dump({:level=>"awesome"}),
42
- :head => { :content_type => 'application/json' }
43
- }, err) do |c|
44
- c.response_header.status.should == 200
45
- end
46
- end
47
- end
48
-
49
- it 'rejects deep IDs' do
50
- with_api(HttpShim) do |api|
51
- put_request({
52
- :path => '/v1/infochimps/event/power/level/is/invalid',
53
- :body => MultiJson.dump({:level=>"awesome"}),
54
- :head => { :content_type => 'application/json' }
55
- }, err) do |c|
56
- c.response_header.status.should == 400
57
- end
58
-
59
- get_mongo_db do |db|
60
- db.collection("infochimps.config").find_one({:_id => "power"}).should be_nil
61
- end
62
- end
63
- end
64
-
65
- it 'stores events' do
66
- with_api(HttpShim) do |api|
67
- put_request({
68
- :path => '/v1/infochimps/event/power/level',
69
- :body => MultiJson.dump({:level=>"awesome"}),
70
- :head => { :content_type => 'application/json' }
71
- }, err) do |c|
72
- c.response_header.status.should == 200
73
- end
74
-
75
- get_mongo_db do |db|
76
- doc = db.collection("infochimps.power.events").find_one({:_id => "level"})
77
- doc["_id"].should eql "level"
78
- doc["d"].should eql ({"level" => "awesome"})
79
- doc["t"].should be_within(1).of Time.now
80
- end
81
- end
82
- end
83
-
84
- it 'retrieves events' do
85
- current_time = Time.now
86
- with_api(HttpShim) do |api|
87
- put_request({
88
- :path => '/v1/infochimps/event/power/level',
89
- :body => MultiJson.dump({:level=>"awesome", :_timestamp => current_time}),
90
- :head => { :content_type => 'application/json' }
91
- }, err) do |c|
92
- c.response_header.status.should == 200
93
- end
94
- end
95
- with_api(HttpShim) do |api|
96
- get_request({:path => '/v1/infochimps/event/power/level'}, err) do |c|
97
- c.response_header.status.should == 200
98
- MultiJson.load(c.response).should eql ({"level" => "awesome", "_timestamp" => current_time.to_s})
99
- end
100
- end
101
- end
102
- end
103
- end