vayacondios-server 0.2.11 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (142) hide show
  1. data/.gitignore +3 -1
  2. data/.travis.yml +2 -0
  3. data/Gemfile +15 -9
  4. data/LICENSE.md +2 -6
  5. data/Procfile +1 -1
  6. data/README.md +656 -111
  7. data/Rakefile +89 -6
  8. data/bin/vcd +10 -0
  9. data/bin/vcd-server +8 -0
  10. data/config/database.yml +6 -0
  11. data/config/spec.example.yml +18 -0
  12. data/config/vayacondios.example.yml +15 -0
  13. data/config/vcd-server.rb +37 -0
  14. data/examples/configuration.rb +56 -0
  15. data/examples/event_stream.rb +19 -0
  16. data/examples/simple.rb +61 -0
  17. data/features/event.feature +319 -0
  18. data/features/events.feature +208 -0
  19. data/features/stash.feature +840 -0
  20. data/features/stashes.feature +492 -0
  21. data/features/step_definitions/stash_steps.rb +113 -0
  22. data/features/stream.feature +30 -0
  23. data/features/support/em.rb +14 -0
  24. data/features/support/env.rb +13 -0
  25. data/lib/vayacondios/configuration.rb +63 -0
  26. data/lib/vayacondios/server/api.rb +126 -0
  27. data/lib/vayacondios/server/api_options.rb +56 -0
  28. data/lib/vayacondios/server/configuration.rb +23 -0
  29. data/lib/vayacondios/server/driver.rb +71 -0
  30. data/lib/vayacondios/server/drivers/mongo.rb +126 -0
  31. data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
  32. data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
  33. data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
  34. data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
  35. data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
  36. data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
  37. data/lib/vayacondios/server/models/document.rb +87 -0
  38. data/lib/vayacondios/server/models/event.rb +198 -0
  39. data/lib/vayacondios/server/models/stash.rb +100 -0
  40. data/lib/vayacondios/server.rb +35 -0
  41. data/lib/vayacondios-server.rb +19 -13
  42. data/lib/vayacondios.rb +22 -0
  43. data/pom.xml +124 -4
  44. data/spec/configuration_spec.rb +41 -0
  45. data/spec/server/api_options_spec.rb +32 -0
  46. data/spec/server/api_spec.rb +279 -0
  47. data/spec/server/configuration_spec.rb +27 -0
  48. data/spec/server/drivers/mongo_spec.rb +107 -0
  49. data/spec/server/handlers/event_handler_spec.rb +62 -0
  50. data/spec/server/handlers/events_handler_spec.rb +51 -0
  51. data/spec/server/handlers/stash_handler_spec.rb +68 -0
  52. data/spec/server/handlers/stashes_handler_spec.rb +50 -0
  53. data/spec/server/handlers/stream_handler_spec.rb +5 -0
  54. data/spec/server/models/document_spec.rb +9 -0
  55. data/spec/server/models/event_spec.rb +185 -0
  56. data/spec/server/models/stash_spec.rb +95 -0
  57. data/spec/spec_helper.rb +23 -3
  58. data/spec/support/database_helper.rb +42 -0
  59. data/spec/support/log_helper.rb +19 -0
  60. data/spec/support/shared_context_for_events.rb +22 -0
  61. data/spec/support/shared_context_for_stashes.rb +24 -0
  62. data/spec/support/shared_examples_for_handlers.rb +32 -0
  63. data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
  64. data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
  65. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
  66. data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
  67. data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
  68. data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
  69. data/vayacondios-server.gemspec +9 -9
  70. metadata +127 -122
  71. checksums.yaml +0 -15
  72. data/.rspec +0 -2
  73. data/.yardopts +0 -10
  74. data/Guardfile +0 -41
  75. data/app/http_shim.rb +0 -71
  76. data/bin/vcd.sh +0 -27
  77. data/config/http_shim.rb +0 -43
  78. data/config/vayacondios.example.yaml +0 -7
  79. data/config/vayacondios.yaml +0 -7
  80. data/examples/java/ItemSetTest.java +0 -76
  81. data/lib/tasks/publish.rake +0 -23
  82. data/lib/tasks/spec.rake +0 -11
  83. data/lib/tasks/yard.rake +0 -2
  84. data/lib/vayacondios/client/config.rb +0 -7
  85. data/lib/vayacondios/client/configliere.rb +0 -38
  86. data/lib/vayacondios/client/cube_client.rb +0 -39
  87. data/lib/vayacondios/client/http_client.rb +0 -49
  88. data/lib/vayacondios/client/itemset.rb +0 -130
  89. data/lib/vayacondios/client/legacy_switch.rb +0 -43
  90. data/lib/vayacondios/client/notifier.rb +0 -123
  91. data/lib/vayacondios/client/zabbix_client.rb +0 -148
  92. data/lib/vayacondios/legacy_switch.rb +0 -43
  93. data/lib/vayacondios/server/errors/bad_request.rb +0 -6
  94. data/lib/vayacondios/server/errors/not_found.rb +0 -6
  95. data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
  96. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
  97. data/lib/vayacondios/server/legacy_switch.rb +0 -43
  98. data/lib/vayacondios/server/model/config_document.rb +0 -89
  99. data/lib/vayacondios/server/model/document.rb +0 -25
  100. data/lib/vayacondios/server/model/event_document.rb +0 -94
  101. data/lib/vayacondios/server/model/itemset_document.rb +0 -126
  102. data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
  103. data/lib/vayacondios/server/rack/jsonize.rb +0 -43
  104. data/lib/vayacondios/server/rack/params.rb +0 -50
  105. data/lib/vayacondios/server/rack/path.rb +0 -23
  106. data/lib/vayacondios/server/rack/path_validation.rb +0 -22
  107. data/lib/vayacondios/version.rb +0 -3
  108. data/lib/vayacondios-client.rb +0 -22
  109. data/scripts/hadoop_monitor/configurable.rb +0 -66
  110. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
  111. data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
  112. data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
  113. data/scripts/hadoop_monitor/hadoopable.rb +0 -65
  114. data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
  115. data/scripts/s3_cataloger/buckets +0 -33
  116. data/scripts/s3_cataloger/foreach_bucket +0 -88
  117. data/scripts/s3_cataloger/parse_ls.py +0 -391
  118. data/spec/client/itemset_legacy_spec.rb +0 -55
  119. data/spec/client/itemset_spec.rb +0 -60
  120. data/spec/client/notifier_spec.rb +0 -120
  121. data/spec/server/config_spec.rb +0 -113
  122. data/spec/server/event_spec.rb +0 -103
  123. data/spec/server/itemset_legacy_spec.rb +0 -320
  124. data/spec/server/itemset_spec.rb +0 -317
  125. data/spec/server/rack/extract_methods_spec.rb +0 -60
  126. data/spec/server/rack/path_spec.rb +0 -36
  127. data/spec/server/rack/path_validation_spec.rb +0 -22
  128. data/spec/server/server_spec.rb +0 -20
  129. data/spec/support/mongo_cleaner.rb +0 -32
  130. data/src/main/java/ItemSetTest.java +0 -76
  131. data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
  132. data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
  133. data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
  134. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
  135. data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
  136. data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
  137. data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
  138. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
  139. data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
  140. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
  141. data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
  142. data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,267 @@
1
+ package com.infochimps.vayacondios;
2
+
3
+ import java.util.Map;
4
+ import java.util.HashMap;
5
+ import java.util.List;
6
+ import java.util.ArrayList;
7
+ import java.util.Set;
8
+
9
+ import java.net.UnknownHostException;
10
+
11
+ import org.junit.Before;
12
+ import org.junit.After;
13
+ import org.junit.Test;
14
+ import org.junit.Ignore;
15
+ import org.junit.runner.RunWith;
16
+ import org.junit.runners.JUnit4;
17
+ import static org.junit.Assert.assertEquals;
18
+ import com.infochimps.vayacondios.test.IntegrationTest;
19
+ import org.junit.experimental.categories.Category;
20
+
21
+ import com.mongodb.MongoClient;
22
+ import com.mongodb.DB;
23
+
24
+ @RunWith(JUnit4.class)
25
+ @Category(IntegrationTest.class)
26
+ public class HTTPClientIT {
27
+
28
+ private String organization = "organization";
29
+ private String topic = "topic";
30
+ private String id = "id";
31
+
32
+ private HTTPClient client;
33
+
34
+ private Map event() {
35
+ Map e = new HashMap();
36
+ e.put("foo", "bar");
37
+ e.put("baz", 12.0);
38
+ return e;
39
+ }
40
+
41
+ private List list() {
42
+ List l = new ArrayList();
43
+ l.add(1.0);
44
+ l.add("2");
45
+ l.add("three");
46
+ return l;
47
+ }
48
+
49
+ private Map stash() {
50
+ Map s = new HashMap();
51
+ s.put("map", event());
52
+ s.put("list", list());
53
+ s.put("string", "hello");
54
+ s.put("double", 3.1415);
55
+ return s;
56
+ }
57
+
58
+ private Map query() {
59
+ Map q = new HashMap();
60
+ q.put("foo", "bar");
61
+ return q;
62
+ }
63
+
64
+ @Before
65
+ public void createClient() {
66
+ client = new HTTPClient(organization);
67
+ }
68
+
69
+ @Before
70
+ public void cleanMongo() {
71
+ try {
72
+ MongoClient mongo = new MongoClient( "localhost" , 27017 );
73
+ DB database = mongo.getDB( "vayacondios_test" );
74
+ Set<String> collections = database.getCollectionNames();
75
+ for (String collection : collections) {
76
+ if (! collection.matches("^system.*$")) {
77
+ database.getCollection(collection).drop();
78
+ }
79
+ }
80
+ } catch (UnknownHostException e) {
81
+ System.err.println("Could not connect to MongoDB, aborting");
82
+ }
83
+ }
84
+
85
+ @After
86
+ public void closeClient() throws InterruptedException {
87
+ client.close();
88
+ }
89
+
90
+ @Test
91
+ public void canGetSetHost() {
92
+ assertEquals(client.host(), HTTPClient.DEFAULT_HOST);
93
+ }
94
+
95
+ @Test
96
+ public void canGetSetPort() {
97
+ assertEquals(client.port(), HTTPClient.DEFAULT_PORT);
98
+ }
99
+
100
+ @Test
101
+ public void eventsEmpty() {
102
+ assertEquals(0, client.events(topic, query()).size());
103
+ }
104
+
105
+ @Test
106
+ public void announce() {
107
+ client.announce(topic, event());
108
+ client.announce(topic, event());
109
+ client.announce(topic, event());
110
+ assertEquals(3, client.events(topic, query()).size());
111
+ }
112
+
113
+ @Test
114
+ public void announceWithId() {
115
+ client.announce(topic, event(), "1");
116
+ client.announce(topic, event(), "2");
117
+ client.announce(topic, event(), "2");
118
+ assertEquals(2, client.events(topic, query()).size());
119
+ }
120
+
121
+ @Test
122
+ public void get() {
123
+ client.set(topic, stash());
124
+ Map s = client.get(topic);
125
+ assertEquals("hello", s.get("string"));
126
+ assertEquals(3.1415, s.get("double"));
127
+ }
128
+
129
+ @Test
130
+ public void getMap() {
131
+ client.set(topic, stash());
132
+ Map e = client.getMap(topic, "map");
133
+ assertEquals("bar", e.get("foo"));
134
+ }
135
+
136
+ @Test
137
+ public void getList() {
138
+ client.set(topic, stash());
139
+ List l = client.getList(topic, "list");
140
+ assertEquals("2", l.get(1));
141
+ assertEquals("three", l.get(2));
142
+ }
143
+
144
+ @Test
145
+ public void getString() {
146
+ client.set(topic, stash());
147
+ String s = client.getString(topic, "string");
148
+ assertEquals("hello", s);
149
+ }
150
+
151
+ @Test
152
+ public void getDouble() {
153
+ client.set(topic, stash());
154
+ Double d = client.getDouble(topic, "double");
155
+ assertEquals((Double) 3.1415, d);
156
+ }
157
+
158
+ @Test
159
+ public void setOverwrites() {
160
+ client.set(topic, stash());
161
+ client.set(topic, event());
162
+ Map s = client.get(topic);
163
+ assertEquals(null, s.get("string"));
164
+ assertEquals("bar", s.get("foo"));
165
+ }
166
+
167
+ @Test
168
+ public void setWithIdMapOverwrites() {
169
+ client.set(topic, stash());
170
+ client.set(topic, "map", stash());
171
+ Map s = client.getMap(topic, "map");
172
+ assertEquals(null, s.get("foo"));
173
+ assertEquals("hello", s.get("string"));
174
+ }
175
+
176
+ @Test
177
+ public void setWithIdListOvewrites() {
178
+ client.set(topic, stash());
179
+ List ol = list();
180
+ ol.set(0, "WOW");
181
+ client.set(topic, "list", ol);
182
+ List nl = client.getList(topic, "list");
183
+ assertEquals("WOW", nl.get(0));
184
+ assertEquals("2", nl.get(1));
185
+ }
186
+
187
+ @Test
188
+ public void setWithIdStringOvewrites() {
189
+ client.set(topic, stash());
190
+ client.set(topic, "string", "goodbye");
191
+ String s = client.getString(topic, "string");
192
+ assertEquals("goodbye", s);
193
+ }
194
+
195
+ @Test
196
+ public void setWithIdDoubleOvewrites() {
197
+ client.set(topic, stash());
198
+ client.set(topic, "double", 2.718);
199
+ Double d = client.getDouble(topic, "double");
200
+ assertEquals((double) 2.718, d, 0.001);
201
+ }
202
+
203
+ @Test
204
+ public void mergeMerges() {
205
+ Map s = stash();
206
+ client.set(topic, s);
207
+ s.put("newstring", "goodbye");
208
+ client.set(topic, s);
209
+
210
+ s = client.get(topic);
211
+ assertEquals("hello", s.get("string"));
212
+ assertEquals("goodbye", s.get("newstring"));
213
+ }
214
+
215
+ @Test
216
+ public void mergeWithIdMapMerges() throws InterruptedException {
217
+ client.set(topic, stash());
218
+ Map e = new HashMap();
219
+ e.put("bang", "boof");
220
+ client.merge(topic, "map", e);
221
+ Thread.sleep(100); // need to give the server time to process the merge
222
+ Map s = client.getMap(topic, "map");
223
+ assertEquals("bar", s.get("foo"));
224
+ assertEquals("boof", s.get("bang"));
225
+ }
226
+
227
+ public void mergeWithIdListConcatenates() throws InterruptedException {
228
+ client.set(topic, stash());
229
+ client.merge(topic, "list", list());
230
+ Thread.sleep(100); // need to give the server time to process the merge
231
+ List l = client.getList(topic, "list");
232
+ assertEquals(6, l.size());
233
+ }
234
+
235
+ public void mergeWithIdStringConcatenates() throws InterruptedException {
236
+ client.set(topic, stash());
237
+ client.merge(topic, "string", "goodbye");
238
+ Thread.sleep(100); // need to give the server time to process the merge
239
+ String s = client.getString(topic, "string");
240
+ assertEquals("hellogoodbye", s);
241
+ }
242
+
243
+ public void mergeWithIdDoubleIncrements() throws InterruptedException {
244
+ client.set(topic, stash());
245
+ client.merge(topic, "double", 3.1415);
246
+ Thread.sleep(100); // need to give the server time to process the merge
247
+ Double d = client.getDouble(topic, "double");
248
+ assertEquals((double) 6.2830, d, 0.001);
249
+ }
250
+
251
+ public void delete() throws InterruptedException {
252
+ client.set(topic, stash());
253
+ client.delete(topic);
254
+ Thread.sleep(100); // need to give the server time to process the delete
255
+ Map s = client.get(topic);
256
+ assertEquals(null, s);
257
+ }
258
+
259
+ public void deleteWithId() throws InterruptedException {
260
+ client.set(topic, stash());
261
+ client.delete(topic, "map");
262
+ Thread.sleep(100); // need to give the server time to process the delete
263
+ Map s = client.getMap(topic, "map");
264
+ assertEquals(null, s);
265
+ }
266
+
267
+ }
@@ -1,31 +1,31 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
 
3
3
  $:.push File.expand_path('../lib', __FILE__)
4
- require 'vayacondios/version'
4
+ require 'vayacondios'
5
5
 
6
6
  Gem::Specification.new do |gem|
7
7
  gem.name = 'vayacondios-server'
8
- gem.version = Vayacondios::VERSION
8
+ gem.version = Vayacondios::GEM_VERSION
9
9
  gem.authors = ['Philip (flip) Kromer', 'Travis Dempsey', 'Huston Hoburg', 'Logan Lowell']
10
10
  gem.homepage = 'https://github.com/infochimps-labs/vayacondios'
11
+ gem.email = 'coders@infochimps.com'
12
+ gem.licenses = ['Apache 2.0']
11
13
  gem.summary = 'Data goes in. The right thing happens'
12
14
  gem.description = "Simple enough to use in a shell script, performant enough to use everywhere. Dios mío! Record that metric, ese!"
13
15
 
14
- gem.files = `git ls-files`.split("\n")
15
- gem.executables = []
16
+ gem.files = `git ls-files`.split("\n").reject{ |f| f =~ /client/ }
17
+ gem.executables = ['vcd-server']
16
18
  gem.test_files = gem.files.grep(/^spec/)
17
19
  gem.require_paths = ['lib']
18
20
 
19
21
  gem.add_dependency('configliere', '>= 0.4.16')
20
22
  gem.add_dependency('gorillib', '>= 0.4.2')
21
23
  gem.add_dependency('multi_json', '>= 1.3.6')
24
+ gem.add_dependency('goliath-chimp', '>= 0.0.2')
22
25
 
23
- gem.add_dependency('eventmachine', '~> 1.0.0.beta.4')
26
+ gem.add_dependency('eventmachine', '~> 1.0')
24
27
  gem.add_dependency('goliath', '~> 1.0')
25
28
  gem.add_dependency('em-http-request', '~> 1.0')
26
29
  gem.add_dependency('em-mongo', '~> 0.4.3')
27
- gem.add_dependency('foreman')
28
-
29
- gem.add_development_dependency('rake')
30
- gem.add_development_dependency('mongo')
30
+ gem.add_dependency('bson_ext')
31
31
  end