vayacondios-server 0.2.11 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. data/.gitignore +3 -1
  2. data/.travis.yml +2 -0
  3. data/Gemfile +15 -9
  4. data/LICENSE.md +2 -6
  5. data/Procfile +1 -1
  6. data/README.md +656 -111
  7. data/Rakefile +89 -6
  8. data/bin/vcd +10 -0
  9. data/bin/vcd-server +8 -0
  10. data/config/database.yml +6 -0
  11. data/config/spec.example.yml +18 -0
  12. data/config/vayacondios.example.yml +15 -0
  13. data/config/vcd-server.rb +37 -0
  14. data/examples/configuration.rb +56 -0
  15. data/examples/event_stream.rb +19 -0
  16. data/examples/simple.rb +61 -0
  17. data/features/event.feature +319 -0
  18. data/features/events.feature +208 -0
  19. data/features/stash.feature +840 -0
  20. data/features/stashes.feature +492 -0
  21. data/features/step_definitions/stash_steps.rb +113 -0
  22. data/features/stream.feature +30 -0
  23. data/features/support/em.rb +14 -0
  24. data/features/support/env.rb +13 -0
  25. data/lib/vayacondios/configuration.rb +63 -0
  26. data/lib/vayacondios/server/api.rb +126 -0
  27. data/lib/vayacondios/server/api_options.rb +56 -0
  28. data/lib/vayacondios/server/configuration.rb +23 -0
  29. data/lib/vayacondios/server/driver.rb +71 -0
  30. data/lib/vayacondios/server/drivers/mongo.rb +126 -0
  31. data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
  32. data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
  33. data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
  34. data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
  35. data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
  36. data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
  37. data/lib/vayacondios/server/models/document.rb +87 -0
  38. data/lib/vayacondios/server/models/event.rb +198 -0
  39. data/lib/vayacondios/server/models/stash.rb +100 -0
  40. data/lib/vayacondios/server.rb +35 -0
  41. data/lib/vayacondios-server.rb +19 -13
  42. data/lib/vayacondios.rb +22 -0
  43. data/pom.xml +124 -4
  44. data/spec/configuration_spec.rb +41 -0
  45. data/spec/server/api_options_spec.rb +32 -0
  46. data/spec/server/api_spec.rb +279 -0
  47. data/spec/server/configuration_spec.rb +27 -0
  48. data/spec/server/drivers/mongo_spec.rb +107 -0
  49. data/spec/server/handlers/event_handler_spec.rb +62 -0
  50. data/spec/server/handlers/events_handler_spec.rb +51 -0
  51. data/spec/server/handlers/stash_handler_spec.rb +68 -0
  52. data/spec/server/handlers/stashes_handler_spec.rb +50 -0
  53. data/spec/server/handlers/stream_handler_spec.rb +5 -0
  54. data/spec/server/models/document_spec.rb +9 -0
  55. data/spec/server/models/event_spec.rb +185 -0
  56. data/spec/server/models/stash_spec.rb +95 -0
  57. data/spec/spec_helper.rb +23 -3
  58. data/spec/support/database_helper.rb +42 -0
  59. data/spec/support/log_helper.rb +19 -0
  60. data/spec/support/shared_context_for_events.rb +22 -0
  61. data/spec/support/shared_context_for_stashes.rb +24 -0
  62. data/spec/support/shared_examples_for_handlers.rb +32 -0
  63. data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
  64. data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
  65. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
  66. data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
  67. data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
  68. data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
  69. data/vayacondios-server.gemspec +9 -9
  70. metadata +127 -122
  71. checksums.yaml +0 -15
  72. data/.rspec +0 -2
  73. data/.yardopts +0 -10
  74. data/Guardfile +0 -41
  75. data/app/http_shim.rb +0 -71
  76. data/bin/vcd.sh +0 -27
  77. data/config/http_shim.rb +0 -43
  78. data/config/vayacondios.example.yaml +0 -7
  79. data/config/vayacondios.yaml +0 -7
  80. data/examples/java/ItemSetTest.java +0 -76
  81. data/lib/tasks/publish.rake +0 -23
  82. data/lib/tasks/spec.rake +0 -11
  83. data/lib/tasks/yard.rake +0 -2
  84. data/lib/vayacondios/client/config.rb +0 -7
  85. data/lib/vayacondios/client/configliere.rb +0 -38
  86. data/lib/vayacondios/client/cube_client.rb +0 -39
  87. data/lib/vayacondios/client/http_client.rb +0 -49
  88. data/lib/vayacondios/client/itemset.rb +0 -130
  89. data/lib/vayacondios/client/legacy_switch.rb +0 -43
  90. data/lib/vayacondios/client/notifier.rb +0 -123
  91. data/lib/vayacondios/client/zabbix_client.rb +0 -148
  92. data/lib/vayacondios/legacy_switch.rb +0 -43
  93. data/lib/vayacondios/server/errors/bad_request.rb +0 -6
  94. data/lib/vayacondios/server/errors/not_found.rb +0 -6
  95. data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
  96. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
  97. data/lib/vayacondios/server/legacy_switch.rb +0 -43
  98. data/lib/vayacondios/server/model/config_document.rb +0 -89
  99. data/lib/vayacondios/server/model/document.rb +0 -25
  100. data/lib/vayacondios/server/model/event_document.rb +0 -94
  101. data/lib/vayacondios/server/model/itemset_document.rb +0 -126
  102. data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
  103. data/lib/vayacondios/server/rack/jsonize.rb +0 -43
  104. data/lib/vayacondios/server/rack/params.rb +0 -50
  105. data/lib/vayacondios/server/rack/path.rb +0 -23
  106. data/lib/vayacondios/server/rack/path_validation.rb +0 -22
  107. data/lib/vayacondios/version.rb +0 -3
  108. data/lib/vayacondios-client.rb +0 -22
  109. data/scripts/hadoop_monitor/configurable.rb +0 -66
  110. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
  111. data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
  112. data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
  113. data/scripts/hadoop_monitor/hadoopable.rb +0 -65
  114. data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
  115. data/scripts/s3_cataloger/buckets +0 -33
  116. data/scripts/s3_cataloger/foreach_bucket +0 -88
  117. data/scripts/s3_cataloger/parse_ls.py +0 -391
  118. data/spec/client/itemset_legacy_spec.rb +0 -55
  119. data/spec/client/itemset_spec.rb +0 -60
  120. data/spec/client/notifier_spec.rb +0 -120
  121. data/spec/server/config_spec.rb +0 -113
  122. data/spec/server/event_spec.rb +0 -103
  123. data/spec/server/itemset_legacy_spec.rb +0 -320
  124. data/spec/server/itemset_spec.rb +0 -317
  125. data/spec/server/rack/extract_methods_spec.rb +0 -60
  126. data/spec/server/rack/path_spec.rb +0 -36
  127. data/spec/server/rack/path_validation_spec.rb +0 -22
  128. data/spec/server/server_spec.rb +0 -20
  129. data/spec/support/mongo_cleaner.rb +0 -32
  130. data/src/main/java/ItemSetTest.java +0 -76
  131. data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
  132. data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
  133. data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
  134. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
  135. data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
  136. data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
  137. data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
  138. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
  139. data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
  140. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
  141. data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
  142. data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,267 @@
1
+ package com.infochimps.vayacondios;
2
+
3
+ import java.util.Map;
4
+ import java.util.HashMap;
5
+ import java.util.List;
6
+ import java.util.ArrayList;
7
+ import java.util.Set;
8
+
9
+ import java.net.UnknownHostException;
10
+
11
+ import org.junit.Before;
12
+ import org.junit.After;
13
+ import org.junit.Test;
14
+ import org.junit.Ignore;
15
+ import org.junit.runner.RunWith;
16
+ import org.junit.runners.JUnit4;
17
+ import static org.junit.Assert.assertEquals;
18
+ import com.infochimps.vayacondios.test.IntegrationTest;
19
+ import org.junit.experimental.categories.Category;
20
+
21
+ import com.mongodb.MongoClient;
22
+ import com.mongodb.DB;
23
+
24
+ @RunWith(JUnit4.class)
25
+ @Category(IntegrationTest.class)
26
+ public class HTTPClientIT {
27
+
28
+ private String organization = "organization";
29
+ private String topic = "topic";
30
+ private String id = "id";
31
+
32
+ private HTTPClient client;
33
+
34
+ private Map event() {
35
+ Map e = new HashMap();
36
+ e.put("foo", "bar");
37
+ e.put("baz", 12.0);
38
+ return e;
39
+ }
40
+
41
+ private List list() {
42
+ List l = new ArrayList();
43
+ l.add(1.0);
44
+ l.add("2");
45
+ l.add("three");
46
+ return l;
47
+ }
48
+
49
+ private Map stash() {
50
+ Map s = new HashMap();
51
+ s.put("map", event());
52
+ s.put("list", list());
53
+ s.put("string", "hello");
54
+ s.put("double", 3.1415);
55
+ return s;
56
+ }
57
+
58
+ private Map query() {
59
+ Map q = new HashMap();
60
+ q.put("foo", "bar");
61
+ return q;
62
+ }
63
+
64
+ @Before
65
+ public void createClient() {
66
+ client = new HTTPClient(organization);
67
+ }
68
+
69
+ @Before
70
+ public void cleanMongo() {
71
+ try {
72
+ MongoClient mongo = new MongoClient( "localhost" , 27017 );
73
+ DB database = mongo.getDB( "vayacondios_test" );
74
+ Set<String> collections = database.getCollectionNames();
75
+ for (String collection : collections) {
76
+ if (! collection.matches("^system.*$")) {
77
+ database.getCollection(collection).drop();
78
+ }
79
+ }
80
+ } catch (UnknownHostException e) {
81
+ System.err.println("Could not connect to MongoDB, aborting");
82
+ }
83
+ }
84
+
85
+ @After
86
+ public void closeClient() throws InterruptedException {
87
+ client.close();
88
+ }
89
+
90
+ @Test
91
+ public void canGetSetHost() {
92
+ assertEquals(client.host(), HTTPClient.DEFAULT_HOST);
93
+ }
94
+
95
+ @Test
96
+ public void canGetSetPort() {
97
+ assertEquals(client.port(), HTTPClient.DEFAULT_PORT);
98
+ }
99
+
100
+ @Test
101
+ public void eventsEmpty() {
102
+ assertEquals(0, client.events(topic, query()).size());
103
+ }
104
+
105
+ @Test
106
+ public void announce() {
107
+ client.announce(topic, event());
108
+ client.announce(topic, event());
109
+ client.announce(topic, event());
110
+ assertEquals(3, client.events(topic, query()).size());
111
+ }
112
+
113
+ @Test
114
+ public void announceWithId() {
115
+ client.announce(topic, event(), "1");
116
+ client.announce(topic, event(), "2");
117
+ client.announce(topic, event(), "2");
118
+ assertEquals(2, client.events(topic, query()).size());
119
+ }
120
+
121
+ @Test
122
+ public void get() {
123
+ client.set(topic, stash());
124
+ Map s = client.get(topic);
125
+ assertEquals("hello", s.get("string"));
126
+ assertEquals(3.1415, s.get("double"));
127
+ }
128
+
129
+ @Test
130
+ public void getMap() {
131
+ client.set(topic, stash());
132
+ Map e = client.getMap(topic, "map");
133
+ assertEquals("bar", e.get("foo"));
134
+ }
135
+
136
+ @Test
137
+ public void getList() {
138
+ client.set(topic, stash());
139
+ List l = client.getList(topic, "list");
140
+ assertEquals("2", l.get(1));
141
+ assertEquals("three", l.get(2));
142
+ }
143
+
144
+ @Test
145
+ public void getString() {
146
+ client.set(topic, stash());
147
+ String s = client.getString(topic, "string");
148
+ assertEquals("hello", s);
149
+ }
150
+
151
+ @Test
152
+ public void getDouble() {
153
+ client.set(topic, stash());
154
+ Double d = client.getDouble(topic, "double");
155
+ assertEquals((Double) 3.1415, d);
156
+ }
157
+
158
+ @Test
159
+ public void setOverwrites() {
160
+ client.set(topic, stash());
161
+ client.set(topic, event());
162
+ Map s = client.get(topic);
163
+ assertEquals(null, s.get("string"));
164
+ assertEquals("bar", s.get("foo"));
165
+ }
166
+
167
+ @Test
168
+ public void setWithIdMapOverwrites() {
169
+ client.set(topic, stash());
170
+ client.set(topic, "map", stash());
171
+ Map s = client.getMap(topic, "map");
172
+ assertEquals(null, s.get("foo"));
173
+ assertEquals("hello", s.get("string"));
174
+ }
175
+
176
+ @Test
177
+ public void setWithIdListOvewrites() {
178
+ client.set(topic, stash());
179
+ List ol = list();
180
+ ol.set(0, "WOW");
181
+ client.set(topic, "list", ol);
182
+ List nl = client.getList(topic, "list");
183
+ assertEquals("WOW", nl.get(0));
184
+ assertEquals("2", nl.get(1));
185
+ }
186
+
187
+ @Test
188
+ public void setWithIdStringOvewrites() {
189
+ client.set(topic, stash());
190
+ client.set(topic, "string", "goodbye");
191
+ String s = client.getString(topic, "string");
192
+ assertEquals("goodbye", s);
193
+ }
194
+
195
+ @Test
196
+ public void setWithIdDoubleOvewrites() {
197
+ client.set(topic, stash());
198
+ client.set(topic, "double", 2.718);
199
+ Double d = client.getDouble(topic, "double");
200
+ assertEquals((double) 2.718, d, 0.001);
201
+ }
202
+
203
+ @Test
204
+ public void mergeMerges() {
205
+ Map s = stash();
206
+ client.set(topic, s);
207
+ s.put("newstring", "goodbye");
208
+ client.set(topic, s);
209
+
210
+ s = client.get(topic);
211
+ assertEquals("hello", s.get("string"));
212
+ assertEquals("goodbye", s.get("newstring"));
213
+ }
214
+
215
+ @Test
216
+ public void mergeWithIdMapMerges() throws InterruptedException {
217
+ client.set(topic, stash());
218
+ Map e = new HashMap();
219
+ e.put("bang", "boof");
220
+ client.merge(topic, "map", e);
221
+ Thread.sleep(100); // need to give the server time to process the merge
222
+ Map s = client.getMap(topic, "map");
223
+ assertEquals("bar", s.get("foo"));
224
+ assertEquals("boof", s.get("bang"));
225
+ }
226
+
227
+ public void mergeWithIdListConcatenates() throws InterruptedException {
228
+ client.set(topic, stash());
229
+ client.merge(topic, "list", list());
230
+ Thread.sleep(100); // need to give the server time to process the merge
231
+ List l = client.getList(topic, "list");
232
+ assertEquals(6, l.size());
233
+ }
234
+
235
+ public void mergeWithIdStringConcatenates() throws InterruptedException {
236
+ client.set(topic, stash());
237
+ client.merge(topic, "string", "goodbye");
238
+ Thread.sleep(100); // need to give the server time to process the merge
239
+ String s = client.getString(topic, "string");
240
+ assertEquals("hellogoodbye", s);
241
+ }
242
+
243
+ public void mergeWithIdDoubleIncrements() throws InterruptedException {
244
+ client.set(topic, stash());
245
+ client.merge(topic, "double", 3.1415);
246
+ Thread.sleep(100); // need to give the server time to process the merge
247
+ Double d = client.getDouble(topic, "double");
248
+ assertEquals((double) 6.2830, d, 0.001);
249
+ }
250
+
251
+ public void delete() throws InterruptedException {
252
+ client.set(topic, stash());
253
+ client.delete(topic);
254
+ Thread.sleep(100); // need to give the server time to process the delete
255
+ Map s = client.get(topic);
256
+ assertEquals(null, s);
257
+ }
258
+
259
+ public void deleteWithId() throws InterruptedException {
260
+ client.set(topic, stash());
261
+ client.delete(topic, "map");
262
+ Thread.sleep(100); // need to give the server time to process the delete
263
+ Map s = client.getMap(topic, "map");
264
+ assertEquals(null, s);
265
+ }
266
+
267
+ }
@@ -1,31 +1,31 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
 
3
3
  $:.push File.expand_path('../lib', __FILE__)
4
- require 'vayacondios/version'
4
+ require 'vayacondios'
5
5
 
6
6
  Gem::Specification.new do |gem|
7
7
  gem.name = 'vayacondios-server'
8
- gem.version = Vayacondios::VERSION
8
+ gem.version = Vayacondios::GEM_VERSION
9
9
  gem.authors = ['Philip (flip) Kromer', 'Travis Dempsey', 'Huston Hoburg', 'Logan Lowell']
10
10
  gem.homepage = 'https://github.com/infochimps-labs/vayacondios'
11
+ gem.email = 'coders@infochimps.com'
12
+ gem.licenses = ['Apache 2.0']
11
13
  gem.summary = 'Data goes in. The right thing happens'
12
14
  gem.description = "Simple enough to use in a shell script, performant enough to use everywhere. Dios mío! Record that metric, ese!"
13
15
 
14
- gem.files = `git ls-files`.split("\n")
15
- gem.executables = []
16
+ gem.files = `git ls-files`.split("\n").reject{ |f| f =~ /client/ }
17
+ gem.executables = ['vcd-server']
16
18
  gem.test_files = gem.files.grep(/^spec/)
17
19
  gem.require_paths = ['lib']
18
20
 
19
21
  gem.add_dependency('configliere', '>= 0.4.16')
20
22
  gem.add_dependency('gorillib', '>= 0.4.2')
21
23
  gem.add_dependency('multi_json', '>= 1.3.6')
24
+ gem.add_dependency('goliath-chimp', '>= 0.0.2')
22
25
 
23
- gem.add_dependency('eventmachine', '~> 1.0.0.beta.4')
26
+ gem.add_dependency('eventmachine', '~> 1.0')
24
27
  gem.add_dependency('goliath', '~> 1.0')
25
28
  gem.add_dependency('em-http-request', '~> 1.0')
26
29
  gem.add_dependency('em-mongo', '~> 0.4.3')
27
- gem.add_dependency('foreman')
28
-
29
- gem.add_development_dependency('rake')
30
- gem.add_development_dependency('mongo')
30
+ gem.add_dependency('bson_ext')
31
31
  end