vayacondios-server 0.2.11 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -1
- data/.travis.yml +2 -0
- data/Gemfile +15 -9
- data/LICENSE.md +2 -6
- data/Procfile +1 -1
- data/README.md +656 -111
- data/Rakefile +89 -6
- data/bin/vcd +10 -0
- data/bin/vcd-server +8 -0
- data/config/database.yml +6 -0
- data/config/spec.example.yml +18 -0
- data/config/vayacondios.example.yml +15 -0
- data/config/vcd-server.rb +37 -0
- data/examples/configuration.rb +56 -0
- data/examples/event_stream.rb +19 -0
- data/examples/simple.rb +61 -0
- data/features/event.feature +319 -0
- data/features/events.feature +208 -0
- data/features/stash.feature +840 -0
- data/features/stashes.feature +492 -0
- data/features/step_definitions/stash_steps.rb +113 -0
- data/features/stream.feature +30 -0
- data/features/support/em.rb +14 -0
- data/features/support/env.rb +13 -0
- data/lib/vayacondios/configuration.rb +63 -0
- data/lib/vayacondios/server/api.rb +126 -0
- data/lib/vayacondios/server/api_options.rb +56 -0
- data/lib/vayacondios/server/configuration.rb +23 -0
- data/lib/vayacondios/server/driver.rb +71 -0
- data/lib/vayacondios/server/drivers/mongo.rb +126 -0
- data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
- data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
- data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
- data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
- data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
- data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
- data/lib/vayacondios/server/models/document.rb +87 -0
- data/lib/vayacondios/server/models/event.rb +198 -0
- data/lib/vayacondios/server/models/stash.rb +100 -0
- data/lib/vayacondios/server.rb +35 -0
- data/lib/vayacondios-server.rb +19 -13
- data/lib/vayacondios.rb +22 -0
- data/pom.xml +124 -4
- data/spec/configuration_spec.rb +41 -0
- data/spec/server/api_options_spec.rb +32 -0
- data/spec/server/api_spec.rb +279 -0
- data/spec/server/configuration_spec.rb +27 -0
- data/spec/server/drivers/mongo_spec.rb +107 -0
- data/spec/server/handlers/event_handler_spec.rb +62 -0
- data/spec/server/handlers/events_handler_spec.rb +51 -0
- data/spec/server/handlers/stash_handler_spec.rb +68 -0
- data/spec/server/handlers/stashes_handler_spec.rb +50 -0
- data/spec/server/handlers/stream_handler_spec.rb +5 -0
- data/spec/server/models/document_spec.rb +9 -0
- data/spec/server/models/event_spec.rb +185 -0
- data/spec/server/models/stash_spec.rb +95 -0
- data/spec/spec_helper.rb +23 -3
- data/spec/support/database_helper.rb +42 -0
- data/spec/support/log_helper.rb +19 -0
- data/spec/support/shared_context_for_events.rb +22 -0
- data/spec/support/shared_context_for_stashes.rb +24 -0
- data/spec/support/shared_examples_for_handlers.rb +32 -0
- data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
- data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
- data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
- data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
- data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
- data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
- data/vayacondios-server.gemspec +9 -9
- metadata +127 -122
- checksums.yaml +0 -15
- data/.rspec +0 -2
- data/.yardopts +0 -10
- data/Guardfile +0 -41
- data/app/http_shim.rb +0 -71
- data/bin/vcd.sh +0 -27
- data/config/http_shim.rb +0 -43
- data/config/vayacondios.example.yaml +0 -7
- data/config/vayacondios.yaml +0 -7
- data/examples/java/ItemSetTest.java +0 -76
- data/lib/tasks/publish.rake +0 -23
- data/lib/tasks/spec.rake +0 -11
- data/lib/tasks/yard.rake +0 -2
- data/lib/vayacondios/client/config.rb +0 -7
- data/lib/vayacondios/client/configliere.rb +0 -38
- data/lib/vayacondios/client/cube_client.rb +0 -39
- data/lib/vayacondios/client/http_client.rb +0 -49
- data/lib/vayacondios/client/itemset.rb +0 -130
- data/lib/vayacondios/client/legacy_switch.rb +0 -43
- data/lib/vayacondios/client/notifier.rb +0 -123
- data/lib/vayacondios/client/zabbix_client.rb +0 -148
- data/lib/vayacondios/legacy_switch.rb +0 -43
- data/lib/vayacondios/server/errors/bad_request.rb +0 -6
- data/lib/vayacondios/server/errors/not_found.rb +0 -6
- data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
- data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
- data/lib/vayacondios/server/legacy_switch.rb +0 -43
- data/lib/vayacondios/server/model/config_document.rb +0 -89
- data/lib/vayacondios/server/model/document.rb +0 -25
- data/lib/vayacondios/server/model/event_document.rb +0 -94
- data/lib/vayacondios/server/model/itemset_document.rb +0 -126
- data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
- data/lib/vayacondios/server/rack/jsonize.rb +0 -43
- data/lib/vayacondios/server/rack/params.rb +0 -50
- data/lib/vayacondios/server/rack/path.rb +0 -23
- data/lib/vayacondios/server/rack/path_validation.rb +0 -22
- data/lib/vayacondios/version.rb +0 -3
- data/lib/vayacondios-client.rb +0 -22
- data/scripts/hadoop_monitor/configurable.rb +0 -66
- data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
- data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
- data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
- data/scripts/hadoop_monitor/hadoopable.rb +0 -65
- data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
- data/scripts/s3_cataloger/buckets +0 -33
- data/scripts/s3_cataloger/foreach_bucket +0 -88
- data/scripts/s3_cataloger/parse_ls.py +0 -391
- data/spec/client/itemset_legacy_spec.rb +0 -55
- data/spec/client/itemset_spec.rb +0 -60
- data/spec/client/notifier_spec.rb +0 -120
- data/spec/server/config_spec.rb +0 -113
- data/spec/server/event_spec.rb +0 -103
- data/spec/server/itemset_legacy_spec.rb +0 -320
- data/spec/server/itemset_spec.rb +0 -317
- data/spec/server/rack/extract_methods_spec.rb +0 -60
- data/spec/server/rack/path_spec.rb +0 -36
- data/spec/server/rack/path_validation_spec.rb +0 -22
- data/spec/server/server_spec.rb +0 -20
- data/spec/support/mongo_cleaner.rb +0 -32
- data/src/main/java/ItemSetTest.java +0 -76
- data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
- data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
- data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
- data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
- data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
- data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
- data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
- data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
- data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
- data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
- data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
- data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,267 @@
|
|
1
|
+
package com.infochimps.vayacondios;
|
2
|
+
|
3
|
+
import java.util.Map;
|
4
|
+
import java.util.HashMap;
|
5
|
+
import java.util.List;
|
6
|
+
import java.util.ArrayList;
|
7
|
+
import java.util.Set;
|
8
|
+
|
9
|
+
import java.net.UnknownHostException;
|
10
|
+
|
11
|
+
import org.junit.Before;
|
12
|
+
import org.junit.After;
|
13
|
+
import org.junit.Test;
|
14
|
+
import org.junit.Ignore;
|
15
|
+
import org.junit.runner.RunWith;
|
16
|
+
import org.junit.runners.JUnit4;
|
17
|
+
import static org.junit.Assert.assertEquals;
|
18
|
+
import com.infochimps.vayacondios.test.IntegrationTest;
|
19
|
+
import org.junit.experimental.categories.Category;
|
20
|
+
|
21
|
+
import com.mongodb.MongoClient;
|
22
|
+
import com.mongodb.DB;
|
23
|
+
|
24
|
+
@RunWith(JUnit4.class)
|
25
|
+
@Category(IntegrationTest.class)
|
26
|
+
public class HTTPClientIT {
|
27
|
+
|
28
|
+
private String organization = "organization";
|
29
|
+
private String topic = "topic";
|
30
|
+
private String id = "id";
|
31
|
+
|
32
|
+
private HTTPClient client;
|
33
|
+
|
34
|
+
private Map event() {
|
35
|
+
Map e = new HashMap();
|
36
|
+
e.put("foo", "bar");
|
37
|
+
e.put("baz", 12.0);
|
38
|
+
return e;
|
39
|
+
}
|
40
|
+
|
41
|
+
private List list() {
|
42
|
+
List l = new ArrayList();
|
43
|
+
l.add(1.0);
|
44
|
+
l.add("2");
|
45
|
+
l.add("three");
|
46
|
+
return l;
|
47
|
+
}
|
48
|
+
|
49
|
+
private Map stash() {
|
50
|
+
Map s = new HashMap();
|
51
|
+
s.put("map", event());
|
52
|
+
s.put("list", list());
|
53
|
+
s.put("string", "hello");
|
54
|
+
s.put("double", 3.1415);
|
55
|
+
return s;
|
56
|
+
}
|
57
|
+
|
58
|
+
private Map query() {
|
59
|
+
Map q = new HashMap();
|
60
|
+
q.put("foo", "bar");
|
61
|
+
return q;
|
62
|
+
}
|
63
|
+
|
64
|
+
@Before
|
65
|
+
public void createClient() {
|
66
|
+
client = new HTTPClient(organization);
|
67
|
+
}
|
68
|
+
|
69
|
+
@Before
|
70
|
+
public void cleanMongo() {
|
71
|
+
try {
|
72
|
+
MongoClient mongo = new MongoClient( "localhost" , 27017 );
|
73
|
+
DB database = mongo.getDB( "vayacondios_test" );
|
74
|
+
Set<String> collections = database.getCollectionNames();
|
75
|
+
for (String collection : collections) {
|
76
|
+
if (! collection.matches("^system.*$")) {
|
77
|
+
database.getCollection(collection).drop();
|
78
|
+
}
|
79
|
+
}
|
80
|
+
} catch (UnknownHostException e) {
|
81
|
+
System.err.println("Could not connect to MongoDB, aborting");
|
82
|
+
}
|
83
|
+
}
|
84
|
+
|
85
|
+
@After
|
86
|
+
public void closeClient() throws InterruptedException {
|
87
|
+
client.close();
|
88
|
+
}
|
89
|
+
|
90
|
+
@Test
|
91
|
+
public void canGetSetHost() {
|
92
|
+
assertEquals(client.host(), HTTPClient.DEFAULT_HOST);
|
93
|
+
}
|
94
|
+
|
95
|
+
@Test
|
96
|
+
public void canGetSetPort() {
|
97
|
+
assertEquals(client.port(), HTTPClient.DEFAULT_PORT);
|
98
|
+
}
|
99
|
+
|
100
|
+
@Test
|
101
|
+
public void eventsEmpty() {
|
102
|
+
assertEquals(0, client.events(topic, query()).size());
|
103
|
+
}
|
104
|
+
|
105
|
+
@Test
|
106
|
+
public void announce() {
|
107
|
+
client.announce(topic, event());
|
108
|
+
client.announce(topic, event());
|
109
|
+
client.announce(topic, event());
|
110
|
+
assertEquals(3, client.events(topic, query()).size());
|
111
|
+
}
|
112
|
+
|
113
|
+
@Test
|
114
|
+
public void announceWithId() {
|
115
|
+
client.announce(topic, event(), "1");
|
116
|
+
client.announce(topic, event(), "2");
|
117
|
+
client.announce(topic, event(), "2");
|
118
|
+
assertEquals(2, client.events(topic, query()).size());
|
119
|
+
}
|
120
|
+
|
121
|
+
@Test
|
122
|
+
public void get() {
|
123
|
+
client.set(topic, stash());
|
124
|
+
Map s = client.get(topic);
|
125
|
+
assertEquals("hello", s.get("string"));
|
126
|
+
assertEquals(3.1415, s.get("double"));
|
127
|
+
}
|
128
|
+
|
129
|
+
@Test
|
130
|
+
public void getMap() {
|
131
|
+
client.set(topic, stash());
|
132
|
+
Map e = client.getMap(topic, "map");
|
133
|
+
assertEquals("bar", e.get("foo"));
|
134
|
+
}
|
135
|
+
|
136
|
+
@Test
|
137
|
+
public void getList() {
|
138
|
+
client.set(topic, stash());
|
139
|
+
List l = client.getList(topic, "list");
|
140
|
+
assertEquals("2", l.get(1));
|
141
|
+
assertEquals("three", l.get(2));
|
142
|
+
}
|
143
|
+
|
144
|
+
@Test
|
145
|
+
public void getString() {
|
146
|
+
client.set(topic, stash());
|
147
|
+
String s = client.getString(topic, "string");
|
148
|
+
assertEquals("hello", s);
|
149
|
+
}
|
150
|
+
|
151
|
+
@Test
|
152
|
+
public void getDouble() {
|
153
|
+
client.set(topic, stash());
|
154
|
+
Double d = client.getDouble(topic, "double");
|
155
|
+
assertEquals((Double) 3.1415, d);
|
156
|
+
}
|
157
|
+
|
158
|
+
@Test
|
159
|
+
public void setOverwrites() {
|
160
|
+
client.set(topic, stash());
|
161
|
+
client.set(topic, event());
|
162
|
+
Map s = client.get(topic);
|
163
|
+
assertEquals(null, s.get("string"));
|
164
|
+
assertEquals("bar", s.get("foo"));
|
165
|
+
}
|
166
|
+
|
167
|
+
@Test
|
168
|
+
public void setWithIdMapOverwrites() {
|
169
|
+
client.set(topic, stash());
|
170
|
+
client.set(topic, "map", stash());
|
171
|
+
Map s = client.getMap(topic, "map");
|
172
|
+
assertEquals(null, s.get("foo"));
|
173
|
+
assertEquals("hello", s.get("string"));
|
174
|
+
}
|
175
|
+
|
176
|
+
@Test
|
177
|
+
public void setWithIdListOvewrites() {
|
178
|
+
client.set(topic, stash());
|
179
|
+
List ol = list();
|
180
|
+
ol.set(0, "WOW");
|
181
|
+
client.set(topic, "list", ol);
|
182
|
+
List nl = client.getList(topic, "list");
|
183
|
+
assertEquals("WOW", nl.get(0));
|
184
|
+
assertEquals("2", nl.get(1));
|
185
|
+
}
|
186
|
+
|
187
|
+
@Test
|
188
|
+
public void setWithIdStringOvewrites() {
|
189
|
+
client.set(topic, stash());
|
190
|
+
client.set(topic, "string", "goodbye");
|
191
|
+
String s = client.getString(topic, "string");
|
192
|
+
assertEquals("goodbye", s);
|
193
|
+
}
|
194
|
+
|
195
|
+
@Test
|
196
|
+
public void setWithIdDoubleOvewrites() {
|
197
|
+
client.set(topic, stash());
|
198
|
+
client.set(topic, "double", 2.718);
|
199
|
+
Double d = client.getDouble(topic, "double");
|
200
|
+
assertEquals((double) 2.718, d, 0.001);
|
201
|
+
}
|
202
|
+
|
203
|
+
@Test
|
204
|
+
public void mergeMerges() {
|
205
|
+
Map s = stash();
|
206
|
+
client.set(topic, s);
|
207
|
+
s.put("newstring", "goodbye");
|
208
|
+
client.set(topic, s);
|
209
|
+
|
210
|
+
s = client.get(topic);
|
211
|
+
assertEquals("hello", s.get("string"));
|
212
|
+
assertEquals("goodbye", s.get("newstring"));
|
213
|
+
}
|
214
|
+
|
215
|
+
@Test
|
216
|
+
public void mergeWithIdMapMerges() throws InterruptedException {
|
217
|
+
client.set(topic, stash());
|
218
|
+
Map e = new HashMap();
|
219
|
+
e.put("bang", "boof");
|
220
|
+
client.merge(topic, "map", e);
|
221
|
+
Thread.sleep(100); // need to give the server time to process the merge
|
222
|
+
Map s = client.getMap(topic, "map");
|
223
|
+
assertEquals("bar", s.get("foo"));
|
224
|
+
assertEquals("boof", s.get("bang"));
|
225
|
+
}
|
226
|
+
|
227
|
+
public void mergeWithIdListConcatenates() throws InterruptedException {
|
228
|
+
client.set(topic, stash());
|
229
|
+
client.merge(topic, "list", list());
|
230
|
+
Thread.sleep(100); // need to give the server time to process the merge
|
231
|
+
List l = client.getList(topic, "list");
|
232
|
+
assertEquals(6, l.size());
|
233
|
+
}
|
234
|
+
|
235
|
+
public void mergeWithIdStringConcatenates() throws InterruptedException {
|
236
|
+
client.set(topic, stash());
|
237
|
+
client.merge(topic, "string", "goodbye");
|
238
|
+
Thread.sleep(100); // need to give the server time to process the merge
|
239
|
+
String s = client.getString(topic, "string");
|
240
|
+
assertEquals("hellogoodbye", s);
|
241
|
+
}
|
242
|
+
|
243
|
+
public void mergeWithIdDoubleIncrements() throws InterruptedException {
|
244
|
+
client.set(topic, stash());
|
245
|
+
client.merge(topic, "double", 3.1415);
|
246
|
+
Thread.sleep(100); // need to give the server time to process the merge
|
247
|
+
Double d = client.getDouble(topic, "double");
|
248
|
+
assertEquals((double) 6.2830, d, 0.001);
|
249
|
+
}
|
250
|
+
|
251
|
+
public void delete() throws InterruptedException {
|
252
|
+
client.set(topic, stash());
|
253
|
+
client.delete(topic);
|
254
|
+
Thread.sleep(100); // need to give the server time to process the delete
|
255
|
+
Map s = client.get(topic);
|
256
|
+
assertEquals(null, s);
|
257
|
+
}
|
258
|
+
|
259
|
+
public void deleteWithId() throws InterruptedException {
|
260
|
+
client.set(topic, stash());
|
261
|
+
client.delete(topic, "map");
|
262
|
+
Thread.sleep(100); // need to give the server time to process the delete
|
263
|
+
Map s = client.getMap(topic, "map");
|
264
|
+
assertEquals(null, s);
|
265
|
+
}
|
266
|
+
|
267
|
+
}
|
data/vayacondios-server.gemspec
CHANGED
@@ -1,31 +1,31 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
|
3
3
|
$:.push File.expand_path('../lib', __FILE__)
|
4
|
-
require 'vayacondios
|
4
|
+
require 'vayacondios'
|
5
5
|
|
6
6
|
Gem::Specification.new do |gem|
|
7
7
|
gem.name = 'vayacondios-server'
|
8
|
-
gem.version = Vayacondios::
|
8
|
+
gem.version = Vayacondios::GEM_VERSION
|
9
9
|
gem.authors = ['Philip (flip) Kromer', 'Travis Dempsey', 'Huston Hoburg', 'Logan Lowell']
|
10
10
|
gem.homepage = 'https://github.com/infochimps-labs/vayacondios'
|
11
|
+
gem.email = 'coders@infochimps.com'
|
12
|
+
gem.licenses = ['Apache 2.0']
|
11
13
|
gem.summary = 'Data goes in. The right thing happens'
|
12
14
|
gem.description = "Simple enough to use in a shell script, performant enough to use everywhere. Dios mío! Record that metric, ese!"
|
13
15
|
|
14
|
-
gem.files = `git ls-files`.split("\n")
|
15
|
-
gem.executables = []
|
16
|
+
gem.files = `git ls-files`.split("\n").reject{ |f| f =~ /client/ }
|
17
|
+
gem.executables = ['vcd-server']
|
16
18
|
gem.test_files = gem.files.grep(/^spec/)
|
17
19
|
gem.require_paths = ['lib']
|
18
20
|
|
19
21
|
gem.add_dependency('configliere', '>= 0.4.16')
|
20
22
|
gem.add_dependency('gorillib', '>= 0.4.2')
|
21
23
|
gem.add_dependency('multi_json', '>= 1.3.6')
|
24
|
+
gem.add_dependency('goliath-chimp', '>= 0.0.2')
|
22
25
|
|
23
|
-
gem.add_dependency('eventmachine', '~> 1.0
|
26
|
+
gem.add_dependency('eventmachine', '~> 1.0')
|
24
27
|
gem.add_dependency('goliath', '~> 1.0')
|
25
28
|
gem.add_dependency('em-http-request', '~> 1.0')
|
26
29
|
gem.add_dependency('em-mongo', '~> 0.4.3')
|
27
|
-
gem.add_dependency('
|
28
|
-
|
29
|
-
gem.add_development_dependency('rake')
|
30
|
-
gem.add_development_dependency('mongo')
|
30
|
+
gem.add_dependency('bson_ext')
|
31
31
|
end
|