vayacondios-server 0.2.11 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -1
- data/.travis.yml +2 -0
- data/Gemfile +15 -9
- data/LICENSE.md +2 -6
- data/Procfile +1 -1
- data/README.md +656 -111
- data/Rakefile +89 -6
- data/bin/vcd +10 -0
- data/bin/vcd-server +8 -0
- data/config/database.yml +6 -0
- data/config/spec.example.yml +18 -0
- data/config/vayacondios.example.yml +15 -0
- data/config/vcd-server.rb +37 -0
- data/examples/configuration.rb +56 -0
- data/examples/event_stream.rb +19 -0
- data/examples/simple.rb +61 -0
- data/features/event.feature +319 -0
- data/features/events.feature +208 -0
- data/features/stash.feature +840 -0
- data/features/stashes.feature +492 -0
- data/features/step_definitions/stash_steps.rb +113 -0
- data/features/stream.feature +30 -0
- data/features/support/em.rb +14 -0
- data/features/support/env.rb +13 -0
- data/lib/vayacondios/configuration.rb +63 -0
- data/lib/vayacondios/server/api.rb +126 -0
- data/lib/vayacondios/server/api_options.rb +56 -0
- data/lib/vayacondios/server/configuration.rb +23 -0
- data/lib/vayacondios/server/driver.rb +71 -0
- data/lib/vayacondios/server/drivers/mongo.rb +126 -0
- data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
- data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
- data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
- data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
- data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
- data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
- data/lib/vayacondios/server/models/document.rb +87 -0
- data/lib/vayacondios/server/models/event.rb +198 -0
- data/lib/vayacondios/server/models/stash.rb +100 -0
- data/lib/vayacondios/server.rb +35 -0
- data/lib/vayacondios-server.rb +19 -13
- data/lib/vayacondios.rb +22 -0
- data/pom.xml +124 -4
- data/spec/configuration_spec.rb +41 -0
- data/spec/server/api_options_spec.rb +32 -0
- data/spec/server/api_spec.rb +279 -0
- data/spec/server/configuration_spec.rb +27 -0
- data/spec/server/drivers/mongo_spec.rb +107 -0
- data/spec/server/handlers/event_handler_spec.rb +62 -0
- data/spec/server/handlers/events_handler_spec.rb +51 -0
- data/spec/server/handlers/stash_handler_spec.rb +68 -0
- data/spec/server/handlers/stashes_handler_spec.rb +50 -0
- data/spec/server/handlers/stream_handler_spec.rb +5 -0
- data/spec/server/models/document_spec.rb +9 -0
- data/spec/server/models/event_spec.rb +185 -0
- data/spec/server/models/stash_spec.rb +95 -0
- data/spec/spec_helper.rb +23 -3
- data/spec/support/database_helper.rb +42 -0
- data/spec/support/log_helper.rb +19 -0
- data/spec/support/shared_context_for_events.rb +22 -0
- data/spec/support/shared_context_for_stashes.rb +24 -0
- data/spec/support/shared_examples_for_handlers.rb +32 -0
- data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
- data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
- data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
- data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
- data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
- data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
- data/vayacondios-server.gemspec +9 -9
- metadata +127 -122
- checksums.yaml +0 -15
- data/.rspec +0 -2
- data/.yardopts +0 -10
- data/Guardfile +0 -41
- data/app/http_shim.rb +0 -71
- data/bin/vcd.sh +0 -27
- data/config/http_shim.rb +0 -43
- data/config/vayacondios.example.yaml +0 -7
- data/config/vayacondios.yaml +0 -7
- data/examples/java/ItemSetTest.java +0 -76
- data/lib/tasks/publish.rake +0 -23
- data/lib/tasks/spec.rake +0 -11
- data/lib/tasks/yard.rake +0 -2
- data/lib/vayacondios/client/config.rb +0 -7
- data/lib/vayacondios/client/configliere.rb +0 -38
- data/lib/vayacondios/client/cube_client.rb +0 -39
- data/lib/vayacondios/client/http_client.rb +0 -49
- data/lib/vayacondios/client/itemset.rb +0 -130
- data/lib/vayacondios/client/legacy_switch.rb +0 -43
- data/lib/vayacondios/client/notifier.rb +0 -123
- data/lib/vayacondios/client/zabbix_client.rb +0 -148
- data/lib/vayacondios/legacy_switch.rb +0 -43
- data/lib/vayacondios/server/errors/bad_request.rb +0 -6
- data/lib/vayacondios/server/errors/not_found.rb +0 -6
- data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
- data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
- data/lib/vayacondios/server/legacy_switch.rb +0 -43
- data/lib/vayacondios/server/model/config_document.rb +0 -89
- data/lib/vayacondios/server/model/document.rb +0 -25
- data/lib/vayacondios/server/model/event_document.rb +0 -94
- data/lib/vayacondios/server/model/itemset_document.rb +0 -126
- data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
- data/lib/vayacondios/server/rack/jsonize.rb +0 -43
- data/lib/vayacondios/server/rack/params.rb +0 -50
- data/lib/vayacondios/server/rack/path.rb +0 -23
- data/lib/vayacondios/server/rack/path_validation.rb +0 -22
- data/lib/vayacondios/version.rb +0 -3
- data/lib/vayacondios-client.rb +0 -22
- data/scripts/hadoop_monitor/configurable.rb +0 -66
- data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
- data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
- data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
- data/scripts/hadoop_monitor/hadoopable.rb +0 -65
- data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
- data/scripts/s3_cataloger/buckets +0 -33
- data/scripts/s3_cataloger/foreach_bucket +0 -88
- data/scripts/s3_cataloger/parse_ls.py +0 -391
- data/spec/client/itemset_legacy_spec.rb +0 -55
- data/spec/client/itemset_spec.rb +0 -60
- data/spec/client/notifier_spec.rb +0 -120
- data/spec/server/config_spec.rb +0 -113
- data/spec/server/event_spec.rb +0 -103
- data/spec/server/itemset_legacy_spec.rb +0 -320
- data/spec/server/itemset_spec.rb +0 -317
- data/spec/server/rack/extract_methods_spec.rb +0 -60
- data/spec/server/rack/path_spec.rb +0 -36
- data/spec/server/rack/path_validation_spec.rb +0 -22
- data/spec/server/server_spec.rb +0 -20
- data/spec/support/mongo_cleaner.rb +0 -32
- data/src/main/java/ItemSetTest.java +0 -76
- data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
- data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
- data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
- data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
- data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
- data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
- data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
- data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
- data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
- data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
- data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
- data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,342 @@
|
|
1
|
+
package com.infochimps.vayacondios;
|
2
|
+
|
3
|
+
import java.util.Map;
|
4
|
+
import java.util.List;
|
5
|
+
import java.io.IOException;
|
6
|
+
|
7
|
+
import org.slf4j.Logger;
|
8
|
+
import org.slf4j.LoggerFactory;
|
9
|
+
|
10
|
+
/** Base class for all concrete implementations of a Vayacondios client.
|
11
|
+
* <p>
|
12
|
+
* Is fixed to a specific organization at instantiation time since the
|
13
|
+
* common use case is reading/writing data for a single organization
|
14
|
+
* at a time. If reading/writing data for multiple organizations
|
15
|
+
* simultaneously, use multiple client instances.
|
16
|
+
* <p>
|
17
|
+
* Can be instantiated in "dry-run" mode in which it will not actually
|
18
|
+
* make any requests to the server, merely log its output.
|
19
|
+
* <p>
|
20
|
+
* Implements the {@link VayacondiosClient} interface by wrapping each
|
21
|
+
* method of is API (e.g. - {@link VayacondiosClient#announce(String
|
22
|
+
* topic, Map event)}) with logic for handling arguments, errors, and
|
23
|
+
* "dry-run" mode, ultimately delegating to a (protected) method to
|
24
|
+
* perform the actual request (e.g. - {@link
|
25
|
+
* BaseClient#performAnnounce(String topic, Map event)}). Subclasses
|
26
|
+
* should override these methods to provide a concrete implementation
|
27
|
+
* of a Vayacondios client.
|
28
|
+
*/
|
29
|
+
public class BaseClient implements VayacondiosClient {
|
30
|
+
|
31
|
+
//----------------------------------------------------------------------------
|
32
|
+
// Initialization & Properties
|
33
|
+
//----------------------------------------------------------------------------
|
34
|
+
|
35
|
+
public static String VERSION = "v2";
|
36
|
+
|
37
|
+
private static Logger LOG = LoggerFactory.getLogger(BaseClient.class);
|
38
|
+
|
39
|
+
private String _organization;
|
40
|
+
private Boolean _dryRun;
|
41
|
+
|
42
|
+
/**
|
43
|
+
* Create a new BaseClient instance for the given
|
44
|
+
* organization. optionally in "dry-run" mode.
|
45
|
+
*
|
46
|
+
* <blockquote><pre>{@code
|
47
|
+
* VayacondiosClient client = new BaseClient("my_organization", true); // client in dry-run mode
|
48
|
+
* }</pre></blockquote>
|
49
|
+
*
|
50
|
+
* @param organization name of the organization to read/write data for
|
51
|
+
* @param shouldDryRun whether or not to enter "dry-run" mode
|
52
|
+
*/
|
53
|
+
public BaseClient(String organization, Boolean shouldDryRun) {
|
54
|
+
this._organization = organization;
|
55
|
+
this._dryRun = shouldDryRun;
|
56
|
+
}
|
57
|
+
|
58
|
+
/**
|
59
|
+
* Create a new BaseClient instance for the given organization.
|
60
|
+
*
|
61
|
+
* @param organization name of the organization to read/write data for
|
62
|
+
*/
|
63
|
+
public BaseClient(String organization) {
|
64
|
+
this(organization, false);
|
65
|
+
}
|
66
|
+
|
67
|
+
/**
|
68
|
+
* Is this client in "dry-run" mode?
|
69
|
+
* <p>
|
70
|
+
* When in "dry-run" mode, the client will not actually make any
|
71
|
+
* requests to the server, it will merely log all its attempts at
|
72
|
+
* an elevated level.
|
73
|
+
* @return whether or not the client is in "dry-run" mode
|
74
|
+
*/
|
75
|
+
public Boolean dryRun() {
|
76
|
+
return _dryRun;
|
77
|
+
}
|
78
|
+
|
79
|
+
/**
|
80
|
+
* The organization this client will read/write data for.
|
81
|
+
*
|
82
|
+
* @return name of the organization
|
83
|
+
*/
|
84
|
+
public String organization() {
|
85
|
+
return _organization;
|
86
|
+
}
|
87
|
+
|
88
|
+
//----------------------------------------------------------------------------
|
89
|
+
// Public API
|
90
|
+
//----------------------------------------------------------------------------
|
91
|
+
|
92
|
+
/**
|
93
|
+
* {@inheritDoc}
|
94
|
+
*/
|
95
|
+
@Override
|
96
|
+
public void announce(String topic, Map<String,Object> event) {
|
97
|
+
logRequest("Announcing <" + topic + ">");
|
98
|
+
if (dryRun()) return;
|
99
|
+
try {
|
100
|
+
performAnnounce(topic, event);
|
101
|
+
} catch (IOException e) {
|
102
|
+
LOG.error("Announcing <" + topic + ">", e);
|
103
|
+
}
|
104
|
+
}
|
105
|
+
/**
|
106
|
+
* {@inheritDoc}
|
107
|
+
*/
|
108
|
+
@Override
|
109
|
+
public void announce(String topic, Map<String,Object> event, String id) {
|
110
|
+
logRequest("Announcing <" + topic + "/" + id + ">");
|
111
|
+
if (dryRun()) return;
|
112
|
+
try {
|
113
|
+
performAnnounce(topic, event, id);
|
114
|
+
} catch (IOException e) {
|
115
|
+
LOG.error("Announcing <" + topic + "/" + id + ">", e);
|
116
|
+
}
|
117
|
+
}
|
118
|
+
|
119
|
+
/**
|
120
|
+
* {@inheritDoc}
|
121
|
+
*/
|
122
|
+
@Override
|
123
|
+
public List<Map<String,Object>> events(String topic, Map<String,Object> query) {
|
124
|
+
logRequest("Searching events <" + topic + ">");
|
125
|
+
if (dryRun()) return null;
|
126
|
+
try {
|
127
|
+
return performEvents(topic, query);
|
128
|
+
} catch (IOException e) {
|
129
|
+
LOG.error("Searching events <" + topic + ">", e);
|
130
|
+
return null;
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
134
|
+
/**
|
135
|
+
* {@inheritDoc}
|
136
|
+
*/
|
137
|
+
@Override
|
138
|
+
public Map<String,Object> get(String topic) {
|
139
|
+
logRequest("Fetching <" + topic + ">");
|
140
|
+
if (dryRun()) return null;
|
141
|
+
try {
|
142
|
+
return performGet(topic);
|
143
|
+
} catch (IOException e) {
|
144
|
+
LOG.error("Fetching <" + topic + ">");
|
145
|
+
return null;
|
146
|
+
}
|
147
|
+
}
|
148
|
+
|
149
|
+
/**
|
150
|
+
* {@inheritDoc}
|
151
|
+
*/
|
152
|
+
@Override
|
153
|
+
public Map<String,Object> getMap(String topic, String id) {
|
154
|
+
logRequest("Fetching Map <" + topic + "/" + id + ">");
|
155
|
+
if (dryRun()) return null;
|
156
|
+
try {
|
157
|
+
return performGetMap(topic, id);
|
158
|
+
} catch (IOException e) {
|
159
|
+
LOG.error("Fetching Map <" + topic + "/" + id + ">", e);
|
160
|
+
return null;
|
161
|
+
}
|
162
|
+
}
|
163
|
+
/**
|
164
|
+
* {@inheritDoc}
|
165
|
+
*/
|
166
|
+
@Override
|
167
|
+
public List getList(String topic, String id) {
|
168
|
+
logRequest("Fetching List <" + topic + "/" + id + ">");
|
169
|
+
if (dryRun()) return null;
|
170
|
+
try {
|
171
|
+
return performGetList(topic, id);
|
172
|
+
} catch (IOException e) {
|
173
|
+
LOG.error("Fetching List <" + topic + "/" + id + ">", e);
|
174
|
+
return null;
|
175
|
+
}
|
176
|
+
}
|
177
|
+
/**
|
178
|
+
* {@inheritDoc}
|
179
|
+
*/
|
180
|
+
@Override
|
181
|
+
public String getString(String topic, String id) {
|
182
|
+
logRequest("Fetching String <" + topic + "/" + id + ">");
|
183
|
+
if (dryRun()) return null;
|
184
|
+
try {
|
185
|
+
return performGetString(topic, id);
|
186
|
+
} catch (IOException e) {
|
187
|
+
LOG.error("Fetching String <" + topic + "/" + id + ">", e);
|
188
|
+
return null;
|
189
|
+
}
|
190
|
+
}
|
191
|
+
/**
|
192
|
+
* {@inheritDoc}
|
193
|
+
*/
|
194
|
+
@Override
|
195
|
+
public Double getDouble(String topic, String id) {
|
196
|
+
logRequest("Fetching Double <" + topic + "/" + id + ">");
|
197
|
+
if (dryRun()) return null;
|
198
|
+
try {
|
199
|
+
return performGetDouble(topic, id);
|
200
|
+
} catch (IOException e) {
|
201
|
+
LOG.error("Fetching Double <" + topic + "/" + id + ">", e);
|
202
|
+
return null;
|
203
|
+
}
|
204
|
+
}
|
205
|
+
|
206
|
+
/**
|
207
|
+
* {@inheritDoc}
|
208
|
+
*/
|
209
|
+
@Override
|
210
|
+
public List<Map<String,Object>> stashes(Map<String,Object> query) {
|
211
|
+
logRequest("Searching stashes");
|
212
|
+
if (dryRun()) return null;
|
213
|
+
try {
|
214
|
+
return performStashes(query);
|
215
|
+
} catch (IOException e) {
|
216
|
+
LOG.error("Searching stashes", e);
|
217
|
+
return null;
|
218
|
+
}
|
219
|
+
}
|
220
|
+
|
221
|
+
/**
|
222
|
+
* {@inheritDoc}
|
223
|
+
*/
|
224
|
+
@Override
|
225
|
+
public void set(String topic, Map<String,Object> value) {
|
226
|
+
logRequest("Replacing <" + topic + ">");
|
227
|
+
if (dryRun()) return;
|
228
|
+
try {
|
229
|
+
performSet(topic, value);
|
230
|
+
} catch (IOException e) {
|
231
|
+
LOG.error("Replacing <" + topic + ">", e);
|
232
|
+
}
|
233
|
+
}
|
234
|
+
/**
|
235
|
+
* {@inheritDoc}
|
236
|
+
*/
|
237
|
+
@Override
|
238
|
+
public void set(String topic, String id, Object value) {
|
239
|
+
logRequest("Replacing <" + topic + "/" + id + ">");
|
240
|
+
if (dryRun()) return;
|
241
|
+
try {
|
242
|
+
performSet(topic, id, value);
|
243
|
+
} catch (IOException e) {
|
244
|
+
LOG.error("Replacing <" + topic + "/" + id + ">", e);
|
245
|
+
}
|
246
|
+
}
|
247
|
+
|
248
|
+
/**
|
249
|
+
* {@inheritDoc}
|
250
|
+
*/
|
251
|
+
@Override
|
252
|
+
public void merge(String topic, Map<String,Object> value) {
|
253
|
+
logRequest("Merging <" + topic + ">");
|
254
|
+
if (dryRun()) return;
|
255
|
+
try {
|
256
|
+
performMerge(topic, value);
|
257
|
+
} catch (IOException e) {
|
258
|
+
LOG.error("Merging <" + topic + ">", e);
|
259
|
+
}
|
260
|
+
}
|
261
|
+
/**
|
262
|
+
* {@inheritDoc}
|
263
|
+
*/
|
264
|
+
@Override
|
265
|
+
public void merge(String topic, String id, Object value) {
|
266
|
+
logRequest("Merging <" + topic + "/" + id + ">");
|
267
|
+
if (dryRun()) return;
|
268
|
+
try {
|
269
|
+
performMerge(topic, id, value);
|
270
|
+
} catch (IOException e) {
|
271
|
+
LOG.error("Merging <" + topic + "/" + id + ">", e);
|
272
|
+
}
|
273
|
+
}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* {@inheritDoc}
|
277
|
+
*/
|
278
|
+
@Override
|
279
|
+
public void delete(String topic) {
|
280
|
+
logRequest("Deleting <" + topic + ">");
|
281
|
+
if (dryRun()) return;
|
282
|
+
try {
|
283
|
+
performDelete(topic);
|
284
|
+
} catch (IOException e) {
|
285
|
+
LOG.error("Deleting <" + topic + ">");
|
286
|
+
}
|
287
|
+
}
|
288
|
+
/**
|
289
|
+
* {@inheritDoc}
|
290
|
+
*/
|
291
|
+
@Override
|
292
|
+
public void delete(String topic, String id) {
|
293
|
+
logRequest("Deleting <" + topic + "/" + id + ">");
|
294
|
+
if (dryRun()) return;
|
295
|
+
try {
|
296
|
+
performDelete(topic, id);
|
297
|
+
} catch (IOException e) {
|
298
|
+
LOG.error("Deleting <" + topic + "/" + id + ">");
|
299
|
+
}
|
300
|
+
}
|
301
|
+
|
302
|
+
/**
|
303
|
+
* {@inheritDoc}
|
304
|
+
*/
|
305
|
+
@Override
|
306
|
+
public void close() {};
|
307
|
+
|
308
|
+
//----------------------------------------------------------------------------
|
309
|
+
// Private API
|
310
|
+
//----------------------------------------------------------------------------
|
311
|
+
|
312
|
+
protected void performAnnounce(String topic, Map<String,Object> event, String id) throws IOException {}
|
313
|
+
protected void performAnnounce(String topic, Map<String,Object> event) throws IOException {}
|
314
|
+
|
315
|
+
protected List<Map<String,Object>> performEvents(String topic, Map<String,Object> query) throws IOException { return null; }
|
316
|
+
|
317
|
+
protected Map<String,Object> performGet(String topic) throws IOException { return null; }
|
318
|
+
|
319
|
+
protected Map<String,Object> performGetMap(String topic, String id) throws IOException { return null; }
|
320
|
+
protected List performGetList(String topic, String id) throws IOException { return null; }
|
321
|
+
protected String performGetString(String topic, String id) throws IOException { return null; }
|
322
|
+
protected Double performGetDouble(String topic, String id) throws IOException { return null; }
|
323
|
+
|
324
|
+
protected List<Map<String,Object>> performStashes(Map<String,Object> query) throws IOException { return null; }
|
325
|
+
|
326
|
+
protected void performMerge(String topic, String id, Object value) throws IOException {}
|
327
|
+
protected void performMerge(String topic, Map<String,Object> value) throws IOException {}
|
328
|
+
|
329
|
+
protected void performSet(String topic, String id, Object value) throws IOException {}
|
330
|
+
protected void performSet(String topic, Map<String,Object> value) throws IOException {}
|
331
|
+
|
332
|
+
protected void performDelete(String topic, String id) throws IOException {}
|
333
|
+
protected void performDelete(String topic) throws IOException {}
|
334
|
+
|
335
|
+
private void logRequest(String message) {
|
336
|
+
if (dryRun()) {
|
337
|
+
LOG.info(message);
|
338
|
+
} else {
|
339
|
+
LOG.debug(message);
|
340
|
+
}
|
341
|
+
}
|
342
|
+
}
|