vayacondios-server 0.2.11 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -1
- data/.travis.yml +2 -0
- data/Gemfile +15 -9
- data/LICENSE.md +2 -6
- data/Procfile +1 -1
- data/README.md +656 -111
- data/Rakefile +89 -6
- data/bin/vcd +10 -0
- data/bin/vcd-server +8 -0
- data/config/database.yml +6 -0
- data/config/spec.example.yml +18 -0
- data/config/vayacondios.example.yml +15 -0
- data/config/vcd-server.rb +37 -0
- data/examples/configuration.rb +56 -0
- data/examples/event_stream.rb +19 -0
- data/examples/simple.rb +61 -0
- data/features/event.feature +319 -0
- data/features/events.feature +208 -0
- data/features/stash.feature +840 -0
- data/features/stashes.feature +492 -0
- data/features/step_definitions/stash_steps.rb +113 -0
- data/features/stream.feature +30 -0
- data/features/support/em.rb +14 -0
- data/features/support/env.rb +13 -0
- data/lib/vayacondios/configuration.rb +63 -0
- data/lib/vayacondios/server/api.rb +126 -0
- data/lib/vayacondios/server/api_options.rb +56 -0
- data/lib/vayacondios/server/configuration.rb +23 -0
- data/lib/vayacondios/server/driver.rb +71 -0
- data/lib/vayacondios/server/drivers/mongo.rb +126 -0
- data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
- data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
- data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
- data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
- data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
- data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
- data/lib/vayacondios/server/models/document.rb +87 -0
- data/lib/vayacondios/server/models/event.rb +198 -0
- data/lib/vayacondios/server/models/stash.rb +100 -0
- data/lib/vayacondios/server.rb +35 -0
- data/lib/vayacondios-server.rb +19 -13
- data/lib/vayacondios.rb +22 -0
- data/pom.xml +124 -4
- data/spec/configuration_spec.rb +41 -0
- data/spec/server/api_options_spec.rb +32 -0
- data/spec/server/api_spec.rb +279 -0
- data/spec/server/configuration_spec.rb +27 -0
- data/spec/server/drivers/mongo_spec.rb +107 -0
- data/spec/server/handlers/event_handler_spec.rb +62 -0
- data/spec/server/handlers/events_handler_spec.rb +51 -0
- data/spec/server/handlers/stash_handler_spec.rb +68 -0
- data/spec/server/handlers/stashes_handler_spec.rb +50 -0
- data/spec/server/handlers/stream_handler_spec.rb +5 -0
- data/spec/server/models/document_spec.rb +9 -0
- data/spec/server/models/event_spec.rb +185 -0
- data/spec/server/models/stash_spec.rb +95 -0
- data/spec/spec_helper.rb +23 -3
- data/spec/support/database_helper.rb +42 -0
- data/spec/support/log_helper.rb +19 -0
- data/spec/support/shared_context_for_events.rb +22 -0
- data/spec/support/shared_context_for_stashes.rb +24 -0
- data/spec/support/shared_examples_for_handlers.rb +32 -0
- data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
- data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
- data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
- data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
- data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
- data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
- data/vayacondios-server.gemspec +9 -9
- metadata +127 -122
- checksums.yaml +0 -15
- data/.rspec +0 -2
- data/.yardopts +0 -10
- data/Guardfile +0 -41
- data/app/http_shim.rb +0 -71
- data/bin/vcd.sh +0 -27
- data/config/http_shim.rb +0 -43
- data/config/vayacondios.example.yaml +0 -7
- data/config/vayacondios.yaml +0 -7
- data/examples/java/ItemSetTest.java +0 -76
- data/lib/tasks/publish.rake +0 -23
- data/lib/tasks/spec.rake +0 -11
- data/lib/tasks/yard.rake +0 -2
- data/lib/vayacondios/client/config.rb +0 -7
- data/lib/vayacondios/client/configliere.rb +0 -38
- data/lib/vayacondios/client/cube_client.rb +0 -39
- data/lib/vayacondios/client/http_client.rb +0 -49
- data/lib/vayacondios/client/itemset.rb +0 -130
- data/lib/vayacondios/client/legacy_switch.rb +0 -43
- data/lib/vayacondios/client/notifier.rb +0 -123
- data/lib/vayacondios/client/zabbix_client.rb +0 -148
- data/lib/vayacondios/legacy_switch.rb +0 -43
- data/lib/vayacondios/server/errors/bad_request.rb +0 -6
- data/lib/vayacondios/server/errors/not_found.rb +0 -6
- data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
- data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
- data/lib/vayacondios/server/legacy_switch.rb +0 -43
- data/lib/vayacondios/server/model/config_document.rb +0 -89
- data/lib/vayacondios/server/model/document.rb +0 -25
- data/lib/vayacondios/server/model/event_document.rb +0 -94
- data/lib/vayacondios/server/model/itemset_document.rb +0 -126
- data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
- data/lib/vayacondios/server/rack/jsonize.rb +0 -43
- data/lib/vayacondios/server/rack/params.rb +0 -50
- data/lib/vayacondios/server/rack/path.rb +0 -23
- data/lib/vayacondios/server/rack/path_validation.rb +0 -22
- data/lib/vayacondios/version.rb +0 -3
- data/lib/vayacondios-client.rb +0 -22
- data/scripts/hadoop_monitor/configurable.rb +0 -66
- data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
- data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
- data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
- data/scripts/hadoop_monitor/hadoopable.rb +0 -65
- data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
- data/scripts/s3_cataloger/buckets +0 -33
- data/scripts/s3_cataloger/foreach_bucket +0 -88
- data/scripts/s3_cataloger/parse_ls.py +0 -391
- data/spec/client/itemset_legacy_spec.rb +0 -55
- data/spec/client/itemset_spec.rb +0 -60
- data/spec/client/notifier_spec.rb +0 -120
- data/spec/server/config_spec.rb +0 -113
- data/spec/server/event_spec.rb +0 -103
- data/spec/server/itemset_legacy_spec.rb +0 -320
- data/spec/server/itemset_spec.rb +0 -317
- data/spec/server/rack/extract_methods_spec.rb +0 -60
- data/spec/server/rack/path_spec.rb +0 -36
- data/spec/server/rack/path_validation_spec.rb +0 -22
- data/spec/server/server_spec.rb +0 -20
- data/spec/support/mongo_cleaner.rb +0 -32
- data/src/main/java/ItemSetTest.java +0 -76
- data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
- data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
- data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
- data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
- data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
- data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
- data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
- data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
- data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
- data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
- data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
- data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,342 @@
|
|
1
|
+
package com.infochimps.vayacondios;
|
2
|
+
|
3
|
+
import java.util.Map;
|
4
|
+
import java.util.List;
|
5
|
+
import java.io.IOException;
|
6
|
+
|
7
|
+
import org.slf4j.Logger;
|
8
|
+
import org.slf4j.LoggerFactory;
|
9
|
+
|
10
|
+
/** Base class for all concrete implementations of a Vayacondios client.
|
11
|
+
* <p>
|
12
|
+
* Is fixed to a specific organization at instantiation time since the
|
13
|
+
* common use case is reading/writing data for a single organization
|
14
|
+
* at a time. If reading/writing data for multiple organizations
|
15
|
+
* simultaneously, use multiple client instances.
|
16
|
+
* <p>
|
17
|
+
* Can be instantiated in "dry-run" mode in which it will not actually
|
18
|
+
* make any requests to the server, merely log its output.
|
19
|
+
* <p>
|
20
|
+
* Implements the {@link VayacondiosClient} interface by wrapping each
|
21
|
+
* method of is API (e.g. - {@link VayacondiosClient#announce(String
|
22
|
+
* topic, Map event)}) with logic for handling arguments, errors, and
|
23
|
+
* "dry-run" mode, ultimately delegating to a (protected) method to
|
24
|
+
* perform the actual request (e.g. - {@link
|
25
|
+
* BaseClient#performAnnounce(String topic, Map event)}). Subclasses
|
26
|
+
* should override these methods to provide a concrete implementation
|
27
|
+
* of a Vayacondios client.
|
28
|
+
*/
|
29
|
+
public class BaseClient implements VayacondiosClient {
|
30
|
+
|
31
|
+
//----------------------------------------------------------------------------
|
32
|
+
// Initialization & Properties
|
33
|
+
//----------------------------------------------------------------------------
|
34
|
+
|
35
|
+
public static String VERSION = "v2";
|
36
|
+
|
37
|
+
private static Logger LOG = LoggerFactory.getLogger(BaseClient.class);
|
38
|
+
|
39
|
+
private String _organization;
|
40
|
+
private Boolean _dryRun;
|
41
|
+
|
42
|
+
/**
|
43
|
+
* Create a new BaseClient instance for the given
|
44
|
+
* organization. optionally in "dry-run" mode.
|
45
|
+
*
|
46
|
+
* <blockquote><pre>{@code
|
47
|
+
* VayacondiosClient client = new BaseClient("my_organization", true); // client in dry-run mode
|
48
|
+
* }</pre></blockquote>
|
49
|
+
*
|
50
|
+
* @param organization name of the organization to read/write data for
|
51
|
+
* @param shouldDryRun whether or not to enter "dry-run" mode
|
52
|
+
*/
|
53
|
+
public BaseClient(String organization, Boolean shouldDryRun) {
|
54
|
+
this._organization = organization;
|
55
|
+
this._dryRun = shouldDryRun;
|
56
|
+
}
|
57
|
+
|
58
|
+
/**
|
59
|
+
* Create a new BaseClient instance for the given organization.
|
60
|
+
*
|
61
|
+
* @param organization name of the organization to read/write data for
|
62
|
+
*/
|
63
|
+
public BaseClient(String organization) {
|
64
|
+
this(organization, false);
|
65
|
+
}
|
66
|
+
|
67
|
+
/**
|
68
|
+
* Is this client in "dry-run" mode?
|
69
|
+
* <p>
|
70
|
+
* When in "dry-run" mode, the client will not actually make any
|
71
|
+
* requests to the server, it will merely log all its attempts at
|
72
|
+
* an elevated level.
|
73
|
+
* @return whether or not the client is in "dry-run" mode
|
74
|
+
*/
|
75
|
+
public Boolean dryRun() {
|
76
|
+
return _dryRun;
|
77
|
+
}
|
78
|
+
|
79
|
+
/**
|
80
|
+
* The organization this client will read/write data for.
|
81
|
+
*
|
82
|
+
* @return name of the organization
|
83
|
+
*/
|
84
|
+
public String organization() {
|
85
|
+
return _organization;
|
86
|
+
}
|
87
|
+
|
88
|
+
//----------------------------------------------------------------------------
|
89
|
+
// Public API
|
90
|
+
//----------------------------------------------------------------------------
|
91
|
+
|
92
|
+
/**
|
93
|
+
* {@inheritDoc}
|
94
|
+
*/
|
95
|
+
@Override
|
96
|
+
public void announce(String topic, Map<String,Object> event) {
|
97
|
+
logRequest("Announcing <" + topic + ">");
|
98
|
+
if (dryRun()) return;
|
99
|
+
try {
|
100
|
+
performAnnounce(topic, event);
|
101
|
+
} catch (IOException e) {
|
102
|
+
LOG.error("Announcing <" + topic + ">", e);
|
103
|
+
}
|
104
|
+
}
|
105
|
+
/**
|
106
|
+
* {@inheritDoc}
|
107
|
+
*/
|
108
|
+
@Override
|
109
|
+
public void announce(String topic, Map<String,Object> event, String id) {
|
110
|
+
logRequest("Announcing <" + topic + "/" + id + ">");
|
111
|
+
if (dryRun()) return;
|
112
|
+
try {
|
113
|
+
performAnnounce(topic, event, id);
|
114
|
+
} catch (IOException e) {
|
115
|
+
LOG.error("Announcing <" + topic + "/" + id + ">", e);
|
116
|
+
}
|
117
|
+
}
|
118
|
+
|
119
|
+
/**
|
120
|
+
* {@inheritDoc}
|
121
|
+
*/
|
122
|
+
@Override
|
123
|
+
public List<Map<String,Object>> events(String topic, Map<String,Object> query) {
|
124
|
+
logRequest("Searching events <" + topic + ">");
|
125
|
+
if (dryRun()) return null;
|
126
|
+
try {
|
127
|
+
return performEvents(topic, query);
|
128
|
+
} catch (IOException e) {
|
129
|
+
LOG.error("Searching events <" + topic + ">", e);
|
130
|
+
return null;
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
134
|
+
/**
|
135
|
+
* {@inheritDoc}
|
136
|
+
*/
|
137
|
+
@Override
|
138
|
+
public Map<String,Object> get(String topic) {
|
139
|
+
logRequest("Fetching <" + topic + ">");
|
140
|
+
if (dryRun()) return null;
|
141
|
+
try {
|
142
|
+
return performGet(topic);
|
143
|
+
} catch (IOException e) {
|
144
|
+
LOG.error("Fetching <" + topic + ">");
|
145
|
+
return null;
|
146
|
+
}
|
147
|
+
}
|
148
|
+
|
149
|
+
/**
|
150
|
+
* {@inheritDoc}
|
151
|
+
*/
|
152
|
+
@Override
|
153
|
+
public Map<String,Object> getMap(String topic, String id) {
|
154
|
+
logRequest("Fetching Map <" + topic + "/" + id + ">");
|
155
|
+
if (dryRun()) return null;
|
156
|
+
try {
|
157
|
+
return performGetMap(topic, id);
|
158
|
+
} catch (IOException e) {
|
159
|
+
LOG.error("Fetching Map <" + topic + "/" + id + ">", e);
|
160
|
+
return null;
|
161
|
+
}
|
162
|
+
}
|
163
|
+
/**
|
164
|
+
* {@inheritDoc}
|
165
|
+
*/
|
166
|
+
@Override
|
167
|
+
public List getList(String topic, String id) {
|
168
|
+
logRequest("Fetching List <" + topic + "/" + id + ">");
|
169
|
+
if (dryRun()) return null;
|
170
|
+
try {
|
171
|
+
return performGetList(topic, id);
|
172
|
+
} catch (IOException e) {
|
173
|
+
LOG.error("Fetching List <" + topic + "/" + id + ">", e);
|
174
|
+
return null;
|
175
|
+
}
|
176
|
+
}
|
177
|
+
/**
|
178
|
+
* {@inheritDoc}
|
179
|
+
*/
|
180
|
+
@Override
|
181
|
+
public String getString(String topic, String id) {
|
182
|
+
logRequest("Fetching String <" + topic + "/" + id + ">");
|
183
|
+
if (dryRun()) return null;
|
184
|
+
try {
|
185
|
+
return performGetString(topic, id);
|
186
|
+
} catch (IOException e) {
|
187
|
+
LOG.error("Fetching String <" + topic + "/" + id + ">", e);
|
188
|
+
return null;
|
189
|
+
}
|
190
|
+
}
|
191
|
+
/**
|
192
|
+
* {@inheritDoc}
|
193
|
+
*/
|
194
|
+
@Override
|
195
|
+
public Double getDouble(String topic, String id) {
|
196
|
+
logRequest("Fetching Double <" + topic + "/" + id + ">");
|
197
|
+
if (dryRun()) return null;
|
198
|
+
try {
|
199
|
+
return performGetDouble(topic, id);
|
200
|
+
} catch (IOException e) {
|
201
|
+
LOG.error("Fetching Double <" + topic + "/" + id + ">", e);
|
202
|
+
return null;
|
203
|
+
}
|
204
|
+
}
|
205
|
+
|
206
|
+
/**
|
207
|
+
* {@inheritDoc}
|
208
|
+
*/
|
209
|
+
@Override
|
210
|
+
public List<Map<String,Object>> stashes(Map<String,Object> query) {
|
211
|
+
logRequest("Searching stashes");
|
212
|
+
if (dryRun()) return null;
|
213
|
+
try {
|
214
|
+
return performStashes(query);
|
215
|
+
} catch (IOException e) {
|
216
|
+
LOG.error("Searching stashes", e);
|
217
|
+
return null;
|
218
|
+
}
|
219
|
+
}
|
220
|
+
|
221
|
+
/**
|
222
|
+
* {@inheritDoc}
|
223
|
+
*/
|
224
|
+
@Override
|
225
|
+
public void set(String topic, Map<String,Object> value) {
|
226
|
+
logRequest("Replacing <" + topic + ">");
|
227
|
+
if (dryRun()) return;
|
228
|
+
try {
|
229
|
+
performSet(topic, value);
|
230
|
+
} catch (IOException e) {
|
231
|
+
LOG.error("Replacing <" + topic + ">", e);
|
232
|
+
}
|
233
|
+
}
|
234
|
+
/**
|
235
|
+
* {@inheritDoc}
|
236
|
+
*/
|
237
|
+
@Override
|
238
|
+
public void set(String topic, String id, Object value) {
|
239
|
+
logRequest("Replacing <" + topic + "/" + id + ">");
|
240
|
+
if (dryRun()) return;
|
241
|
+
try {
|
242
|
+
performSet(topic, id, value);
|
243
|
+
} catch (IOException e) {
|
244
|
+
LOG.error("Replacing <" + topic + "/" + id + ">", e);
|
245
|
+
}
|
246
|
+
}
|
247
|
+
|
248
|
+
/**
|
249
|
+
* {@inheritDoc}
|
250
|
+
*/
|
251
|
+
@Override
|
252
|
+
public void merge(String topic, Map<String,Object> value) {
|
253
|
+
logRequest("Merging <" + topic + ">");
|
254
|
+
if (dryRun()) return;
|
255
|
+
try {
|
256
|
+
performMerge(topic, value);
|
257
|
+
} catch (IOException e) {
|
258
|
+
LOG.error("Merging <" + topic + ">", e);
|
259
|
+
}
|
260
|
+
}
|
261
|
+
/**
|
262
|
+
* {@inheritDoc}
|
263
|
+
*/
|
264
|
+
@Override
|
265
|
+
public void merge(String topic, String id, Object value) {
|
266
|
+
logRequest("Merging <" + topic + "/" + id + ">");
|
267
|
+
if (dryRun()) return;
|
268
|
+
try {
|
269
|
+
performMerge(topic, id, value);
|
270
|
+
} catch (IOException e) {
|
271
|
+
LOG.error("Merging <" + topic + "/" + id + ">", e);
|
272
|
+
}
|
273
|
+
}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* {@inheritDoc}
|
277
|
+
*/
|
278
|
+
@Override
|
279
|
+
public void delete(String topic) {
|
280
|
+
logRequest("Deleting <" + topic + ">");
|
281
|
+
if (dryRun()) return;
|
282
|
+
try {
|
283
|
+
performDelete(topic);
|
284
|
+
} catch (IOException e) {
|
285
|
+
LOG.error("Deleting <" + topic + ">");
|
286
|
+
}
|
287
|
+
}
|
288
|
+
/**
|
289
|
+
* {@inheritDoc}
|
290
|
+
*/
|
291
|
+
@Override
|
292
|
+
public void delete(String topic, String id) {
|
293
|
+
logRequest("Deleting <" + topic + "/" + id + ">");
|
294
|
+
if (dryRun()) return;
|
295
|
+
try {
|
296
|
+
performDelete(topic, id);
|
297
|
+
} catch (IOException e) {
|
298
|
+
LOG.error("Deleting <" + topic + "/" + id + ">");
|
299
|
+
}
|
300
|
+
}
|
301
|
+
|
302
|
+
/**
|
303
|
+
* {@inheritDoc}
|
304
|
+
*/
|
305
|
+
@Override
|
306
|
+
public void close() {};
|
307
|
+
|
308
|
+
//----------------------------------------------------------------------------
|
309
|
+
// Private API
|
310
|
+
//----------------------------------------------------------------------------
|
311
|
+
|
312
|
+
protected void performAnnounce(String topic, Map<String,Object> event, String id) throws IOException {}
|
313
|
+
protected void performAnnounce(String topic, Map<String,Object> event) throws IOException {}
|
314
|
+
|
315
|
+
protected List<Map<String,Object>> performEvents(String topic, Map<String,Object> query) throws IOException { return null; }
|
316
|
+
|
317
|
+
protected Map<String,Object> performGet(String topic) throws IOException { return null; }
|
318
|
+
|
319
|
+
protected Map<String,Object> performGetMap(String topic, String id) throws IOException { return null; }
|
320
|
+
protected List performGetList(String topic, String id) throws IOException { return null; }
|
321
|
+
protected String performGetString(String topic, String id) throws IOException { return null; }
|
322
|
+
protected Double performGetDouble(String topic, String id) throws IOException { return null; }
|
323
|
+
|
324
|
+
protected List<Map<String,Object>> performStashes(Map<String,Object> query) throws IOException { return null; }
|
325
|
+
|
326
|
+
protected void performMerge(String topic, String id, Object value) throws IOException {}
|
327
|
+
protected void performMerge(String topic, Map<String,Object> value) throws IOException {}
|
328
|
+
|
329
|
+
protected void performSet(String topic, String id, Object value) throws IOException {}
|
330
|
+
protected void performSet(String topic, Map<String,Object> value) throws IOException {}
|
331
|
+
|
332
|
+
protected void performDelete(String topic, String id) throws IOException {}
|
333
|
+
protected void performDelete(String topic) throws IOException {}
|
334
|
+
|
335
|
+
private void logRequest(String message) {
|
336
|
+
if (dryRun()) {
|
337
|
+
LOG.info(message);
|
338
|
+
} else {
|
339
|
+
LOG.debug(message);
|
340
|
+
}
|
341
|
+
}
|
342
|
+
}
|