vayacondios-server 0.1.2 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -0
- data/app/http_shim.rb +1 -5
- data/lib/vayacondios-client.rb +2 -0
- data/lib/vayacondios-server.rb +1 -0
- data/lib/vayacondios/client/cube_client.rb +39 -0
- data/lib/vayacondios/client/itemset.rb +43 -28
- data/lib/vayacondios/client/notifier.rb +24 -1
- data/lib/vayacondios/client/zabbix_client.rb +148 -0
- data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -1
- data/lib/vayacondios/server/model/itemset_document.rb +8 -4
- data/lib/vayacondios/server/rack/assume_json.rb +13 -0
- data/lib/vayacondios/server/rack/extract_methods.rb +11 -1
- data/lib/vayacondios/version.rb +1 -1
- data/pom.xml +97 -0
- data/scripts/hadoop_monitor/configurable.rb +1 -1
- data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +6 -3
- data/scripts/hadoop_monitor/hadoop_client.rb +20 -19
- data/scripts/hadoop_monitor/hadoop_monitor.rb +3 -3
- data/scripts/hadoop_monitor/hadoopable.rb +3 -3
- data/scripts/hadoop_monitor/machine_monitor.rb +2 -2
- data/spec/client/itemset_spec.rb +8 -8
- data/spec/server/itemset_spec.rb +4 -4
- data/src/main/java/com/infochimps/util/CurrentClass.java +26 -0
- data/src/main/java/com/infochimps/util/DebugUtil.java +38 -0
- data/src/main/java/com/infochimps/util/HttpHelper.java +112 -0
- data/src/main/java/com/infochimps/vayacondios/ItemSets.java +456 -0
- data/src/main/java/com/infochimps/vayacondios/Organization.java +49 -0
- data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +13 -0
- data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +68 -0
- data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +88 -0
- data/vayacondios-client.gemspec +2 -2
- data/vayacondios-server.gemspec +4 -2
- metadata +37 -9
@@ -0,0 +1,456 @@
|
|
1
|
+
package com.infochimps.vayacondios;
|
2
|
+
|
3
|
+
import static com.infochimps.util.CurrentClass.getLogger;
|
4
|
+
import com.infochimps.util.DebugUtil;
|
5
|
+
|
6
|
+
import com.google.gson.Gson;
|
7
|
+
import com.google.gson.GsonBuilder;
|
8
|
+
import com.google.gson.JsonElement;
|
9
|
+
import com.google.gson.JsonIOException;
|
10
|
+
import com.google.gson.JsonObject;
|
11
|
+
import com.google.gson.JsonParser;
|
12
|
+
import com.google.gson.JsonPrimitive;
|
13
|
+
import com.google.gson.JsonParseException;
|
14
|
+
import com.google.gson.JsonSerializer;
|
15
|
+
import com.google.gson.JsonSerializationContext;
|
16
|
+
import com.google.gson.JsonSyntaxException;
|
17
|
+
|
18
|
+
import java.io.BufferedReader;
|
19
|
+
import java.util.HashMap;
|
20
|
+
import java.io.InputStream;
|
21
|
+
import java.io.OutputStream;
|
22
|
+
import java.io.OutputStreamWriter;
|
23
|
+
import java.io.IOException;
|
24
|
+
import java.lang.reflect.Type;
|
25
|
+
import java.net.HttpURLConnection;
|
26
|
+
import java.util.ArrayList;
|
27
|
+
import java.util.List;
|
28
|
+
|
29
|
+
import java.net.URL;
|
30
|
+
|
31
|
+
import org.slf4j.Logger;
|
32
|
+
|
33
|
+
/**
|
34
|
+
* This is the first level of the Vayacondios path-building hierarchy
|
35
|
+
* that is capable of directly manipulating itemsets.
|
36
|
+
*/
|
37
|
+
public class ItemSets extends Organization {
|
38
|
+
public ItemSets(PathBuilder delegate) { super(delegate); }
|
39
|
+
|
40
|
+
public ItemSets(Organization org) {
|
41
|
+
super(org);
|
42
|
+
_org = org;
|
43
|
+
}
|
44
|
+
|
45
|
+
//----------------------------------------------------------------------------
|
46
|
+
// next in path hierarchy
|
47
|
+
//----------------------------------------------------------------------------
|
48
|
+
|
49
|
+
/**
|
50
|
+
* @param topic Vayacondios topic. see Vayacondios documentation for
|
51
|
+
* details.
|
52
|
+
* @return new Topic path builder for this organization
|
53
|
+
*/
|
54
|
+
public Topic topic(String topic) { return new Topic(this, topic); }
|
55
|
+
|
56
|
+
//----------------------------------------------------------------------------
|
57
|
+
// public operations
|
58
|
+
//----------------------------------------------------------------------------
|
59
|
+
|
60
|
+
/**
|
61
|
+
* Fetches the current value of an itemset.
|
62
|
+
*
|
63
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
64
|
+
* documentation for further details.
|
65
|
+
* @param id A Vayacondios id, together with the server,
|
66
|
+
* organization, and topic, specifies a unique
|
67
|
+
* itemset.
|
68
|
+
* @return a collection of items
|
69
|
+
*/
|
70
|
+
public List<Item> fetch(String topic, String id) throws IOException {
|
71
|
+
BufferedReader reader = openUrl(urlString(PATH_COMPONENT, topic, id));
|
72
|
+
String line = reader.readLine();
|
73
|
+
JsonElement response;
|
74
|
+
JsonElement itemSet;
|
75
|
+
|
76
|
+
ArrayList<Item> result = new ArrayList<Item>();
|
77
|
+
|
78
|
+
// assume Vayacondios response comes in a single line
|
79
|
+
if (line != null &&
|
80
|
+
(response = PARSER.parse(line)).isJsonObject() &&
|
81
|
+
(itemSet = (response.getAsJsonObject().get("contents"))).isJsonArray()) {
|
82
|
+
for (JsonElement elem : itemSet.getAsJsonArray()) {
|
83
|
+
if (!elem.isJsonPrimitive()) {
|
84
|
+
LOG.warn("ignoring non-primitive in itemset: " + elem);
|
85
|
+
continue;
|
86
|
+
}
|
87
|
+
|
88
|
+
JsonPrimitive item = elem.getAsJsonPrimitive();
|
89
|
+
if (item.isBoolean()) result.add(new Item(item.getAsBoolean()));
|
90
|
+
else if (item.isNumber()) result.add(new Item(item.getAsNumber()));
|
91
|
+
else if (item.isString()) result.add(new Item(item.getAsString()));
|
92
|
+
|
93
|
+
else LOG.warn("ignoring unrecognized type in itemset: " + item);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
if ((line = reader.readLine()) != null)
|
98
|
+
LOG.warn("expected eof but saw " + line);
|
99
|
+
|
100
|
+
reader.close();
|
101
|
+
|
102
|
+
return result;
|
103
|
+
}
|
104
|
+
|
105
|
+
/**
|
106
|
+
* Creates a new itemset with the specified topic and id, clobbering
|
107
|
+
* any existing itemset with the same topic and id.
|
108
|
+
*
|
109
|
+
* @param items items whose existence should be ensured in the set.
|
110
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
111
|
+
* documentation for further details.
|
112
|
+
* @param id A Vayacondios id, together with the server,
|
113
|
+
* organization, and topic, specifies a unique
|
114
|
+
* itemset.
|
115
|
+
*/
|
116
|
+
public void create(List<Item> items,
|
117
|
+
String topic,
|
118
|
+
String id) throws IOException {
|
119
|
+
mutate("PUT", items, topic, id);
|
120
|
+
}
|
121
|
+
|
122
|
+
/**
|
123
|
+
* Ensures the absence of the specified items from the specified itemset.q
|
124
|
+
*
|
125
|
+
* @param items items whose absence should be ensured in the set.
|
126
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
127
|
+
* documentation for further details.
|
128
|
+
* @param id A Vayacondios id, together with the server,
|
129
|
+
* organization, and topic, specifies a unique
|
130
|
+
* itemset.
|
131
|
+
*/
|
132
|
+
public void remove(List<Item> items,
|
133
|
+
String topic,
|
134
|
+
String id) throws IOException {
|
135
|
+
mutate("DELETE", items, topic, id);
|
136
|
+
}
|
137
|
+
|
138
|
+
/**
|
139
|
+
* Updates the current value of an itemset, ensuring the existence
|
140
|
+
* of the specified items.
|
141
|
+
*
|
142
|
+
* @param items items whose existence should be ensured in the set.
|
143
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
144
|
+
* documentation for further details.
|
145
|
+
* @param id A Vayacondios id, together with the server,
|
146
|
+
* organization, and topic, specifies a unique
|
147
|
+
* itemset.
|
148
|
+
*/
|
149
|
+
public void update(List<Item> items,
|
150
|
+
String topic,
|
151
|
+
String id) throws IOException {
|
152
|
+
mutate("PATCH", items, topic, id);
|
153
|
+
}
|
154
|
+
|
155
|
+
//----------------------------------------------------------------------------
|
156
|
+
// API HTTP path components
|
157
|
+
//----------------------------------------------------------------------------
|
158
|
+
|
159
|
+
String getOrganization() {
|
160
|
+
return ((Organization)getDelegate()).getOrganization();
|
161
|
+
}
|
162
|
+
|
163
|
+
//----------------------------------------------------------------------------
|
164
|
+
|
165
|
+
protected void mutate(String method,
|
166
|
+
List<Item> items,
|
167
|
+
String topic,
|
168
|
+
String id) throws IOException {
|
169
|
+
|
170
|
+
// serialize the items
|
171
|
+
HashMap content = new HashMap();
|
172
|
+
content.put("contents", items);
|
173
|
+
String body = GSON.toJson(content);
|
174
|
+
// connect to our standard path
|
175
|
+
URL url = new URL(urlString(PATH_COMPONENT, topic, id));
|
176
|
+
HttpURLConnection connection = (HttpURLConnection)
|
177
|
+
((Boolean.valueOf(System.getProperty("ics.http.use_charles"))) ?
|
178
|
+
url.openConnection(DebugUtil.useCharles()) : url.openConnection());
|
179
|
+
|
180
|
+
// configure connection
|
181
|
+
connection.setDoOutput(true);
|
182
|
+
|
183
|
+
// NOTE: Uncommenting this (and not calling
|
184
|
+
// connection.getInputStream()) causes Java to hang without
|
185
|
+
// sending its payload.
|
186
|
+
|
187
|
+
// connection.setDoInput(false);
|
188
|
+
|
189
|
+
if (method.equals("DELETE")) {
|
190
|
+
connection.setRequestMethod("PUT");
|
191
|
+
connection.setRequestProperty("X-Method", "DELETE");
|
192
|
+
} else if (method.equals("PATCH")) {
|
193
|
+
connection.setRequestMethod("PUT");
|
194
|
+
connection.setRequestProperty("X-Method", "PATCH");
|
195
|
+
} else connection.setRequestMethod(method);
|
196
|
+
connection.setRequestProperty("Content-Type", "application/json");
|
197
|
+
connection.setRequestProperty("Accept", "*/*");
|
198
|
+
connection.setRequestProperty("Content-Length",
|
199
|
+
Integer.toString(body.getBytes().length));
|
200
|
+
connection.setUseCaches(false);
|
201
|
+
|
202
|
+
LOG.debug("sending: " + body);
|
203
|
+
LOG.debug("via " +
|
204
|
+
connection.getRequestMethod() +
|
205
|
+
" to " +
|
206
|
+
urlString(PATH_COMPONENT, topic, id));
|
207
|
+
|
208
|
+
// connect and write
|
209
|
+
OutputStream os = connection.getOutputStream();
|
210
|
+
os.write(body.getBytes("UTF-8"));
|
211
|
+
os.flush();
|
212
|
+
os.close();
|
213
|
+
|
214
|
+
// ignore reponse
|
215
|
+
InputStream is = connection.getInputStream();
|
216
|
+
|
217
|
+
LOG.trace("ignoring response from Vayacondios.");
|
218
|
+
byte buf[] = new byte[256];
|
219
|
+
while (is.read(buf) != -1);
|
220
|
+
LOG.trace("response ignored.");
|
221
|
+
is.close();
|
222
|
+
|
223
|
+
// fin.
|
224
|
+
connection.disconnect();
|
225
|
+
}
|
226
|
+
|
227
|
+
private Organization _org;
|
228
|
+
|
229
|
+
private static final JsonParser PARSER = new JsonParser();
|
230
|
+
private static final Logger LOG = getLogger();
|
231
|
+
private static final String PATH_COMPONENT = "itemset";
|
232
|
+
private static final Gson GSON = new GsonBuilder().
|
233
|
+
registerTypeAdapter(Item.class, new Item.Serializer()).
|
234
|
+
create();
|
235
|
+
|
236
|
+
//============================================================================
|
237
|
+
// Topic
|
238
|
+
//============================================================================
|
239
|
+
|
240
|
+
/**
|
241
|
+
* A Topic may have many ids, each of which points to a unique
|
242
|
+
* Vayacondios itemset.
|
243
|
+
*/
|
244
|
+
public static class Topic extends ItemSets {
|
245
|
+
public Topic(PathBuilder delegate) { super(delegate); }
|
246
|
+
|
247
|
+
public Topic(ItemSets sets, String topic) {
|
248
|
+
super(sets);
|
249
|
+
_sets = sets;
|
250
|
+
_topic = topic;
|
251
|
+
}
|
252
|
+
|
253
|
+
//--------------------------------------------------------------------------
|
254
|
+
// next in path hierarchy
|
255
|
+
//--------------------------------------------------------------------------
|
256
|
+
|
257
|
+
/**
|
258
|
+
* @param id Vayacondios id. see Vayacondios documentation for
|
259
|
+
* details.
|
260
|
+
* @return new itemset for this topic
|
261
|
+
*/
|
262
|
+
public ItemSet itemSet(String id) { return new ItemSet(this, id); }
|
263
|
+
|
264
|
+
//--------------------------------------------------------------------------
|
265
|
+
// public operations
|
266
|
+
//--------------------------------------------------------------------------
|
267
|
+
|
268
|
+
/**
|
269
|
+
* @see #ItemSets.fetch
|
270
|
+
*/
|
271
|
+
public List<Item> fetch(String id) throws IOException {
|
272
|
+
return fetch(getTopic(), id);
|
273
|
+
}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* @see #ItemSets.create
|
277
|
+
*/
|
278
|
+
public void create(List<Item> items, String id) throws IOException {
|
279
|
+
create(items, getTopic(), id);
|
280
|
+
}
|
281
|
+
|
282
|
+
/**
|
283
|
+
* @see #ItemSets.remove
|
284
|
+
*/
|
285
|
+
public void remove(List<Item> items, String id) throws IOException {
|
286
|
+
remove(items, getTopic(), id);
|
287
|
+
}
|
288
|
+
|
289
|
+
/**
|
290
|
+
* @see #ItemSets.update
|
291
|
+
*/
|
292
|
+
public void update(List<Item> items, String id) throws IOException {
|
293
|
+
update(items, getTopic(), id);
|
294
|
+
}
|
295
|
+
|
296
|
+
//--------------------------------------------------------------------------
|
297
|
+
// API HTTP path components
|
298
|
+
//--------------------------------------------------------------------------
|
299
|
+
|
300
|
+
protected String getTopic() { return _topic; }
|
301
|
+
|
302
|
+
//--------------------------------------------------------------------------
|
303
|
+
// fields
|
304
|
+
//--------------------------------------------------------------------------
|
305
|
+
|
306
|
+
private ItemSets _sets;
|
307
|
+
private String _topic;
|
308
|
+
}
|
309
|
+
|
310
|
+
//============================================================================
|
311
|
+
// ItemSet
|
312
|
+
//============================================================================
|
313
|
+
|
314
|
+
/**
|
315
|
+
* A Vayacodios Itemset is manipulated using four API methods
|
316
|
+
* implemented in terms of the Vayacondios API.
|
317
|
+
*/
|
318
|
+
public static class ItemSet extends Topic {
|
319
|
+
public ItemSet(PathBuilder delegate) { super(delegate); }
|
320
|
+
|
321
|
+
public ItemSet(Topic topic, String id) {
|
322
|
+
super(topic);
|
323
|
+
_topic = topic;
|
324
|
+
_id = id;
|
325
|
+
}
|
326
|
+
|
327
|
+
//--------------------------------------------------------------------------
|
328
|
+
// public operations
|
329
|
+
//--------------------------------------------------------------------------
|
330
|
+
|
331
|
+
/**
|
332
|
+
* @see ItemSets.fetch
|
333
|
+
*/
|
334
|
+
public List<Item> fetch() throws IOException {
|
335
|
+
return fetch(getId());
|
336
|
+
}
|
337
|
+
|
338
|
+
/**
|
339
|
+
* @see ItemSets.create
|
340
|
+
*/
|
341
|
+
public void create(List<Item> items) throws IOException {
|
342
|
+
create(items, getId());
|
343
|
+
}
|
344
|
+
|
345
|
+
/**
|
346
|
+
* @see ItemSets.remove
|
347
|
+
*/
|
348
|
+
public void remove(List<Item> items) throws IOException {
|
349
|
+
remove(items, getId());
|
350
|
+
}
|
351
|
+
|
352
|
+
/**
|
353
|
+
* @see ItemSets.update
|
354
|
+
*/
|
355
|
+
public void update(List<Item> items) throws IOException {
|
356
|
+
update(items, getId());
|
357
|
+
}
|
358
|
+
|
359
|
+
//--------------------------------------------------------------------------
|
360
|
+
// API HTTP path components
|
361
|
+
//--------------------------------------------------------------------------
|
362
|
+
|
363
|
+
protected String getId() {
|
364
|
+
return _id;
|
365
|
+
}
|
366
|
+
|
367
|
+
protected String getTopic() {
|
368
|
+
return ((Topic)getDelegate()).getTopic();
|
369
|
+
}
|
370
|
+
|
371
|
+
//--------------------------------------------------------------------------
|
372
|
+
// fields
|
373
|
+
//--------------------------------------------------------------------------
|
374
|
+
|
375
|
+
private Topic _topic;
|
376
|
+
private String _id;
|
377
|
+
}
|
378
|
+
|
379
|
+
//============================================================================
|
380
|
+
// Item
|
381
|
+
//============================================================================
|
382
|
+
|
383
|
+
/**
|
384
|
+
* A Vayacondios item can be a boolean, a number, or a string.
|
385
|
+
*/
|
386
|
+
public static class Item {
|
387
|
+
static class Serializer implements JsonSerializer {
|
388
|
+
public JsonElement serialize(Object item,
|
389
|
+
Type typeOfId,
|
390
|
+
JsonSerializationContext context) {
|
391
|
+
return GSON.toJsonTree(Item.class.isAssignableFrom(item.getClass()) ?
|
392
|
+
((Item)item).getObject() : item);
|
393
|
+
}
|
394
|
+
private static final Gson GSON = new Gson();
|
395
|
+
private static final Logger LOG = getLogger();
|
396
|
+
}
|
397
|
+
|
398
|
+
public Item(Boolean b) {
|
399
|
+
_item = b;
|
400
|
+
_type = TYPE.BOOLEAN;
|
401
|
+
}
|
402
|
+
|
403
|
+
public Item(Number n) {
|
404
|
+
_item = n;
|
405
|
+
_type = TYPE.NUMBER;
|
406
|
+
}
|
407
|
+
|
408
|
+
public Item(String s) {
|
409
|
+
_item = s;
|
410
|
+
_type = TYPE.STRING;
|
411
|
+
}
|
412
|
+
|
413
|
+
public Boolean getAsBoolean() {
|
414
|
+
if (_type != TYPE.BOOLEAN)
|
415
|
+
throw new ClassCastException("item is not a boolean");
|
416
|
+
return (Boolean)_item;
|
417
|
+
}
|
418
|
+
|
419
|
+
public Number getAsNumber() {
|
420
|
+
if (_type != TYPE.NUMBER)
|
421
|
+
throw new ClassCastException("item is not a number");
|
422
|
+
return (Number)_item;
|
423
|
+
}
|
424
|
+
|
425
|
+
public String getAsString() {
|
426
|
+
if (_type != TYPE.STRING)
|
427
|
+
throw new ClassCastException("item is not a string");
|
428
|
+
return (String)_item;
|
429
|
+
}
|
430
|
+
|
431
|
+
public Boolean isBoolean() { return (_type == TYPE.BOOLEAN); }
|
432
|
+
public Boolean isNumber() { return (_type == TYPE.NUMBER ); }
|
433
|
+
public Boolean isString() { return (_type == TYPE.STRING ); }
|
434
|
+
|
435
|
+
public Object getObject() { return _item; }
|
436
|
+
|
437
|
+
public TYPE getType() { return _type; }
|
438
|
+
|
439
|
+
public String toString() {
|
440
|
+
return _item.toString() + ":" + _type;
|
441
|
+
}
|
442
|
+
|
443
|
+
public boolean equals(Object other) {
|
444
|
+
return (Item.class.isAssignableFrom(other.getClass())) ?
|
445
|
+
_item.equals(((Item)other).getObject()) : _item.equals(other);
|
446
|
+
}
|
447
|
+
|
448
|
+
//--------------------------------------------------------------------------
|
449
|
+
// fields
|
450
|
+
//--------------------------------------------------------------------------
|
451
|
+
|
452
|
+
private Object _item;
|
453
|
+
private TYPE _type;
|
454
|
+
private enum TYPE {BOOLEAN, NUMBER, STRING}
|
455
|
+
}
|
456
|
+
}
|