vayacondios-server 0.1.2 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -0
- data/app/http_shim.rb +1 -5
- data/lib/vayacondios-client.rb +2 -0
- data/lib/vayacondios-server.rb +1 -0
- data/lib/vayacondios/client/cube_client.rb +39 -0
- data/lib/vayacondios/client/itemset.rb +43 -28
- data/lib/vayacondios/client/notifier.rb +24 -1
- data/lib/vayacondios/client/zabbix_client.rb +148 -0
- data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -1
- data/lib/vayacondios/server/model/itemset_document.rb +8 -4
- data/lib/vayacondios/server/rack/assume_json.rb +13 -0
- data/lib/vayacondios/server/rack/extract_methods.rb +11 -1
- data/lib/vayacondios/version.rb +1 -1
- data/pom.xml +97 -0
- data/scripts/hadoop_monitor/configurable.rb +1 -1
- data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +6 -3
- data/scripts/hadoop_monitor/hadoop_client.rb +20 -19
- data/scripts/hadoop_monitor/hadoop_monitor.rb +3 -3
- data/scripts/hadoop_monitor/hadoopable.rb +3 -3
- data/scripts/hadoop_monitor/machine_monitor.rb +2 -2
- data/spec/client/itemset_spec.rb +8 -8
- data/spec/server/itemset_spec.rb +4 -4
- data/src/main/java/com/infochimps/util/CurrentClass.java +26 -0
- data/src/main/java/com/infochimps/util/DebugUtil.java +38 -0
- data/src/main/java/com/infochimps/util/HttpHelper.java +112 -0
- data/src/main/java/com/infochimps/vayacondios/ItemSets.java +456 -0
- data/src/main/java/com/infochimps/vayacondios/Organization.java +49 -0
- data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +13 -0
- data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +68 -0
- data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +88 -0
- data/vayacondios-client.gemspec +2 -2
- data/vayacondios-server.gemspec +4 -2
- metadata +37 -9
@@ -0,0 +1,456 @@
|
|
1
|
+
package com.infochimps.vayacondios;
|
2
|
+
|
3
|
+
import static com.infochimps.util.CurrentClass.getLogger;
|
4
|
+
import com.infochimps.util.DebugUtil;
|
5
|
+
|
6
|
+
import com.google.gson.Gson;
|
7
|
+
import com.google.gson.GsonBuilder;
|
8
|
+
import com.google.gson.JsonElement;
|
9
|
+
import com.google.gson.JsonIOException;
|
10
|
+
import com.google.gson.JsonObject;
|
11
|
+
import com.google.gson.JsonParser;
|
12
|
+
import com.google.gson.JsonPrimitive;
|
13
|
+
import com.google.gson.JsonParseException;
|
14
|
+
import com.google.gson.JsonSerializer;
|
15
|
+
import com.google.gson.JsonSerializationContext;
|
16
|
+
import com.google.gson.JsonSyntaxException;
|
17
|
+
|
18
|
+
import java.io.BufferedReader;
|
19
|
+
import java.util.HashMap;
|
20
|
+
import java.io.InputStream;
|
21
|
+
import java.io.OutputStream;
|
22
|
+
import java.io.OutputStreamWriter;
|
23
|
+
import java.io.IOException;
|
24
|
+
import java.lang.reflect.Type;
|
25
|
+
import java.net.HttpURLConnection;
|
26
|
+
import java.util.ArrayList;
|
27
|
+
import java.util.List;
|
28
|
+
|
29
|
+
import java.net.URL;
|
30
|
+
|
31
|
+
import org.slf4j.Logger;
|
32
|
+
|
33
|
+
/**
|
34
|
+
* This is the first level of the Vayacondios path-building hierarchy
|
35
|
+
* that is capable of directly manipulating itemsets.
|
36
|
+
*/
|
37
|
+
public class ItemSets extends Organization {
|
38
|
+
public ItemSets(PathBuilder delegate) { super(delegate); }
|
39
|
+
|
40
|
+
public ItemSets(Organization org) {
|
41
|
+
super(org);
|
42
|
+
_org = org;
|
43
|
+
}
|
44
|
+
|
45
|
+
//----------------------------------------------------------------------------
|
46
|
+
// next in path hierarchy
|
47
|
+
//----------------------------------------------------------------------------
|
48
|
+
|
49
|
+
/**
|
50
|
+
* @param topic Vayacondios topic. see Vayacondios documentation for
|
51
|
+
* details.
|
52
|
+
* @return new Topic path builder for this organization
|
53
|
+
*/
|
54
|
+
public Topic topic(String topic) { return new Topic(this, topic); }
|
55
|
+
|
56
|
+
//----------------------------------------------------------------------------
|
57
|
+
// public operations
|
58
|
+
//----------------------------------------------------------------------------
|
59
|
+
|
60
|
+
/**
|
61
|
+
* Fetches the current value of an itemset.
|
62
|
+
*
|
63
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
64
|
+
* documentation for further details.
|
65
|
+
* @param id A Vayacondios id, together with the server,
|
66
|
+
* organization, and topic, specifies a unique
|
67
|
+
* itemset.
|
68
|
+
* @return a collection of items
|
69
|
+
*/
|
70
|
+
public List<Item> fetch(String topic, String id) throws IOException {
|
71
|
+
BufferedReader reader = openUrl(urlString(PATH_COMPONENT, topic, id));
|
72
|
+
String line = reader.readLine();
|
73
|
+
JsonElement response;
|
74
|
+
JsonElement itemSet;
|
75
|
+
|
76
|
+
ArrayList<Item> result = new ArrayList<Item>();
|
77
|
+
|
78
|
+
// assume Vayacondios response comes in a single line
|
79
|
+
if (line != null &&
|
80
|
+
(response = PARSER.parse(line)).isJsonObject() &&
|
81
|
+
(itemSet = (response.getAsJsonObject().get("contents"))).isJsonArray()) {
|
82
|
+
for (JsonElement elem : itemSet.getAsJsonArray()) {
|
83
|
+
if (!elem.isJsonPrimitive()) {
|
84
|
+
LOG.warn("ignoring non-primitive in itemset: " + elem);
|
85
|
+
continue;
|
86
|
+
}
|
87
|
+
|
88
|
+
JsonPrimitive item = elem.getAsJsonPrimitive();
|
89
|
+
if (item.isBoolean()) result.add(new Item(item.getAsBoolean()));
|
90
|
+
else if (item.isNumber()) result.add(new Item(item.getAsNumber()));
|
91
|
+
else if (item.isString()) result.add(new Item(item.getAsString()));
|
92
|
+
|
93
|
+
else LOG.warn("ignoring unrecognized type in itemset: " + item);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
if ((line = reader.readLine()) != null)
|
98
|
+
LOG.warn("expected eof but saw " + line);
|
99
|
+
|
100
|
+
reader.close();
|
101
|
+
|
102
|
+
return result;
|
103
|
+
}
|
104
|
+
|
105
|
+
/**
|
106
|
+
* Creates a new itemset with the specified topic and id, clobbering
|
107
|
+
* any existing itemset with the same topic and id.
|
108
|
+
*
|
109
|
+
* @param items items whose existence should be ensured in the set.
|
110
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
111
|
+
* documentation for further details.
|
112
|
+
* @param id A Vayacondios id, together with the server,
|
113
|
+
* organization, and topic, specifies a unique
|
114
|
+
* itemset.
|
115
|
+
*/
|
116
|
+
public void create(List<Item> items,
|
117
|
+
String topic,
|
118
|
+
String id) throws IOException {
|
119
|
+
mutate("PUT", items, topic, id);
|
120
|
+
}
|
121
|
+
|
122
|
+
/**
|
123
|
+
* Ensures the absence of the specified items from the specified itemset.q
|
124
|
+
*
|
125
|
+
* @param items items whose absence should be ensured in the set.
|
126
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
127
|
+
* documentation for further details.
|
128
|
+
* @param id A Vayacondios id, together with the server,
|
129
|
+
* organization, and topic, specifies a unique
|
130
|
+
* itemset.
|
131
|
+
*/
|
132
|
+
public void remove(List<Item> items,
|
133
|
+
String topic,
|
134
|
+
String id) throws IOException {
|
135
|
+
mutate("DELETE", items, topic, id);
|
136
|
+
}
|
137
|
+
|
138
|
+
/**
|
139
|
+
* Updates the current value of an itemset, ensuring the existence
|
140
|
+
* of the specified items.
|
141
|
+
*
|
142
|
+
* @param items items whose existence should be ensured in the set.
|
143
|
+
* @param topic A Vayacondios topic has many ids. See vayacondios
|
144
|
+
* documentation for further details.
|
145
|
+
* @param id A Vayacondios id, together with the server,
|
146
|
+
* organization, and topic, specifies a unique
|
147
|
+
* itemset.
|
148
|
+
*/
|
149
|
+
public void update(List<Item> items,
|
150
|
+
String topic,
|
151
|
+
String id) throws IOException {
|
152
|
+
mutate("PATCH", items, topic, id);
|
153
|
+
}
|
154
|
+
|
155
|
+
//----------------------------------------------------------------------------
|
156
|
+
// API HTTP path components
|
157
|
+
//----------------------------------------------------------------------------
|
158
|
+
|
159
|
+
String getOrganization() {
|
160
|
+
return ((Organization)getDelegate()).getOrganization();
|
161
|
+
}
|
162
|
+
|
163
|
+
//----------------------------------------------------------------------------
|
164
|
+
|
165
|
+
protected void mutate(String method,
|
166
|
+
List<Item> items,
|
167
|
+
String topic,
|
168
|
+
String id) throws IOException {
|
169
|
+
|
170
|
+
// serialize the items
|
171
|
+
HashMap content = new HashMap();
|
172
|
+
content.put("contents", items);
|
173
|
+
String body = GSON.toJson(content);
|
174
|
+
// connect to our standard path
|
175
|
+
URL url = new URL(urlString(PATH_COMPONENT, topic, id));
|
176
|
+
HttpURLConnection connection = (HttpURLConnection)
|
177
|
+
((Boolean.valueOf(System.getProperty("ics.http.use_charles"))) ?
|
178
|
+
url.openConnection(DebugUtil.useCharles()) : url.openConnection());
|
179
|
+
|
180
|
+
// configure connection
|
181
|
+
connection.setDoOutput(true);
|
182
|
+
|
183
|
+
// NOTE: Uncommenting this (and not calling
|
184
|
+
// connection.getInputStream()) causes Java to hang without
|
185
|
+
// sending its payload.
|
186
|
+
|
187
|
+
// connection.setDoInput(false);
|
188
|
+
|
189
|
+
if (method.equals("DELETE")) {
|
190
|
+
connection.setRequestMethod("PUT");
|
191
|
+
connection.setRequestProperty("X-Method", "DELETE");
|
192
|
+
} else if (method.equals("PATCH")) {
|
193
|
+
connection.setRequestMethod("PUT");
|
194
|
+
connection.setRequestProperty("X-Method", "PATCH");
|
195
|
+
} else connection.setRequestMethod(method);
|
196
|
+
connection.setRequestProperty("Content-Type", "application/json");
|
197
|
+
connection.setRequestProperty("Accept", "*/*");
|
198
|
+
connection.setRequestProperty("Content-Length",
|
199
|
+
Integer.toString(body.getBytes().length));
|
200
|
+
connection.setUseCaches(false);
|
201
|
+
|
202
|
+
LOG.debug("sending: " + body);
|
203
|
+
LOG.debug("via " +
|
204
|
+
connection.getRequestMethod() +
|
205
|
+
" to " +
|
206
|
+
urlString(PATH_COMPONENT, topic, id));
|
207
|
+
|
208
|
+
// connect and write
|
209
|
+
OutputStream os = connection.getOutputStream();
|
210
|
+
os.write(body.getBytes("UTF-8"));
|
211
|
+
os.flush();
|
212
|
+
os.close();
|
213
|
+
|
214
|
+
// ignore reponse
|
215
|
+
InputStream is = connection.getInputStream();
|
216
|
+
|
217
|
+
LOG.trace("ignoring response from Vayacondios.");
|
218
|
+
byte buf[] = new byte[256];
|
219
|
+
while (is.read(buf) != -1);
|
220
|
+
LOG.trace("response ignored.");
|
221
|
+
is.close();
|
222
|
+
|
223
|
+
// fin.
|
224
|
+
connection.disconnect();
|
225
|
+
}
|
226
|
+
|
227
|
+
private Organization _org;
|
228
|
+
|
229
|
+
private static final JsonParser PARSER = new JsonParser();
|
230
|
+
private static final Logger LOG = getLogger();
|
231
|
+
private static final String PATH_COMPONENT = "itemset";
|
232
|
+
private static final Gson GSON = new GsonBuilder().
|
233
|
+
registerTypeAdapter(Item.class, new Item.Serializer()).
|
234
|
+
create();
|
235
|
+
|
236
|
+
//============================================================================
|
237
|
+
// Topic
|
238
|
+
//============================================================================
|
239
|
+
|
240
|
+
/**
|
241
|
+
* A Topic may have many ids, each of which points to a unique
|
242
|
+
* Vayacondios itemset.
|
243
|
+
*/
|
244
|
+
public static class Topic extends ItemSets {
|
245
|
+
public Topic(PathBuilder delegate) { super(delegate); }
|
246
|
+
|
247
|
+
public Topic(ItemSets sets, String topic) {
|
248
|
+
super(sets);
|
249
|
+
_sets = sets;
|
250
|
+
_topic = topic;
|
251
|
+
}
|
252
|
+
|
253
|
+
//--------------------------------------------------------------------------
|
254
|
+
// next in path hierarchy
|
255
|
+
//--------------------------------------------------------------------------
|
256
|
+
|
257
|
+
/**
|
258
|
+
* @param id Vayacondios id. see Vayacondios documentation for
|
259
|
+
* details.
|
260
|
+
* @return new itemset for this topic
|
261
|
+
*/
|
262
|
+
public ItemSet itemSet(String id) { return new ItemSet(this, id); }
|
263
|
+
|
264
|
+
//--------------------------------------------------------------------------
|
265
|
+
// public operations
|
266
|
+
//--------------------------------------------------------------------------
|
267
|
+
|
268
|
+
/**
|
269
|
+
* @see #ItemSets.fetch
|
270
|
+
*/
|
271
|
+
public List<Item> fetch(String id) throws IOException {
|
272
|
+
return fetch(getTopic(), id);
|
273
|
+
}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* @see #ItemSets.create
|
277
|
+
*/
|
278
|
+
public void create(List<Item> items, String id) throws IOException {
|
279
|
+
create(items, getTopic(), id);
|
280
|
+
}
|
281
|
+
|
282
|
+
/**
|
283
|
+
* @see #ItemSets.remove
|
284
|
+
*/
|
285
|
+
public void remove(List<Item> items, String id) throws IOException {
|
286
|
+
remove(items, getTopic(), id);
|
287
|
+
}
|
288
|
+
|
289
|
+
/**
|
290
|
+
* @see #ItemSets.update
|
291
|
+
*/
|
292
|
+
public void update(List<Item> items, String id) throws IOException {
|
293
|
+
update(items, getTopic(), id);
|
294
|
+
}
|
295
|
+
|
296
|
+
//--------------------------------------------------------------------------
|
297
|
+
// API HTTP path components
|
298
|
+
//--------------------------------------------------------------------------
|
299
|
+
|
300
|
+
protected String getTopic() { return _topic; }
|
301
|
+
|
302
|
+
//--------------------------------------------------------------------------
|
303
|
+
// fields
|
304
|
+
//--------------------------------------------------------------------------
|
305
|
+
|
306
|
+
private ItemSets _sets;
|
307
|
+
private String _topic;
|
308
|
+
}
|
309
|
+
|
310
|
+
//============================================================================
|
311
|
+
// ItemSet
|
312
|
+
//============================================================================
|
313
|
+
|
314
|
+
/**
|
315
|
+
* A Vayacodios Itemset is manipulated using four API methods
|
316
|
+
* implemented in terms of the Vayacondios API.
|
317
|
+
*/
|
318
|
+
public static class ItemSet extends Topic {
|
319
|
+
public ItemSet(PathBuilder delegate) { super(delegate); }
|
320
|
+
|
321
|
+
public ItemSet(Topic topic, String id) {
|
322
|
+
super(topic);
|
323
|
+
_topic = topic;
|
324
|
+
_id = id;
|
325
|
+
}
|
326
|
+
|
327
|
+
//--------------------------------------------------------------------------
|
328
|
+
// public operations
|
329
|
+
//--------------------------------------------------------------------------
|
330
|
+
|
331
|
+
/**
|
332
|
+
* @see ItemSets.fetch
|
333
|
+
*/
|
334
|
+
public List<Item> fetch() throws IOException {
|
335
|
+
return fetch(getId());
|
336
|
+
}
|
337
|
+
|
338
|
+
/**
|
339
|
+
* @see ItemSets.create
|
340
|
+
*/
|
341
|
+
public void create(List<Item> items) throws IOException {
|
342
|
+
create(items, getId());
|
343
|
+
}
|
344
|
+
|
345
|
+
/**
|
346
|
+
* @see ItemSets.remove
|
347
|
+
*/
|
348
|
+
public void remove(List<Item> items) throws IOException {
|
349
|
+
remove(items, getId());
|
350
|
+
}
|
351
|
+
|
352
|
+
/**
|
353
|
+
* @see ItemSets.update
|
354
|
+
*/
|
355
|
+
public void update(List<Item> items) throws IOException {
|
356
|
+
update(items, getId());
|
357
|
+
}
|
358
|
+
|
359
|
+
//--------------------------------------------------------------------------
|
360
|
+
// API HTTP path components
|
361
|
+
//--------------------------------------------------------------------------
|
362
|
+
|
363
|
+
protected String getId() {
|
364
|
+
return _id;
|
365
|
+
}
|
366
|
+
|
367
|
+
protected String getTopic() {
|
368
|
+
return ((Topic)getDelegate()).getTopic();
|
369
|
+
}
|
370
|
+
|
371
|
+
//--------------------------------------------------------------------------
|
372
|
+
// fields
|
373
|
+
//--------------------------------------------------------------------------
|
374
|
+
|
375
|
+
private Topic _topic;
|
376
|
+
private String _id;
|
377
|
+
}
|
378
|
+
|
379
|
+
//============================================================================
|
380
|
+
// Item
|
381
|
+
//============================================================================
|
382
|
+
|
383
|
+
/**
|
384
|
+
* A Vayacondios item can be a boolean, a number, or a string.
|
385
|
+
*/
|
386
|
+
public static class Item {
|
387
|
+
static class Serializer implements JsonSerializer {
|
388
|
+
public JsonElement serialize(Object item,
|
389
|
+
Type typeOfId,
|
390
|
+
JsonSerializationContext context) {
|
391
|
+
return GSON.toJsonTree(Item.class.isAssignableFrom(item.getClass()) ?
|
392
|
+
((Item)item).getObject() : item);
|
393
|
+
}
|
394
|
+
private static final Gson GSON = new Gson();
|
395
|
+
private static final Logger LOG = getLogger();
|
396
|
+
}
|
397
|
+
|
398
|
+
public Item(Boolean b) {
|
399
|
+
_item = b;
|
400
|
+
_type = TYPE.BOOLEAN;
|
401
|
+
}
|
402
|
+
|
403
|
+
public Item(Number n) {
|
404
|
+
_item = n;
|
405
|
+
_type = TYPE.NUMBER;
|
406
|
+
}
|
407
|
+
|
408
|
+
public Item(String s) {
|
409
|
+
_item = s;
|
410
|
+
_type = TYPE.STRING;
|
411
|
+
}
|
412
|
+
|
413
|
+
public Boolean getAsBoolean() {
|
414
|
+
if (_type != TYPE.BOOLEAN)
|
415
|
+
throw new ClassCastException("item is not a boolean");
|
416
|
+
return (Boolean)_item;
|
417
|
+
}
|
418
|
+
|
419
|
+
public Number getAsNumber() {
|
420
|
+
if (_type != TYPE.NUMBER)
|
421
|
+
throw new ClassCastException("item is not a number");
|
422
|
+
return (Number)_item;
|
423
|
+
}
|
424
|
+
|
425
|
+
public String getAsString() {
|
426
|
+
if (_type != TYPE.STRING)
|
427
|
+
throw new ClassCastException("item is not a string");
|
428
|
+
return (String)_item;
|
429
|
+
}
|
430
|
+
|
431
|
+
public Boolean isBoolean() { return (_type == TYPE.BOOLEAN); }
|
432
|
+
public Boolean isNumber() { return (_type == TYPE.NUMBER ); }
|
433
|
+
public Boolean isString() { return (_type == TYPE.STRING ); }
|
434
|
+
|
435
|
+
public Object getObject() { return _item; }
|
436
|
+
|
437
|
+
public TYPE getType() { return _type; }
|
438
|
+
|
439
|
+
public String toString() {
|
440
|
+
return _item.toString() + ":" + _type;
|
441
|
+
}
|
442
|
+
|
443
|
+
public boolean equals(Object other) {
|
444
|
+
return (Item.class.isAssignableFrom(other.getClass())) ?
|
445
|
+
_item.equals(((Item)other).getObject()) : _item.equals(other);
|
446
|
+
}
|
447
|
+
|
448
|
+
//--------------------------------------------------------------------------
|
449
|
+
// fields
|
450
|
+
//--------------------------------------------------------------------------
|
451
|
+
|
452
|
+
private Object _item;
|
453
|
+
private TYPE _type;
|
454
|
+
private enum TYPE {BOOLEAN, NUMBER, STRING}
|
455
|
+
}
|
456
|
+
}
|