vayacondios-server 0.1.2 → 0.1.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. data/Gemfile +1 -0
  2. data/app/http_shim.rb +1 -5
  3. data/lib/vayacondios-client.rb +2 -0
  4. data/lib/vayacondios-server.rb +1 -0
  5. data/lib/vayacondios/client/cube_client.rb +39 -0
  6. data/lib/vayacondios/client/itemset.rb +43 -28
  7. data/lib/vayacondios/client/notifier.rb +24 -1
  8. data/lib/vayacondios/client/zabbix_client.rb +148 -0
  9. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -1
  10. data/lib/vayacondios/server/model/itemset_document.rb +8 -4
  11. data/lib/vayacondios/server/rack/assume_json.rb +13 -0
  12. data/lib/vayacondios/server/rack/extract_methods.rb +11 -1
  13. data/lib/vayacondios/version.rb +1 -1
  14. data/pom.xml +97 -0
  15. data/scripts/hadoop_monitor/configurable.rb +1 -1
  16. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +6 -3
  17. data/scripts/hadoop_monitor/hadoop_client.rb +20 -19
  18. data/scripts/hadoop_monitor/hadoop_monitor.rb +3 -3
  19. data/scripts/hadoop_monitor/hadoopable.rb +3 -3
  20. data/scripts/hadoop_monitor/machine_monitor.rb +2 -2
  21. data/spec/client/itemset_spec.rb +8 -8
  22. data/spec/server/itemset_spec.rb +4 -4
  23. data/src/main/java/com/infochimps/util/CurrentClass.java +26 -0
  24. data/src/main/java/com/infochimps/util/DebugUtil.java +38 -0
  25. data/src/main/java/com/infochimps/util/HttpHelper.java +112 -0
  26. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +456 -0
  27. data/src/main/java/com/infochimps/vayacondios/Organization.java +49 -0
  28. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +13 -0
  29. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +68 -0
  30. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +88 -0
  31. data/vayacondios-client.gemspec +2 -2
  32. data/vayacondios-server.gemspec +4 -2
  33. metadata +37 -9
@@ -0,0 +1,456 @@
1
+ package com.infochimps.vayacondios;
2
+
3
+ import static com.infochimps.util.CurrentClass.getLogger;
4
+ import com.infochimps.util.DebugUtil;
5
+
6
+ import com.google.gson.Gson;
7
+ import com.google.gson.GsonBuilder;
8
+ import com.google.gson.JsonElement;
9
+ import com.google.gson.JsonIOException;
10
+ import com.google.gson.JsonObject;
11
+ import com.google.gson.JsonParser;
12
+ import com.google.gson.JsonPrimitive;
13
+ import com.google.gson.JsonParseException;
14
+ import com.google.gson.JsonSerializer;
15
+ import com.google.gson.JsonSerializationContext;
16
+ import com.google.gson.JsonSyntaxException;
17
+
18
+ import java.io.BufferedReader;
19
+ import java.util.HashMap;
20
+ import java.io.InputStream;
21
+ import java.io.OutputStream;
22
+ import java.io.OutputStreamWriter;
23
+ import java.io.IOException;
24
+ import java.lang.reflect.Type;
25
+ import java.net.HttpURLConnection;
26
+ import java.util.ArrayList;
27
+ import java.util.List;
28
+
29
+ import java.net.URL;
30
+
31
+ import org.slf4j.Logger;
32
+
33
+ /**
34
+ * This is the first level of the Vayacondios path-building hierarchy
35
+ * that is capable of directly manipulating itemsets.
36
+ */
37
+ public class ItemSets extends Organization {
38
+ public ItemSets(PathBuilder delegate) { super(delegate); }
39
+
40
+ public ItemSets(Organization org) {
41
+ super(org);
42
+ _org = org;
43
+ }
44
+
45
+ //----------------------------------------------------------------------------
46
+ // next in path hierarchy
47
+ //----------------------------------------------------------------------------
48
+
49
+ /**
50
+ * @param topic Vayacondios topic. see Vayacondios documentation for
51
+ * details.
52
+ * @return new Topic path builder for this organization
53
+ */
54
+ public Topic topic(String topic) { return new Topic(this, topic); }
55
+
56
+ //----------------------------------------------------------------------------
57
+ // public operations
58
+ //----------------------------------------------------------------------------
59
+
60
+ /**
61
+ * Fetches the current value of an itemset.
62
+ *
63
+ * @param topic A Vayacondios topic has many ids. See vayacondios
64
+ * documentation for further details.
65
+ * @param id A Vayacondios id, together with the server,
66
+ * organization, and topic, specifies a unique
67
+ * itemset.
68
+ * @return a collection of items
69
+ */
70
+ public List<Item> fetch(String topic, String id) throws IOException {
71
+ BufferedReader reader = openUrl(urlString(PATH_COMPONENT, topic, id));
72
+ String line = reader.readLine();
73
+ JsonElement response;
74
+ JsonElement itemSet;
75
+
76
+ ArrayList<Item> result = new ArrayList<Item>();
77
+
78
+ // assume Vayacondios response comes in a single line
79
+ if (line != null &&
80
+ (response = PARSER.parse(line)).isJsonObject() &&
81
+ (itemSet = (response.getAsJsonObject().get("contents"))).isJsonArray()) {
82
+ for (JsonElement elem : itemSet.getAsJsonArray()) {
83
+ if (!elem.isJsonPrimitive()) {
84
+ LOG.warn("ignoring non-primitive in itemset: " + elem);
85
+ continue;
86
+ }
87
+
88
+ JsonPrimitive item = elem.getAsJsonPrimitive();
89
+ if (item.isBoolean()) result.add(new Item(item.getAsBoolean()));
90
+ else if (item.isNumber()) result.add(new Item(item.getAsNumber()));
91
+ else if (item.isString()) result.add(new Item(item.getAsString()));
92
+
93
+ else LOG.warn("ignoring unrecognized type in itemset: " + item);
94
+ }
95
+ }
96
+
97
+ if ((line = reader.readLine()) != null)
98
+ LOG.warn("expected eof but saw " + line);
99
+
100
+ reader.close();
101
+
102
+ return result;
103
+ }
104
+
105
+ /**
106
+ * Creates a new itemset with the specified topic and id, clobbering
107
+ * any existing itemset with the same topic and id.
108
+ *
109
+ * @param items items whose existence should be ensured in the set.
110
+ * @param topic A Vayacondios topic has many ids. See vayacondios
111
+ * documentation for further details.
112
+ * @param id A Vayacondios id, together with the server,
113
+ * organization, and topic, specifies a unique
114
+ * itemset.
115
+ */
116
+ public void create(List<Item> items,
117
+ String topic,
118
+ String id) throws IOException {
119
+ mutate("PUT", items, topic, id);
120
+ }
121
+
122
+ /**
123
+ * Ensures the absence of the specified items from the specified itemset.q
124
+ *
125
+ * @param items items whose absence should be ensured in the set.
126
+ * @param topic A Vayacondios topic has many ids. See vayacondios
127
+ * documentation for further details.
128
+ * @param id A Vayacondios id, together with the server,
129
+ * organization, and topic, specifies a unique
130
+ * itemset.
131
+ */
132
+ public void remove(List<Item> items,
133
+ String topic,
134
+ String id) throws IOException {
135
+ mutate("DELETE", items, topic, id);
136
+ }
137
+
138
+ /**
139
+ * Updates the current value of an itemset, ensuring the existence
140
+ * of the specified items.
141
+ *
142
+ * @param items items whose existence should be ensured in the set.
143
+ * @param topic A Vayacondios topic has many ids. See vayacondios
144
+ * documentation for further details.
145
+ * @param id A Vayacondios id, together with the server,
146
+ * organization, and topic, specifies a unique
147
+ * itemset.
148
+ */
149
+ public void update(List<Item> items,
150
+ String topic,
151
+ String id) throws IOException {
152
+ mutate("PATCH", items, topic, id);
153
+ }
154
+
155
+ //----------------------------------------------------------------------------
156
+ // API HTTP path components
157
+ //----------------------------------------------------------------------------
158
+
159
+ String getOrganization() {
160
+ return ((Organization)getDelegate()).getOrganization();
161
+ }
162
+
163
+ //----------------------------------------------------------------------------
164
+
165
+ protected void mutate(String method,
166
+ List<Item> items,
167
+ String topic,
168
+ String id) throws IOException {
169
+
170
+ // serialize the items
171
+ HashMap content = new HashMap();
172
+ content.put("contents", items);
173
+ String body = GSON.toJson(content);
174
+ // connect to our standard path
175
+ URL url = new URL(urlString(PATH_COMPONENT, topic, id));
176
+ HttpURLConnection connection = (HttpURLConnection)
177
+ ((Boolean.valueOf(System.getProperty("ics.http.use_charles"))) ?
178
+ url.openConnection(DebugUtil.useCharles()) : url.openConnection());
179
+
180
+ // configure connection
181
+ connection.setDoOutput(true);
182
+
183
+ // NOTE: Uncommenting this (and not calling
184
+ // connection.getInputStream()) causes Java to hang without
185
+ // sending its payload.
186
+
187
+ // connection.setDoInput(false);
188
+
189
+ if (method.equals("DELETE")) {
190
+ connection.setRequestMethod("PUT");
191
+ connection.setRequestProperty("X-Method", "DELETE");
192
+ } else if (method.equals("PATCH")) {
193
+ connection.setRequestMethod("PUT");
194
+ connection.setRequestProperty("X-Method", "PATCH");
195
+ } else connection.setRequestMethod(method);
196
+ connection.setRequestProperty("Content-Type", "application/json");
197
+ connection.setRequestProperty("Accept", "*/*");
198
+ connection.setRequestProperty("Content-Length",
199
+ Integer.toString(body.getBytes().length));
200
+ connection.setUseCaches(false);
201
+
202
+ LOG.debug("sending: " + body);
203
+ LOG.debug("via " +
204
+ connection.getRequestMethod() +
205
+ " to " +
206
+ urlString(PATH_COMPONENT, topic, id));
207
+
208
+ // connect and write
209
+ OutputStream os = connection.getOutputStream();
210
+ os.write(body.getBytes("UTF-8"));
211
+ os.flush();
212
+ os.close();
213
+
214
+ // ignore reponse
215
+ InputStream is = connection.getInputStream();
216
+
217
+ LOG.trace("ignoring response from Vayacondios.");
218
+ byte buf[] = new byte[256];
219
+ while (is.read(buf) != -1);
220
+ LOG.trace("response ignored.");
221
+ is.close();
222
+
223
+ // fin.
224
+ connection.disconnect();
225
+ }
226
+
227
+ private Organization _org;
228
+
229
+ private static final JsonParser PARSER = new JsonParser();
230
+ private static final Logger LOG = getLogger();
231
+ private static final String PATH_COMPONENT = "itemset";
232
+ private static final Gson GSON = new GsonBuilder().
233
+ registerTypeAdapter(Item.class, new Item.Serializer()).
234
+ create();
235
+
236
+ //============================================================================
237
+ // Topic
238
+ //============================================================================
239
+
240
+ /**
241
+ * A Topic may have many ids, each of which points to a unique
242
+ * Vayacondios itemset.
243
+ */
244
+ public static class Topic extends ItemSets {
245
+ public Topic(PathBuilder delegate) { super(delegate); }
246
+
247
+ public Topic(ItemSets sets, String topic) {
248
+ super(sets);
249
+ _sets = sets;
250
+ _topic = topic;
251
+ }
252
+
253
+ //--------------------------------------------------------------------------
254
+ // next in path hierarchy
255
+ //--------------------------------------------------------------------------
256
+
257
+ /**
258
+ * @param id Vayacondios id. see Vayacondios documentation for
259
+ * details.
260
+ * @return new itemset for this topic
261
+ */
262
+ public ItemSet itemSet(String id) { return new ItemSet(this, id); }
263
+
264
+ //--------------------------------------------------------------------------
265
+ // public operations
266
+ //--------------------------------------------------------------------------
267
+
268
+ /**
269
+ * @see #ItemSets.fetch
270
+ */
271
+ public List<Item> fetch(String id) throws IOException {
272
+ return fetch(getTopic(), id);
273
+ }
274
+
275
+ /**
276
+ * @see #ItemSets.create
277
+ */
278
+ public void create(List<Item> items, String id) throws IOException {
279
+ create(items, getTopic(), id);
280
+ }
281
+
282
+ /**
283
+ * @see #ItemSets.remove
284
+ */
285
+ public void remove(List<Item> items, String id) throws IOException {
286
+ remove(items, getTopic(), id);
287
+ }
288
+
289
+ /**
290
+ * @see #ItemSets.update
291
+ */
292
+ public void update(List<Item> items, String id) throws IOException {
293
+ update(items, getTopic(), id);
294
+ }
295
+
296
+ //--------------------------------------------------------------------------
297
+ // API HTTP path components
298
+ //--------------------------------------------------------------------------
299
+
300
+ protected String getTopic() { return _topic; }
301
+
302
+ //--------------------------------------------------------------------------
303
+ // fields
304
+ //--------------------------------------------------------------------------
305
+
306
+ private ItemSets _sets;
307
+ private String _topic;
308
+ }
309
+
310
+ //============================================================================
311
+ // ItemSet
312
+ //============================================================================
313
+
314
+ /**
315
+ * A Vayacodios Itemset is manipulated using four API methods
316
+ * implemented in terms of the Vayacondios API.
317
+ */
318
+ public static class ItemSet extends Topic {
319
+ public ItemSet(PathBuilder delegate) { super(delegate); }
320
+
321
+ public ItemSet(Topic topic, String id) {
322
+ super(topic);
323
+ _topic = topic;
324
+ _id = id;
325
+ }
326
+
327
+ //--------------------------------------------------------------------------
328
+ // public operations
329
+ //--------------------------------------------------------------------------
330
+
331
+ /**
332
+ * @see ItemSets.fetch
333
+ */
334
+ public List<Item> fetch() throws IOException {
335
+ return fetch(getId());
336
+ }
337
+
338
+ /**
339
+ * @see ItemSets.create
340
+ */
341
+ public void create(List<Item> items) throws IOException {
342
+ create(items, getId());
343
+ }
344
+
345
+ /**
346
+ * @see ItemSets.remove
347
+ */
348
+ public void remove(List<Item> items) throws IOException {
349
+ remove(items, getId());
350
+ }
351
+
352
+ /**
353
+ * @see ItemSets.update
354
+ */
355
+ public void update(List<Item> items) throws IOException {
356
+ update(items, getId());
357
+ }
358
+
359
+ //--------------------------------------------------------------------------
360
+ // API HTTP path components
361
+ //--------------------------------------------------------------------------
362
+
363
+ protected String getId() {
364
+ return _id;
365
+ }
366
+
367
+ protected String getTopic() {
368
+ return ((Topic)getDelegate()).getTopic();
369
+ }
370
+
371
+ //--------------------------------------------------------------------------
372
+ // fields
373
+ //--------------------------------------------------------------------------
374
+
375
+ private Topic _topic;
376
+ private String _id;
377
+ }
378
+
379
+ //============================================================================
380
+ // Item
381
+ //============================================================================
382
+
383
+ /**
384
+ * A Vayacondios item can be a boolean, a number, or a string.
385
+ */
386
+ public static class Item {
387
+ static class Serializer implements JsonSerializer {
388
+ public JsonElement serialize(Object item,
389
+ Type typeOfId,
390
+ JsonSerializationContext context) {
391
+ return GSON.toJsonTree(Item.class.isAssignableFrom(item.getClass()) ?
392
+ ((Item)item).getObject() : item);
393
+ }
394
+ private static final Gson GSON = new Gson();
395
+ private static final Logger LOG = getLogger();
396
+ }
397
+
398
+ public Item(Boolean b) {
399
+ _item = b;
400
+ _type = TYPE.BOOLEAN;
401
+ }
402
+
403
+ public Item(Number n) {
404
+ _item = n;
405
+ _type = TYPE.NUMBER;
406
+ }
407
+
408
+ public Item(String s) {
409
+ _item = s;
410
+ _type = TYPE.STRING;
411
+ }
412
+
413
+ public Boolean getAsBoolean() {
414
+ if (_type != TYPE.BOOLEAN)
415
+ throw new ClassCastException("item is not a boolean");
416
+ return (Boolean)_item;
417
+ }
418
+
419
+ public Number getAsNumber() {
420
+ if (_type != TYPE.NUMBER)
421
+ throw new ClassCastException("item is not a number");
422
+ return (Number)_item;
423
+ }
424
+
425
+ public String getAsString() {
426
+ if (_type != TYPE.STRING)
427
+ throw new ClassCastException("item is not a string");
428
+ return (String)_item;
429
+ }
430
+
431
+ public Boolean isBoolean() { return (_type == TYPE.BOOLEAN); }
432
+ public Boolean isNumber() { return (_type == TYPE.NUMBER ); }
433
+ public Boolean isString() { return (_type == TYPE.STRING ); }
434
+
435
+ public Object getObject() { return _item; }
436
+
437
+ public TYPE getType() { return _type; }
438
+
439
+ public String toString() {
440
+ return _item.toString() + ":" + _type;
441
+ }
442
+
443
+ public boolean equals(Object other) {
444
+ return (Item.class.isAssignableFrom(other.getClass())) ?
445
+ _item.equals(((Item)other).getObject()) : _item.equals(other);
446
+ }
447
+
448
+ //--------------------------------------------------------------------------
449
+ // fields
450
+ //--------------------------------------------------------------------------
451
+
452
+ private Object _item;
453
+ private TYPE _type;
454
+ private enum TYPE {BOOLEAN, NUMBER, STRING}
455
+ }
456
+ }