vayacondios-server 0.1.2 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. data/Gemfile +1 -0
  2. data/app/http_shim.rb +1 -5
  3. data/lib/vayacondios-client.rb +2 -0
  4. data/lib/vayacondios-server.rb +1 -0
  5. data/lib/vayacondios/client/cube_client.rb +39 -0
  6. data/lib/vayacondios/client/itemset.rb +43 -28
  7. data/lib/vayacondios/client/notifier.rb +24 -1
  8. data/lib/vayacondios/client/zabbix_client.rb +148 -0
  9. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -1
  10. data/lib/vayacondios/server/model/itemset_document.rb +8 -4
  11. data/lib/vayacondios/server/rack/assume_json.rb +13 -0
  12. data/lib/vayacondios/server/rack/extract_methods.rb +11 -1
  13. data/lib/vayacondios/version.rb +1 -1
  14. data/pom.xml +97 -0
  15. data/scripts/hadoop_monitor/configurable.rb +1 -1
  16. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +6 -3
  17. data/scripts/hadoop_monitor/hadoop_client.rb +20 -19
  18. data/scripts/hadoop_monitor/hadoop_monitor.rb +3 -3
  19. data/scripts/hadoop_monitor/hadoopable.rb +3 -3
  20. data/scripts/hadoop_monitor/machine_monitor.rb +2 -2
  21. data/spec/client/itemset_spec.rb +8 -8
  22. data/spec/server/itemset_spec.rb +4 -4
  23. data/src/main/java/com/infochimps/util/CurrentClass.java +26 -0
  24. data/src/main/java/com/infochimps/util/DebugUtil.java +38 -0
  25. data/src/main/java/com/infochimps/util/HttpHelper.java +112 -0
  26. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +456 -0
  27. data/src/main/java/com/infochimps/vayacondios/Organization.java +49 -0
  28. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +13 -0
  29. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +68 -0
  30. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +88 -0
  31. data/vayacondios-client.gemspec +2 -2
  32. data/vayacondios-server.gemspec +4 -2
  33. metadata +37 -9
@@ -0,0 +1,456 @@
1
+ package com.infochimps.vayacondios;
2
+
3
+ import static com.infochimps.util.CurrentClass.getLogger;
4
+ import com.infochimps.util.DebugUtil;
5
+
6
+ import com.google.gson.Gson;
7
+ import com.google.gson.GsonBuilder;
8
+ import com.google.gson.JsonElement;
9
+ import com.google.gson.JsonIOException;
10
+ import com.google.gson.JsonObject;
11
+ import com.google.gson.JsonParser;
12
+ import com.google.gson.JsonPrimitive;
13
+ import com.google.gson.JsonParseException;
14
+ import com.google.gson.JsonSerializer;
15
+ import com.google.gson.JsonSerializationContext;
16
+ import com.google.gson.JsonSyntaxException;
17
+
18
+ import java.io.BufferedReader;
19
+ import java.util.HashMap;
20
+ import java.io.InputStream;
21
+ import java.io.OutputStream;
22
+ import java.io.OutputStreamWriter;
23
+ import java.io.IOException;
24
+ import java.lang.reflect.Type;
25
+ import java.net.HttpURLConnection;
26
+ import java.util.ArrayList;
27
+ import java.util.List;
28
+
29
+ import java.net.URL;
30
+
31
+ import org.slf4j.Logger;
32
+
33
+ /**
34
+ * This is the first level of the Vayacondios path-building hierarchy
35
+ * that is capable of directly manipulating itemsets.
36
+ */
37
+ public class ItemSets extends Organization {
38
+ public ItemSets(PathBuilder delegate) { super(delegate); }
39
+
40
+ public ItemSets(Organization org) {
41
+ super(org);
42
+ _org = org;
43
+ }
44
+
45
+ //----------------------------------------------------------------------------
46
+ // next in path hierarchy
47
+ //----------------------------------------------------------------------------
48
+
49
+ /**
50
+ * @param topic Vayacondios topic. see Vayacondios documentation for
51
+ * details.
52
+ * @return new Topic path builder for this organization
53
+ */
54
+ public Topic topic(String topic) { return new Topic(this, topic); }
55
+
56
+ //----------------------------------------------------------------------------
57
+ // public operations
58
+ //----------------------------------------------------------------------------
59
+
60
+ /**
61
+ * Fetches the current value of an itemset.
62
+ *
63
+ * @param topic A Vayacondios topic has many ids. See vayacondios
64
+ * documentation for further details.
65
+ * @param id A Vayacondios id, together with the server,
66
+ * organization, and topic, specifies a unique
67
+ * itemset.
68
+ * @return a collection of items
69
+ */
70
+ public List<Item> fetch(String topic, String id) throws IOException {
71
+ BufferedReader reader = openUrl(urlString(PATH_COMPONENT, topic, id));
72
+ String line = reader.readLine();
73
+ JsonElement response;
74
+ JsonElement itemSet;
75
+
76
+ ArrayList<Item> result = new ArrayList<Item>();
77
+
78
+ // assume Vayacondios response comes in a single line
79
+ if (line != null &&
80
+ (response = PARSER.parse(line)).isJsonObject() &&
81
+ (itemSet = (response.getAsJsonObject().get("contents"))).isJsonArray()) {
82
+ for (JsonElement elem : itemSet.getAsJsonArray()) {
83
+ if (!elem.isJsonPrimitive()) {
84
+ LOG.warn("ignoring non-primitive in itemset: " + elem);
85
+ continue;
86
+ }
87
+
88
+ JsonPrimitive item = elem.getAsJsonPrimitive();
89
+ if (item.isBoolean()) result.add(new Item(item.getAsBoolean()));
90
+ else if (item.isNumber()) result.add(new Item(item.getAsNumber()));
91
+ else if (item.isString()) result.add(new Item(item.getAsString()));
92
+
93
+ else LOG.warn("ignoring unrecognized type in itemset: " + item);
94
+ }
95
+ }
96
+
97
+ if ((line = reader.readLine()) != null)
98
+ LOG.warn("expected eof but saw " + line);
99
+
100
+ reader.close();
101
+
102
+ return result;
103
+ }
104
+
105
+ /**
106
+ * Creates a new itemset with the specified topic and id, clobbering
107
+ * any existing itemset with the same topic and id.
108
+ *
109
+ * @param items items whose existence should be ensured in the set.
110
+ * @param topic A Vayacondios topic has many ids. See vayacondios
111
+ * documentation for further details.
112
+ * @param id A Vayacondios id, together with the server,
113
+ * organization, and topic, specifies a unique
114
+ * itemset.
115
+ */
116
+ public void create(List<Item> items,
117
+ String topic,
118
+ String id) throws IOException {
119
+ mutate("PUT", items, topic, id);
120
+ }
121
+
122
+ /**
123
+ * Ensures the absence of the specified items from the specified itemset.q
124
+ *
125
+ * @param items items whose absence should be ensured in the set.
126
+ * @param topic A Vayacondios topic has many ids. See vayacondios
127
+ * documentation for further details.
128
+ * @param id A Vayacondios id, together with the server,
129
+ * organization, and topic, specifies a unique
130
+ * itemset.
131
+ */
132
+ public void remove(List<Item> items,
133
+ String topic,
134
+ String id) throws IOException {
135
+ mutate("DELETE", items, topic, id);
136
+ }
137
+
138
+ /**
139
+ * Updates the current value of an itemset, ensuring the existence
140
+ * of the specified items.
141
+ *
142
+ * @param items items whose existence should be ensured in the set.
143
+ * @param topic A Vayacondios topic has many ids. See vayacondios
144
+ * documentation for further details.
145
+ * @param id A Vayacondios id, together with the server,
146
+ * organization, and topic, specifies a unique
147
+ * itemset.
148
+ */
149
+ public void update(List<Item> items,
150
+ String topic,
151
+ String id) throws IOException {
152
+ mutate("PATCH", items, topic, id);
153
+ }
154
+
155
+ //----------------------------------------------------------------------------
156
+ // API HTTP path components
157
+ //----------------------------------------------------------------------------
158
+
159
+ String getOrganization() {
160
+ return ((Organization)getDelegate()).getOrganization();
161
+ }
162
+
163
+ //----------------------------------------------------------------------------
164
+
165
+ protected void mutate(String method,
166
+ List<Item> items,
167
+ String topic,
168
+ String id) throws IOException {
169
+
170
+ // serialize the items
171
+ HashMap content = new HashMap();
172
+ content.put("contents", items);
173
+ String body = GSON.toJson(content);
174
+ // connect to our standard path
175
+ URL url = new URL(urlString(PATH_COMPONENT, topic, id));
176
+ HttpURLConnection connection = (HttpURLConnection)
177
+ ((Boolean.valueOf(System.getProperty("ics.http.use_charles"))) ?
178
+ url.openConnection(DebugUtil.useCharles()) : url.openConnection());
179
+
180
+ // configure connection
181
+ connection.setDoOutput(true);
182
+
183
+ // NOTE: Uncommenting this (and not calling
184
+ // connection.getInputStream()) causes Java to hang without
185
+ // sending its payload.
186
+
187
+ // connection.setDoInput(false);
188
+
189
+ if (method.equals("DELETE")) {
190
+ connection.setRequestMethod("PUT");
191
+ connection.setRequestProperty("X-Method", "DELETE");
192
+ } else if (method.equals("PATCH")) {
193
+ connection.setRequestMethod("PUT");
194
+ connection.setRequestProperty("X-Method", "PATCH");
195
+ } else connection.setRequestMethod(method);
196
+ connection.setRequestProperty("Content-Type", "application/json");
197
+ connection.setRequestProperty("Accept", "*/*");
198
+ connection.setRequestProperty("Content-Length",
199
+ Integer.toString(body.getBytes().length));
200
+ connection.setUseCaches(false);
201
+
202
+ LOG.debug("sending: " + body);
203
+ LOG.debug("via " +
204
+ connection.getRequestMethod() +
205
+ " to " +
206
+ urlString(PATH_COMPONENT, topic, id));
207
+
208
+ // connect and write
209
+ OutputStream os = connection.getOutputStream();
210
+ os.write(body.getBytes("UTF-8"));
211
+ os.flush();
212
+ os.close();
213
+
214
+ // ignore reponse
215
+ InputStream is = connection.getInputStream();
216
+
217
+ LOG.trace("ignoring response from Vayacondios.");
218
+ byte buf[] = new byte[256];
219
+ while (is.read(buf) != -1);
220
+ LOG.trace("response ignored.");
221
+ is.close();
222
+
223
+ // fin.
224
+ connection.disconnect();
225
+ }
226
+
227
+ private Organization _org;
228
+
229
+ private static final JsonParser PARSER = new JsonParser();
230
+ private static final Logger LOG = getLogger();
231
+ private static final String PATH_COMPONENT = "itemset";
232
+ private static final Gson GSON = new GsonBuilder().
233
+ registerTypeAdapter(Item.class, new Item.Serializer()).
234
+ create();
235
+
236
+ //============================================================================
237
+ // Topic
238
+ //============================================================================
239
+
240
+ /**
241
+ * A Topic may have many ids, each of which points to a unique
242
+ * Vayacondios itemset.
243
+ */
244
+ public static class Topic extends ItemSets {
245
+ public Topic(PathBuilder delegate) { super(delegate); }
246
+
247
+ public Topic(ItemSets sets, String topic) {
248
+ super(sets);
249
+ _sets = sets;
250
+ _topic = topic;
251
+ }
252
+
253
+ //--------------------------------------------------------------------------
254
+ // next in path hierarchy
255
+ //--------------------------------------------------------------------------
256
+
257
+ /**
258
+ * @param id Vayacondios id. see Vayacondios documentation for
259
+ * details.
260
+ * @return new itemset for this topic
261
+ */
262
+ public ItemSet itemSet(String id) { return new ItemSet(this, id); }
263
+
264
+ //--------------------------------------------------------------------------
265
+ // public operations
266
+ //--------------------------------------------------------------------------
267
+
268
+ /**
269
+ * @see #ItemSets.fetch
270
+ */
271
+ public List<Item> fetch(String id) throws IOException {
272
+ return fetch(getTopic(), id);
273
+ }
274
+
275
+ /**
276
+ * @see #ItemSets.create
277
+ */
278
+ public void create(List<Item> items, String id) throws IOException {
279
+ create(items, getTopic(), id);
280
+ }
281
+
282
+ /**
283
+ * @see #ItemSets.remove
284
+ */
285
+ public void remove(List<Item> items, String id) throws IOException {
286
+ remove(items, getTopic(), id);
287
+ }
288
+
289
+ /**
290
+ * @see #ItemSets.update
291
+ */
292
+ public void update(List<Item> items, String id) throws IOException {
293
+ update(items, getTopic(), id);
294
+ }
295
+
296
+ //--------------------------------------------------------------------------
297
+ // API HTTP path components
298
+ //--------------------------------------------------------------------------
299
+
300
+ protected String getTopic() { return _topic; }
301
+
302
+ //--------------------------------------------------------------------------
303
+ // fields
304
+ //--------------------------------------------------------------------------
305
+
306
+ private ItemSets _sets;
307
+ private String _topic;
308
+ }
309
+
310
+ //============================================================================
311
+ // ItemSet
312
+ //============================================================================
313
+
314
+ /**
315
+ * A Vayacodios Itemset is manipulated using four API methods
316
+ * implemented in terms of the Vayacondios API.
317
+ */
318
+ public static class ItemSet extends Topic {
319
+ public ItemSet(PathBuilder delegate) { super(delegate); }
320
+
321
+ public ItemSet(Topic topic, String id) {
322
+ super(topic);
323
+ _topic = topic;
324
+ _id = id;
325
+ }
326
+
327
+ //--------------------------------------------------------------------------
328
+ // public operations
329
+ //--------------------------------------------------------------------------
330
+
331
+ /**
332
+ * @see ItemSets.fetch
333
+ */
334
+ public List<Item> fetch() throws IOException {
335
+ return fetch(getId());
336
+ }
337
+
338
+ /**
339
+ * @see ItemSets.create
340
+ */
341
+ public void create(List<Item> items) throws IOException {
342
+ create(items, getId());
343
+ }
344
+
345
+ /**
346
+ * @see ItemSets.remove
347
+ */
348
+ public void remove(List<Item> items) throws IOException {
349
+ remove(items, getId());
350
+ }
351
+
352
+ /**
353
+ * @see ItemSets.update
354
+ */
355
+ public void update(List<Item> items) throws IOException {
356
+ update(items, getId());
357
+ }
358
+
359
+ //--------------------------------------------------------------------------
360
+ // API HTTP path components
361
+ //--------------------------------------------------------------------------
362
+
363
+ protected String getId() {
364
+ return _id;
365
+ }
366
+
367
+ protected String getTopic() {
368
+ return ((Topic)getDelegate()).getTopic();
369
+ }
370
+
371
+ //--------------------------------------------------------------------------
372
+ // fields
373
+ //--------------------------------------------------------------------------
374
+
375
+ private Topic _topic;
376
+ private String _id;
377
+ }
378
+
379
+ //============================================================================
380
+ // Item
381
+ //============================================================================
382
+
383
+ /**
384
+ * A Vayacondios item can be a boolean, a number, or a string.
385
+ */
386
+ public static class Item {
387
+ static class Serializer implements JsonSerializer {
388
+ public JsonElement serialize(Object item,
389
+ Type typeOfId,
390
+ JsonSerializationContext context) {
391
+ return GSON.toJsonTree(Item.class.isAssignableFrom(item.getClass()) ?
392
+ ((Item)item).getObject() : item);
393
+ }
394
+ private static final Gson GSON = new Gson();
395
+ private static final Logger LOG = getLogger();
396
+ }
397
+
398
+ public Item(Boolean b) {
399
+ _item = b;
400
+ _type = TYPE.BOOLEAN;
401
+ }
402
+
403
+ public Item(Number n) {
404
+ _item = n;
405
+ _type = TYPE.NUMBER;
406
+ }
407
+
408
+ public Item(String s) {
409
+ _item = s;
410
+ _type = TYPE.STRING;
411
+ }
412
+
413
+ public Boolean getAsBoolean() {
414
+ if (_type != TYPE.BOOLEAN)
415
+ throw new ClassCastException("item is not a boolean");
416
+ return (Boolean)_item;
417
+ }
418
+
419
+ public Number getAsNumber() {
420
+ if (_type != TYPE.NUMBER)
421
+ throw new ClassCastException("item is not a number");
422
+ return (Number)_item;
423
+ }
424
+
425
+ public String getAsString() {
426
+ if (_type != TYPE.STRING)
427
+ throw new ClassCastException("item is not a string");
428
+ return (String)_item;
429
+ }
430
+
431
+ public Boolean isBoolean() { return (_type == TYPE.BOOLEAN); }
432
+ public Boolean isNumber() { return (_type == TYPE.NUMBER ); }
433
+ public Boolean isString() { return (_type == TYPE.STRING ); }
434
+
435
+ public Object getObject() { return _item; }
436
+
437
+ public TYPE getType() { return _type; }
438
+
439
+ public String toString() {
440
+ return _item.toString() + ":" + _type;
441
+ }
442
+
443
+ public boolean equals(Object other) {
444
+ return (Item.class.isAssignableFrom(other.getClass())) ?
445
+ _item.equals(((Item)other).getObject()) : _item.equals(other);
446
+ }
447
+
448
+ //--------------------------------------------------------------------------
449
+ // fields
450
+ //--------------------------------------------------------------------------
451
+
452
+ private Object _item;
453
+ private TYPE _type;
454
+ private enum TYPE {BOOLEAN, NUMBER, STRING}
455
+ }
456
+ }