vayacondios-server 0.2.11 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. data/.gitignore +3 -1
  2. data/.travis.yml +2 -0
  3. data/Gemfile +15 -9
  4. data/LICENSE.md +2 -6
  5. data/Procfile +1 -1
  6. data/README.md +656 -111
  7. data/Rakefile +89 -6
  8. data/bin/vcd +10 -0
  9. data/bin/vcd-server +8 -0
  10. data/config/database.yml +6 -0
  11. data/config/spec.example.yml +18 -0
  12. data/config/vayacondios.example.yml +15 -0
  13. data/config/vcd-server.rb +37 -0
  14. data/examples/configuration.rb +56 -0
  15. data/examples/event_stream.rb +19 -0
  16. data/examples/simple.rb +61 -0
  17. data/features/event.feature +319 -0
  18. data/features/events.feature +208 -0
  19. data/features/stash.feature +840 -0
  20. data/features/stashes.feature +492 -0
  21. data/features/step_definitions/stash_steps.rb +113 -0
  22. data/features/stream.feature +30 -0
  23. data/features/support/em.rb +14 -0
  24. data/features/support/env.rb +13 -0
  25. data/lib/vayacondios/configuration.rb +63 -0
  26. data/lib/vayacondios/server/api.rb +126 -0
  27. data/lib/vayacondios/server/api_options.rb +56 -0
  28. data/lib/vayacondios/server/configuration.rb +23 -0
  29. data/lib/vayacondios/server/driver.rb +71 -0
  30. data/lib/vayacondios/server/drivers/mongo.rb +126 -0
  31. data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
  32. data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
  33. data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
  34. data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
  35. data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
  36. data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
  37. data/lib/vayacondios/server/models/document.rb +87 -0
  38. data/lib/vayacondios/server/models/event.rb +198 -0
  39. data/lib/vayacondios/server/models/stash.rb +100 -0
  40. data/lib/vayacondios/server.rb +35 -0
  41. data/lib/vayacondios-server.rb +19 -13
  42. data/lib/vayacondios.rb +22 -0
  43. data/pom.xml +124 -4
  44. data/spec/configuration_spec.rb +41 -0
  45. data/spec/server/api_options_spec.rb +32 -0
  46. data/spec/server/api_spec.rb +279 -0
  47. data/spec/server/configuration_spec.rb +27 -0
  48. data/spec/server/drivers/mongo_spec.rb +107 -0
  49. data/spec/server/handlers/event_handler_spec.rb +62 -0
  50. data/spec/server/handlers/events_handler_spec.rb +51 -0
  51. data/spec/server/handlers/stash_handler_spec.rb +68 -0
  52. data/spec/server/handlers/stashes_handler_spec.rb +50 -0
  53. data/spec/server/handlers/stream_handler_spec.rb +5 -0
  54. data/spec/server/models/document_spec.rb +9 -0
  55. data/spec/server/models/event_spec.rb +185 -0
  56. data/spec/server/models/stash_spec.rb +95 -0
  57. data/spec/spec_helper.rb +23 -3
  58. data/spec/support/database_helper.rb +42 -0
  59. data/spec/support/log_helper.rb +19 -0
  60. data/spec/support/shared_context_for_events.rb +22 -0
  61. data/spec/support/shared_context_for_stashes.rb +24 -0
  62. data/spec/support/shared_examples_for_handlers.rb +32 -0
  63. data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
  64. data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
  65. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
  66. data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
  67. data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
  68. data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
  69. data/vayacondios-server.gemspec +9 -9
  70. metadata +127 -122
  71. checksums.yaml +0 -15
  72. data/.rspec +0 -2
  73. data/.yardopts +0 -10
  74. data/Guardfile +0 -41
  75. data/app/http_shim.rb +0 -71
  76. data/bin/vcd.sh +0 -27
  77. data/config/http_shim.rb +0 -43
  78. data/config/vayacondios.example.yaml +0 -7
  79. data/config/vayacondios.yaml +0 -7
  80. data/examples/java/ItemSetTest.java +0 -76
  81. data/lib/tasks/publish.rake +0 -23
  82. data/lib/tasks/spec.rake +0 -11
  83. data/lib/tasks/yard.rake +0 -2
  84. data/lib/vayacondios/client/config.rb +0 -7
  85. data/lib/vayacondios/client/configliere.rb +0 -38
  86. data/lib/vayacondios/client/cube_client.rb +0 -39
  87. data/lib/vayacondios/client/http_client.rb +0 -49
  88. data/lib/vayacondios/client/itemset.rb +0 -130
  89. data/lib/vayacondios/client/legacy_switch.rb +0 -43
  90. data/lib/vayacondios/client/notifier.rb +0 -123
  91. data/lib/vayacondios/client/zabbix_client.rb +0 -148
  92. data/lib/vayacondios/legacy_switch.rb +0 -43
  93. data/lib/vayacondios/server/errors/bad_request.rb +0 -6
  94. data/lib/vayacondios/server/errors/not_found.rb +0 -6
  95. data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
  96. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
  97. data/lib/vayacondios/server/legacy_switch.rb +0 -43
  98. data/lib/vayacondios/server/model/config_document.rb +0 -89
  99. data/lib/vayacondios/server/model/document.rb +0 -25
  100. data/lib/vayacondios/server/model/event_document.rb +0 -94
  101. data/lib/vayacondios/server/model/itemset_document.rb +0 -126
  102. data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
  103. data/lib/vayacondios/server/rack/jsonize.rb +0 -43
  104. data/lib/vayacondios/server/rack/params.rb +0 -50
  105. data/lib/vayacondios/server/rack/path.rb +0 -23
  106. data/lib/vayacondios/server/rack/path_validation.rb +0 -22
  107. data/lib/vayacondios/version.rb +0 -3
  108. data/lib/vayacondios-client.rb +0 -22
  109. data/scripts/hadoop_monitor/configurable.rb +0 -66
  110. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
  111. data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
  112. data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
  113. data/scripts/hadoop_monitor/hadoopable.rb +0 -65
  114. data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
  115. data/scripts/s3_cataloger/buckets +0 -33
  116. data/scripts/s3_cataloger/foreach_bucket +0 -88
  117. data/scripts/s3_cataloger/parse_ls.py +0 -391
  118. data/spec/client/itemset_legacy_spec.rb +0 -55
  119. data/spec/client/itemset_spec.rb +0 -60
  120. data/spec/client/notifier_spec.rb +0 -120
  121. data/spec/server/config_spec.rb +0 -113
  122. data/spec/server/event_spec.rb +0 -103
  123. data/spec/server/itemset_legacy_spec.rb +0 -320
  124. data/spec/server/itemset_spec.rb +0 -317
  125. data/spec/server/rack/extract_methods_spec.rb +0 -60
  126. data/spec/server/rack/path_spec.rb +0 -36
  127. data/spec/server/rack/path_validation_spec.rb +0 -22
  128. data/spec/server/server_spec.rb +0 -20
  129. data/spec/support/mongo_cleaner.rb +0 -32
  130. data/src/main/java/ItemSetTest.java +0 -76
  131. data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
  132. data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
  133. data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
  134. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
  135. data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
  136. data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
  137. data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
  138. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
  139. data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
  140. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
  141. data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
  142. data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,342 @@
1
+ package com.infochimps.vayacondios;
2
+
3
+ import java.util.Map;
4
+ import java.util.List;
5
+ import java.io.IOException;
6
+
7
+ import org.slf4j.Logger;
8
+ import org.slf4j.LoggerFactory;
9
+
10
+ /** Base class for all concrete implementations of a Vayacondios client.
11
+ * <p>
12
+ * Is fixed to a specific organization at instantiation time since the
13
+ * common use case is reading/writing data for a single organization
14
+ * at a time. If reading/writing data for multiple organizations
15
+ * simultaneously, use multiple client instances.
16
+ * <p>
17
+ * Can be instantiated in "dry-run" mode in which it will not actually
18
+ * make any requests to the server, merely log its output.
19
+ * <p>
20
+ * Implements the {@link VayacondiosClient} interface by wrapping each
21
+ * method of is API (e.g. - {@link VayacondiosClient#announce(String
22
+ * topic, Map event)}) with logic for handling arguments, errors, and
23
+ * "dry-run" mode, ultimately delegating to a (protected) method to
24
+ * perform the actual request (e.g. - {@link
25
+ * BaseClient#performAnnounce(String topic, Map event)}). Subclasses
26
+ * should override these methods to provide a concrete implementation
27
+ * of a Vayacondios client.
28
+ */
29
+ public class BaseClient implements VayacondiosClient {
30
+
31
+ //----------------------------------------------------------------------------
32
+ // Initialization & Properties
33
+ //----------------------------------------------------------------------------
34
+
35
+ public static String VERSION = "v2";
36
+
37
+ private static Logger LOG = LoggerFactory.getLogger(BaseClient.class);
38
+
39
+ private String _organization;
40
+ private Boolean _dryRun;
41
+
42
+ /**
43
+ * Create a new BaseClient instance for the given
44
+ * organization. optionally in "dry-run" mode.
45
+ *
46
+ * <blockquote><pre>{@code
47
+ * VayacondiosClient client = new BaseClient("my_organization", true); // client in dry-run mode
48
+ * }</pre></blockquote>
49
+ *
50
+ * @param organization name of the organization to read/write data for
51
+ * @param shouldDryRun whether or not to enter "dry-run" mode
52
+ */
53
+ public BaseClient(String organization, Boolean shouldDryRun) {
54
+ this._organization = organization;
55
+ this._dryRun = shouldDryRun;
56
+ }
57
+
58
+ /**
59
+ * Create a new BaseClient instance for the given organization.
60
+ *
61
+ * @param organization name of the organization to read/write data for
62
+ */
63
+ public BaseClient(String organization) {
64
+ this(organization, false);
65
+ }
66
+
67
+ /**
68
+ * Is this client in "dry-run" mode?
69
+ * <p>
70
+ * When in "dry-run" mode, the client will not actually make any
71
+ * requests to the server, it will merely log all its attempts at
72
+ * an elevated level.
73
+ * @return whether or not the client is in "dry-run" mode
74
+ */
75
+ public Boolean dryRun() {
76
+ return _dryRun;
77
+ }
78
+
79
+ /**
80
+ * The organization this client will read/write data for.
81
+ *
82
+ * @return name of the organization
83
+ */
84
+ public String organization() {
85
+ return _organization;
86
+ }
87
+
88
+ //----------------------------------------------------------------------------
89
+ // Public API
90
+ //----------------------------------------------------------------------------
91
+
92
+ /**
93
+ * {@inheritDoc}
94
+ */
95
+ @Override
96
+ public void announce(String topic, Map<String,Object> event) {
97
+ logRequest("Announcing <" + topic + ">");
98
+ if (dryRun()) return;
99
+ try {
100
+ performAnnounce(topic, event);
101
+ } catch (IOException e) {
102
+ LOG.error("Announcing <" + topic + ">", e);
103
+ }
104
+ }
105
+ /**
106
+ * {@inheritDoc}
107
+ */
108
+ @Override
109
+ public void announce(String topic, Map<String,Object> event, String id) {
110
+ logRequest("Announcing <" + topic + "/" + id + ">");
111
+ if (dryRun()) return;
112
+ try {
113
+ performAnnounce(topic, event, id);
114
+ } catch (IOException e) {
115
+ LOG.error("Announcing <" + topic + "/" + id + ">", e);
116
+ }
117
+ }
118
+
119
+ /**
120
+ * {@inheritDoc}
121
+ */
122
+ @Override
123
+ public List<Map<String,Object>> events(String topic, Map<String,Object> query) {
124
+ logRequest("Searching events <" + topic + ">");
125
+ if (dryRun()) return null;
126
+ try {
127
+ return performEvents(topic, query);
128
+ } catch (IOException e) {
129
+ LOG.error("Searching events <" + topic + ">", e);
130
+ return null;
131
+ }
132
+ }
133
+
134
+ /**
135
+ * {@inheritDoc}
136
+ */
137
+ @Override
138
+ public Map<String,Object> get(String topic) {
139
+ logRequest("Fetching <" + topic + ">");
140
+ if (dryRun()) return null;
141
+ try {
142
+ return performGet(topic);
143
+ } catch (IOException e) {
144
+ LOG.error("Fetching <" + topic + ">");
145
+ return null;
146
+ }
147
+ }
148
+
149
+ /**
150
+ * {@inheritDoc}
151
+ */
152
+ @Override
153
+ public Map<String,Object> getMap(String topic, String id) {
154
+ logRequest("Fetching Map <" + topic + "/" + id + ">");
155
+ if (dryRun()) return null;
156
+ try {
157
+ return performGetMap(topic, id);
158
+ } catch (IOException e) {
159
+ LOG.error("Fetching Map <" + topic + "/" + id + ">", e);
160
+ return null;
161
+ }
162
+ }
163
+ /**
164
+ * {@inheritDoc}
165
+ */
166
+ @Override
167
+ public List getList(String topic, String id) {
168
+ logRequest("Fetching List <" + topic + "/" + id + ">");
169
+ if (dryRun()) return null;
170
+ try {
171
+ return performGetList(topic, id);
172
+ } catch (IOException e) {
173
+ LOG.error("Fetching List <" + topic + "/" + id + ">", e);
174
+ return null;
175
+ }
176
+ }
177
+ /**
178
+ * {@inheritDoc}
179
+ */
180
+ @Override
181
+ public String getString(String topic, String id) {
182
+ logRequest("Fetching String <" + topic + "/" + id + ">");
183
+ if (dryRun()) return null;
184
+ try {
185
+ return performGetString(topic, id);
186
+ } catch (IOException e) {
187
+ LOG.error("Fetching String <" + topic + "/" + id + ">", e);
188
+ return null;
189
+ }
190
+ }
191
+ /**
192
+ * {@inheritDoc}
193
+ */
194
+ @Override
195
+ public Double getDouble(String topic, String id) {
196
+ logRequest("Fetching Double <" + topic + "/" + id + ">");
197
+ if (dryRun()) return null;
198
+ try {
199
+ return performGetDouble(topic, id);
200
+ } catch (IOException e) {
201
+ LOG.error("Fetching Double <" + topic + "/" + id + ">", e);
202
+ return null;
203
+ }
204
+ }
205
+
206
+ /**
207
+ * {@inheritDoc}
208
+ */
209
+ @Override
210
+ public List<Map<String,Object>> stashes(Map<String,Object> query) {
211
+ logRequest("Searching stashes");
212
+ if (dryRun()) return null;
213
+ try {
214
+ return performStashes(query);
215
+ } catch (IOException e) {
216
+ LOG.error("Searching stashes", e);
217
+ return null;
218
+ }
219
+ }
220
+
221
+ /**
222
+ * {@inheritDoc}
223
+ */
224
+ @Override
225
+ public void set(String topic, Map<String,Object> value) {
226
+ logRequest("Replacing <" + topic + ">");
227
+ if (dryRun()) return;
228
+ try {
229
+ performSet(topic, value);
230
+ } catch (IOException e) {
231
+ LOG.error("Replacing <" + topic + ">", e);
232
+ }
233
+ }
234
+ /**
235
+ * {@inheritDoc}
236
+ */
237
+ @Override
238
+ public void set(String topic, String id, Object value) {
239
+ logRequest("Replacing <" + topic + "/" + id + ">");
240
+ if (dryRun()) return;
241
+ try {
242
+ performSet(topic, id, value);
243
+ } catch (IOException e) {
244
+ LOG.error("Replacing <" + topic + "/" + id + ">", e);
245
+ }
246
+ }
247
+
248
+ /**
249
+ * {@inheritDoc}
250
+ */
251
+ @Override
252
+ public void merge(String topic, Map<String,Object> value) {
253
+ logRequest("Merging <" + topic + ">");
254
+ if (dryRun()) return;
255
+ try {
256
+ performMerge(topic, value);
257
+ } catch (IOException e) {
258
+ LOG.error("Merging <" + topic + ">", e);
259
+ }
260
+ }
261
+ /**
262
+ * {@inheritDoc}
263
+ */
264
+ @Override
265
+ public void merge(String topic, String id, Object value) {
266
+ logRequest("Merging <" + topic + "/" + id + ">");
267
+ if (dryRun()) return;
268
+ try {
269
+ performMerge(topic, id, value);
270
+ } catch (IOException e) {
271
+ LOG.error("Merging <" + topic + "/" + id + ">", e);
272
+ }
273
+ }
274
+
275
+ /**
276
+ * {@inheritDoc}
277
+ */
278
+ @Override
279
+ public void delete(String topic) {
280
+ logRequest("Deleting <" + topic + ">");
281
+ if (dryRun()) return;
282
+ try {
283
+ performDelete(topic);
284
+ } catch (IOException e) {
285
+ LOG.error("Deleting <" + topic + ">");
286
+ }
287
+ }
288
+ /**
289
+ * {@inheritDoc}
290
+ */
291
+ @Override
292
+ public void delete(String topic, String id) {
293
+ logRequest("Deleting <" + topic + "/" + id + ">");
294
+ if (dryRun()) return;
295
+ try {
296
+ performDelete(topic, id);
297
+ } catch (IOException e) {
298
+ LOG.error("Deleting <" + topic + "/" + id + ">");
299
+ }
300
+ }
301
+
302
+ /**
303
+ * {@inheritDoc}
304
+ */
305
+ @Override
306
+ public void close() {};
307
+
308
+ //----------------------------------------------------------------------------
309
+ // Private API
310
+ //----------------------------------------------------------------------------
311
+
312
+ protected void performAnnounce(String topic, Map<String,Object> event, String id) throws IOException {}
313
+ protected void performAnnounce(String topic, Map<String,Object> event) throws IOException {}
314
+
315
+ protected List<Map<String,Object>> performEvents(String topic, Map<String,Object> query) throws IOException { return null; }
316
+
317
+ protected Map<String,Object> performGet(String topic) throws IOException { return null; }
318
+
319
+ protected Map<String,Object> performGetMap(String topic, String id) throws IOException { return null; }
320
+ protected List performGetList(String topic, String id) throws IOException { return null; }
321
+ protected String performGetString(String topic, String id) throws IOException { return null; }
322
+ protected Double performGetDouble(String topic, String id) throws IOException { return null; }
323
+
324
+ protected List<Map<String,Object>> performStashes(Map<String,Object> query) throws IOException { return null; }
325
+
326
+ protected void performMerge(String topic, String id, Object value) throws IOException {}
327
+ protected void performMerge(String topic, Map<String,Object> value) throws IOException {}
328
+
329
+ protected void performSet(String topic, String id, Object value) throws IOException {}
330
+ protected void performSet(String topic, Map<String,Object> value) throws IOException {}
331
+
332
+ protected void performDelete(String topic, String id) throws IOException {}
333
+ protected void performDelete(String topic) throws IOException {}
334
+
335
+ private void logRequest(String message) {
336
+ if (dryRun()) {
337
+ LOG.info(message);
338
+ } else {
339
+ LOG.debug(message);
340
+ }
341
+ }
342
+ }