vayacondios-server 0.2.11 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (142) hide show
  1. data/.gitignore +3 -1
  2. data/.travis.yml +2 -0
  3. data/Gemfile +15 -9
  4. data/LICENSE.md +2 -6
  5. data/Procfile +1 -1
  6. data/README.md +656 -111
  7. data/Rakefile +89 -6
  8. data/bin/vcd +10 -0
  9. data/bin/vcd-server +8 -0
  10. data/config/database.yml +6 -0
  11. data/config/spec.example.yml +18 -0
  12. data/config/vayacondios.example.yml +15 -0
  13. data/config/vcd-server.rb +37 -0
  14. data/examples/configuration.rb +56 -0
  15. data/examples/event_stream.rb +19 -0
  16. data/examples/simple.rb +61 -0
  17. data/features/event.feature +319 -0
  18. data/features/events.feature +208 -0
  19. data/features/stash.feature +840 -0
  20. data/features/stashes.feature +492 -0
  21. data/features/step_definitions/stash_steps.rb +113 -0
  22. data/features/stream.feature +30 -0
  23. data/features/support/em.rb +14 -0
  24. data/features/support/env.rb +13 -0
  25. data/lib/vayacondios/configuration.rb +63 -0
  26. data/lib/vayacondios/server/api.rb +126 -0
  27. data/lib/vayacondios/server/api_options.rb +56 -0
  28. data/lib/vayacondios/server/configuration.rb +23 -0
  29. data/lib/vayacondios/server/driver.rb +71 -0
  30. data/lib/vayacondios/server/drivers/mongo.rb +126 -0
  31. data/lib/vayacondios/server/handlers/document_handler.rb +81 -0
  32. data/lib/vayacondios/server/handlers/event_handler.rb +31 -26
  33. data/lib/vayacondios/server/handlers/events_handler.rb +31 -0
  34. data/lib/vayacondios/server/handlers/stash_handler.rb +69 -0
  35. data/lib/vayacondios/server/handlers/stashes_handler.rb +49 -0
  36. data/lib/vayacondios/server/handlers/stream_handler.rb +39 -0
  37. data/lib/vayacondios/server/models/document.rb +87 -0
  38. data/lib/vayacondios/server/models/event.rb +198 -0
  39. data/lib/vayacondios/server/models/stash.rb +100 -0
  40. data/lib/vayacondios/server.rb +35 -0
  41. data/lib/vayacondios-server.rb +19 -13
  42. data/lib/vayacondios.rb +22 -0
  43. data/pom.xml +124 -4
  44. data/spec/configuration_spec.rb +41 -0
  45. data/spec/server/api_options_spec.rb +32 -0
  46. data/spec/server/api_spec.rb +279 -0
  47. data/spec/server/configuration_spec.rb +27 -0
  48. data/spec/server/drivers/mongo_spec.rb +107 -0
  49. data/spec/server/handlers/event_handler_spec.rb +62 -0
  50. data/spec/server/handlers/events_handler_spec.rb +51 -0
  51. data/spec/server/handlers/stash_handler_spec.rb +68 -0
  52. data/spec/server/handlers/stashes_handler_spec.rb +50 -0
  53. data/spec/server/handlers/stream_handler_spec.rb +5 -0
  54. data/spec/server/models/document_spec.rb +9 -0
  55. data/spec/server/models/event_spec.rb +185 -0
  56. data/spec/server/models/stash_spec.rb +95 -0
  57. data/spec/spec_helper.rb +23 -3
  58. data/spec/support/database_helper.rb +42 -0
  59. data/spec/support/log_helper.rb +19 -0
  60. data/spec/support/shared_context_for_events.rb +22 -0
  61. data/spec/support/shared_context_for_stashes.rb +24 -0
  62. data/spec/support/shared_examples_for_handlers.rb +32 -0
  63. data/src/main/java/com/infochimps/vayacondios/BaseClient.java +342 -0
  64. data/src/main/java/com/infochimps/vayacondios/HTTPClient.java +426 -0
  65. data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java +487 -65
  66. data/src/main/java/com/infochimps/vayacondios/test/IntegrationTest.java +3 -0
  67. data/src/test/java/com/infochimps/vayacondios/BaseClientTest.java +50 -0
  68. data/src/test/java/com/infochimps/vayacondios/HTTPClientIT.java +267 -0
  69. data/vayacondios-server.gemspec +9 -9
  70. metadata +127 -122
  71. checksums.yaml +0 -15
  72. data/.rspec +0 -2
  73. data/.yardopts +0 -10
  74. data/Guardfile +0 -41
  75. data/app/http_shim.rb +0 -71
  76. data/bin/vcd.sh +0 -27
  77. data/config/http_shim.rb +0 -43
  78. data/config/vayacondios.example.yaml +0 -7
  79. data/config/vayacondios.yaml +0 -7
  80. data/examples/java/ItemSetTest.java +0 -76
  81. data/lib/tasks/publish.rake +0 -23
  82. data/lib/tasks/spec.rake +0 -11
  83. data/lib/tasks/yard.rake +0 -2
  84. data/lib/vayacondios/client/config.rb +0 -7
  85. data/lib/vayacondios/client/configliere.rb +0 -38
  86. data/lib/vayacondios/client/cube_client.rb +0 -39
  87. data/lib/vayacondios/client/http_client.rb +0 -49
  88. data/lib/vayacondios/client/itemset.rb +0 -130
  89. data/lib/vayacondios/client/legacy_switch.rb +0 -43
  90. data/lib/vayacondios/client/notifier.rb +0 -123
  91. data/lib/vayacondios/client/zabbix_client.rb +0 -148
  92. data/lib/vayacondios/legacy_switch.rb +0 -43
  93. data/lib/vayacondios/server/errors/bad_request.rb +0 -6
  94. data/lib/vayacondios/server/errors/not_found.rb +0 -6
  95. data/lib/vayacondios/server/handlers/config_handler.rb +0 -32
  96. data/lib/vayacondios/server/handlers/itemset_handler.rb +0 -60
  97. data/lib/vayacondios/server/legacy_switch.rb +0 -43
  98. data/lib/vayacondios/server/model/config_document.rb +0 -89
  99. data/lib/vayacondios/server/model/document.rb +0 -25
  100. data/lib/vayacondios/server/model/event_document.rb +0 -94
  101. data/lib/vayacondios/server/model/itemset_document.rb +0 -126
  102. data/lib/vayacondios/server/rack/extract_methods.rb +0 -35
  103. data/lib/vayacondios/server/rack/jsonize.rb +0 -43
  104. data/lib/vayacondios/server/rack/params.rb +0 -50
  105. data/lib/vayacondios/server/rack/path.rb +0 -23
  106. data/lib/vayacondios/server/rack/path_validation.rb +0 -22
  107. data/lib/vayacondios/version.rb +0 -3
  108. data/lib/vayacondios-client.rb +0 -22
  109. data/scripts/hadoop_monitor/configurable.rb +0 -66
  110. data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb +0 -45
  111. data/scripts/hadoop_monitor/hadoop_client.rb +0 -273
  112. data/scripts/hadoop_monitor/hadoop_monitor.rb +0 -101
  113. data/scripts/hadoop_monitor/hadoopable.rb +0 -65
  114. data/scripts/hadoop_monitor/machine_monitor.rb +0 -115
  115. data/scripts/s3_cataloger/buckets +0 -33
  116. data/scripts/s3_cataloger/foreach_bucket +0 -88
  117. data/scripts/s3_cataloger/parse_ls.py +0 -391
  118. data/spec/client/itemset_legacy_spec.rb +0 -55
  119. data/spec/client/itemset_spec.rb +0 -60
  120. data/spec/client/notifier_spec.rb +0 -120
  121. data/spec/server/config_spec.rb +0 -113
  122. data/spec/server/event_spec.rb +0 -103
  123. data/spec/server/itemset_legacy_spec.rb +0 -320
  124. data/spec/server/itemset_spec.rb +0 -317
  125. data/spec/server/rack/extract_methods_spec.rb +0 -60
  126. data/spec/server/rack/path_spec.rb +0 -36
  127. data/spec/server/rack/path_validation_spec.rb +0 -22
  128. data/spec/server/server_spec.rb +0 -20
  129. data/spec/support/mongo_cleaner.rb +0 -32
  130. data/src/main/java/ItemSetTest.java +0 -76
  131. data/src/main/java/com/infochimps/util/CurrentClass.java +0 -26
  132. data/src/main/java/com/infochimps/util/DebugUtil.java +0 -38
  133. data/src/main/java/com/infochimps/util/HttpHelper.java +0 -181
  134. data/src/main/java/com/infochimps/vayacondios/ItemSets.java +0 -373
  135. data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java +0 -18
  136. data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java +0 -84
  137. data/src/main/java/com/infochimps/vayacondios/Organization.java +0 -62
  138. data/src/main/java/com/infochimps/vayacondios/PathBuilder.java +0 -13
  139. data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java +0 -218
  140. data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java +0 -108
  141. data/src/test/java/com/infochimps/vayacondios/TestVayacondiosInMemory.java +0 -78
  142. data/vayacondios-client.gemspec +0 -25
@@ -0,0 +1,342 @@
1
+ package com.infochimps.vayacondios;
2
+
3
+ import java.util.Map;
4
+ import java.util.List;
5
+ import java.io.IOException;
6
+
7
+ import org.slf4j.Logger;
8
+ import org.slf4j.LoggerFactory;
9
+
10
+ /** Base class for all concrete implementations of a Vayacondios client.
11
+ * <p>
12
+ * Is fixed to a specific organization at instantiation time since the
13
+ * common use case is reading/writing data for a single organization
14
+ * at a time. If reading/writing data for multiple organizations
15
+ * simultaneously, use multiple client instances.
16
+ * <p>
17
+ * Can be instantiated in "dry-run" mode in which it will not actually
18
+ * make any requests to the server, merely log its output.
19
+ * <p>
20
+ * Implements the {@link VayacondiosClient} interface by wrapping each
21
+ * method of is API (e.g. - {@link VayacondiosClient#announce(String
22
+ * topic, Map event)}) with logic for handling arguments, errors, and
23
+ * "dry-run" mode, ultimately delegating to a (protected) method to
24
+ * perform the actual request (e.g. - {@link
25
+ * BaseClient#performAnnounce(String topic, Map event)}). Subclasses
26
+ * should override these methods to provide a concrete implementation
27
+ * of a Vayacondios client.
28
+ */
29
+ public class BaseClient implements VayacondiosClient {
30
+
31
+ //----------------------------------------------------------------------------
32
+ // Initialization & Properties
33
+ //----------------------------------------------------------------------------
34
+
35
+ public static String VERSION = "v2";
36
+
37
+ private static Logger LOG = LoggerFactory.getLogger(BaseClient.class);
38
+
39
+ private String _organization;
40
+ private Boolean _dryRun;
41
+
42
+ /**
43
+ * Create a new BaseClient instance for the given
44
+ * organization. optionally in "dry-run" mode.
45
+ *
46
+ * <blockquote><pre>{@code
47
+ * VayacondiosClient client = new BaseClient("my_organization", true); // client in dry-run mode
48
+ * }</pre></blockquote>
49
+ *
50
+ * @param organization name of the organization to read/write data for
51
+ * @param shouldDryRun whether or not to enter "dry-run" mode
52
+ */
53
+ public BaseClient(String organization, Boolean shouldDryRun) {
54
+ this._organization = organization;
55
+ this._dryRun = shouldDryRun;
56
+ }
57
+
58
+ /**
59
+ * Create a new BaseClient instance for the given organization.
60
+ *
61
+ * @param organization name of the organization to read/write data for
62
+ */
63
+ public BaseClient(String organization) {
64
+ this(organization, false);
65
+ }
66
+
67
+ /**
68
+ * Is this client in "dry-run" mode?
69
+ * <p>
70
+ * When in "dry-run" mode, the client will not actually make any
71
+ * requests to the server, it will merely log all its attempts at
72
+ * an elevated level.
73
+ * @return whether or not the client is in "dry-run" mode
74
+ */
75
+ public Boolean dryRun() {
76
+ return _dryRun;
77
+ }
78
+
79
+ /**
80
+ * The organization this client will read/write data for.
81
+ *
82
+ * @return name of the organization
83
+ */
84
+ public String organization() {
85
+ return _organization;
86
+ }
87
+
88
+ //----------------------------------------------------------------------------
89
+ // Public API
90
+ //----------------------------------------------------------------------------
91
+
92
+ /**
93
+ * {@inheritDoc}
94
+ */
95
+ @Override
96
+ public void announce(String topic, Map<String,Object> event) {
97
+ logRequest("Announcing <" + topic + ">");
98
+ if (dryRun()) return;
99
+ try {
100
+ performAnnounce(topic, event);
101
+ } catch (IOException e) {
102
+ LOG.error("Announcing <" + topic + ">", e);
103
+ }
104
+ }
105
+ /**
106
+ * {@inheritDoc}
107
+ */
108
+ @Override
109
+ public void announce(String topic, Map<String,Object> event, String id) {
110
+ logRequest("Announcing <" + topic + "/" + id + ">");
111
+ if (dryRun()) return;
112
+ try {
113
+ performAnnounce(topic, event, id);
114
+ } catch (IOException e) {
115
+ LOG.error("Announcing <" + topic + "/" + id + ">", e);
116
+ }
117
+ }
118
+
119
+ /**
120
+ * {@inheritDoc}
121
+ */
122
+ @Override
123
+ public List<Map<String,Object>> events(String topic, Map<String,Object> query) {
124
+ logRequest("Searching events <" + topic + ">");
125
+ if (dryRun()) return null;
126
+ try {
127
+ return performEvents(topic, query);
128
+ } catch (IOException e) {
129
+ LOG.error("Searching events <" + topic + ">", e);
130
+ return null;
131
+ }
132
+ }
133
+
134
+ /**
135
+ * {@inheritDoc}
136
+ */
137
+ @Override
138
+ public Map<String,Object> get(String topic) {
139
+ logRequest("Fetching <" + topic + ">");
140
+ if (dryRun()) return null;
141
+ try {
142
+ return performGet(topic);
143
+ } catch (IOException e) {
144
+ LOG.error("Fetching <" + topic + ">");
145
+ return null;
146
+ }
147
+ }
148
+
149
+ /**
150
+ * {@inheritDoc}
151
+ */
152
+ @Override
153
+ public Map<String,Object> getMap(String topic, String id) {
154
+ logRequest("Fetching Map <" + topic + "/" + id + ">");
155
+ if (dryRun()) return null;
156
+ try {
157
+ return performGetMap(topic, id);
158
+ } catch (IOException e) {
159
+ LOG.error("Fetching Map <" + topic + "/" + id + ">", e);
160
+ return null;
161
+ }
162
+ }
163
+ /**
164
+ * {@inheritDoc}
165
+ */
166
+ @Override
167
+ public List getList(String topic, String id) {
168
+ logRequest("Fetching List <" + topic + "/" + id + ">");
169
+ if (dryRun()) return null;
170
+ try {
171
+ return performGetList(topic, id);
172
+ } catch (IOException e) {
173
+ LOG.error("Fetching List <" + topic + "/" + id + ">", e);
174
+ return null;
175
+ }
176
+ }
177
+ /**
178
+ * {@inheritDoc}
179
+ */
180
+ @Override
181
+ public String getString(String topic, String id) {
182
+ logRequest("Fetching String <" + topic + "/" + id + ">");
183
+ if (dryRun()) return null;
184
+ try {
185
+ return performGetString(topic, id);
186
+ } catch (IOException e) {
187
+ LOG.error("Fetching String <" + topic + "/" + id + ">", e);
188
+ return null;
189
+ }
190
+ }
191
+ /**
192
+ * {@inheritDoc}
193
+ */
194
+ @Override
195
+ public Double getDouble(String topic, String id) {
196
+ logRequest("Fetching Double <" + topic + "/" + id + ">");
197
+ if (dryRun()) return null;
198
+ try {
199
+ return performGetDouble(topic, id);
200
+ } catch (IOException e) {
201
+ LOG.error("Fetching Double <" + topic + "/" + id + ">", e);
202
+ return null;
203
+ }
204
+ }
205
+
206
+ /**
207
+ * {@inheritDoc}
208
+ */
209
+ @Override
210
+ public List<Map<String,Object>> stashes(Map<String,Object> query) {
211
+ logRequest("Searching stashes");
212
+ if (dryRun()) return null;
213
+ try {
214
+ return performStashes(query);
215
+ } catch (IOException e) {
216
+ LOG.error("Searching stashes", e);
217
+ return null;
218
+ }
219
+ }
220
+
221
+ /**
222
+ * {@inheritDoc}
223
+ */
224
+ @Override
225
+ public void set(String topic, Map<String,Object> value) {
226
+ logRequest("Replacing <" + topic + ">");
227
+ if (dryRun()) return;
228
+ try {
229
+ performSet(topic, value);
230
+ } catch (IOException e) {
231
+ LOG.error("Replacing <" + topic + ">", e);
232
+ }
233
+ }
234
+ /**
235
+ * {@inheritDoc}
236
+ */
237
+ @Override
238
+ public void set(String topic, String id, Object value) {
239
+ logRequest("Replacing <" + topic + "/" + id + ">");
240
+ if (dryRun()) return;
241
+ try {
242
+ performSet(topic, id, value);
243
+ } catch (IOException e) {
244
+ LOG.error("Replacing <" + topic + "/" + id + ">", e);
245
+ }
246
+ }
247
+
248
+ /**
249
+ * {@inheritDoc}
250
+ */
251
+ @Override
252
+ public void merge(String topic, Map<String,Object> value) {
253
+ logRequest("Merging <" + topic + ">");
254
+ if (dryRun()) return;
255
+ try {
256
+ performMerge(topic, value);
257
+ } catch (IOException e) {
258
+ LOG.error("Merging <" + topic + ">", e);
259
+ }
260
+ }
261
+ /**
262
+ * {@inheritDoc}
263
+ */
264
+ @Override
265
+ public void merge(String topic, String id, Object value) {
266
+ logRequest("Merging <" + topic + "/" + id + ">");
267
+ if (dryRun()) return;
268
+ try {
269
+ performMerge(topic, id, value);
270
+ } catch (IOException e) {
271
+ LOG.error("Merging <" + topic + "/" + id + ">", e);
272
+ }
273
+ }
274
+
275
+ /**
276
+ * {@inheritDoc}
277
+ */
278
+ @Override
279
+ public void delete(String topic) {
280
+ logRequest("Deleting <" + topic + ">");
281
+ if (dryRun()) return;
282
+ try {
283
+ performDelete(topic);
284
+ } catch (IOException e) {
285
+ LOG.error("Deleting <" + topic + ">");
286
+ }
287
+ }
288
+ /**
289
+ * {@inheritDoc}
290
+ */
291
+ @Override
292
+ public void delete(String topic, String id) {
293
+ logRequest("Deleting <" + topic + "/" + id + ">");
294
+ if (dryRun()) return;
295
+ try {
296
+ performDelete(topic, id);
297
+ } catch (IOException e) {
298
+ LOG.error("Deleting <" + topic + "/" + id + ">");
299
+ }
300
+ }
301
+
302
+ /**
303
+ * {@inheritDoc}
304
+ */
305
+ @Override
306
+ public void close() {};
307
+
308
+ //----------------------------------------------------------------------------
309
+ // Private API
310
+ //----------------------------------------------------------------------------
311
+
312
+ protected void performAnnounce(String topic, Map<String,Object> event, String id) throws IOException {}
313
+ protected void performAnnounce(String topic, Map<String,Object> event) throws IOException {}
314
+
315
+ protected List<Map<String,Object>> performEvents(String topic, Map<String,Object> query) throws IOException { return null; }
316
+
317
+ protected Map<String,Object> performGet(String topic) throws IOException { return null; }
318
+
319
+ protected Map<String,Object> performGetMap(String topic, String id) throws IOException { return null; }
320
+ protected List performGetList(String topic, String id) throws IOException { return null; }
321
+ protected String performGetString(String topic, String id) throws IOException { return null; }
322
+ protected Double performGetDouble(String topic, String id) throws IOException { return null; }
323
+
324
+ protected List<Map<String,Object>> performStashes(Map<String,Object> query) throws IOException { return null; }
325
+
326
+ protected void performMerge(String topic, String id, Object value) throws IOException {}
327
+ protected void performMerge(String topic, Map<String,Object> value) throws IOException {}
328
+
329
+ protected void performSet(String topic, String id, Object value) throws IOException {}
330
+ protected void performSet(String topic, Map<String,Object> value) throws IOException {}
331
+
332
+ protected void performDelete(String topic, String id) throws IOException {}
333
+ protected void performDelete(String topic) throws IOException {}
334
+
335
+ private void logRequest(String message) {
336
+ if (dryRun()) {
337
+ LOG.info(message);
338
+ } else {
339
+ LOG.debug(message);
340
+ }
341
+ }
342
+ }