wukong 3.0.1 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. data/.gitignore +1 -0
  2. data/Gemfile +1 -1
  3. data/README.md +253 -45
  4. data/bin/wu +34 -0
  5. data/bin/wu-source +5 -0
  6. data/examples/Gemfile +0 -1
  7. data/examples/deploy_pack/Gemfile +0 -1
  8. data/examples/improver/tweet_summary.rb +73 -0
  9. data/examples/ruby_project/Gemfile +0 -1
  10. data/examples/splitter.rb +94 -0
  11. data/examples/twitter.rb +5 -0
  12. data/lib/hanuman.rb +1 -1
  13. data/lib/hanuman/graph.rb +39 -22
  14. data/lib/hanuman/stage.rb +46 -13
  15. data/lib/hanuman/tree.rb +67 -0
  16. data/lib/wukong.rb +6 -1
  17. data/lib/wukong/dataflow.rb +19 -48
  18. data/lib/wukong/driver.rb +176 -65
  19. data/lib/wukong/{local → driver}/event_machine_driver.rb +1 -13
  20. data/lib/wukong/driver/wiring.rb +68 -0
  21. data/lib/wukong/local.rb +6 -4
  22. data/lib/wukong/local/runner.rb +14 -16
  23. data/lib/wukong/local/stdio_driver.rb +72 -12
  24. data/lib/wukong/processor.rb +1 -30
  25. data/lib/wukong/runner.rb +2 -0
  26. data/lib/wukong/runner/command_runner.rb +44 -0
  27. data/lib/wukong/source.rb +33 -0
  28. data/lib/wukong/source/source_driver.rb +74 -0
  29. data/lib/wukong/source/source_runner.rb +38 -0
  30. data/lib/wukong/spec_helpers/shared_examples.rb +0 -1
  31. data/lib/wukong/spec_helpers/unit_tests.rb +6 -5
  32. data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb +4 -14
  33. data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +7 -8
  34. data/lib/wukong/version.rb +1 -1
  35. data/lib/wukong/widget/echo.rb +55 -0
  36. data/lib/wukong/widget/{processors.rb → extract.rb} +0 -106
  37. data/lib/wukong/widget/filters.rb +15 -0
  38. data/lib/wukong/widget/logger.rb +56 -0
  39. data/lib/wukong/widget/operators.rb +82 -0
  40. data/lib/wukong/widget/reducers.rb +2 -0
  41. data/lib/wukong/widget/reducers/improver.rb +71 -0
  42. data/lib/wukong/widget/reducers/join_xml.rb +37 -0
  43. data/lib/wukong/widget/serializers.rb +21 -6
  44. data/lib/wukong/widgets.rb +6 -3
  45. data/spec/hanuman/graph_spec.rb +73 -10
  46. data/spec/hanuman/stage_spec.rb +15 -0
  47. data/spec/hanuman/tree_spec.rb +119 -0
  48. data/spec/spec_helper.rb +13 -1
  49. data/spec/support/example_test_helpers.rb +0 -1
  50. data/spec/support/model_test_helpers.rb +1 -1
  51. data/spec/support/shared_context_for_graphs.rb +57 -0
  52. data/spec/support/shared_examples_for_builders.rb +8 -15
  53. data/spec/wukong/driver_spec.rb +152 -0
  54. data/spec/wukong/local/runner_spec.rb +1 -12
  55. data/spec/wukong/local/stdio_driver_spec.rb +73 -0
  56. data/spec/wukong/processor_spec.rb +0 -1
  57. data/spec/wukong/runner_spec.rb +2 -2
  58. data/spec/wukong/source_spec.rb +6 -0
  59. data/spec/wukong/widget/extract_spec.rb +101 -0
  60. data/spec/wukong/widget/logger_spec.rb +23 -0
  61. data/spec/wukong/widget/operators_spec.rb +25 -0
  62. data/spec/wukong/widget/reducers/join_xml_spec.rb +25 -0
  63. data/spec/wukong/wu-source_spec.rb +32 -0
  64. data/spec/wukong/wu_spec.rb +14 -0
  65. data/wukong.gemspec +1 -2
  66. metadata +45 -28
  67. data/lib/wukong/local/tcp_driver.rb +0 -47
  68. data/spec/wu/geo/geolocated_spec.rb +0 -247
  69. data/spec/wukong/widget/processors_spec.rb +0 -125
@@ -0,0 +1,23 @@
1
+ require 'spec_helper'
2
+
3
+ describe :logger do
4
+ it_behaves_like "a processor", :named => :logger
5
+
6
+ it "logs each event at the 'info' level by default" do
7
+ log = double("logger")
8
+ log.should_receive(:info).with('hi there')
9
+ log.should_receive(:info).with('buddy')
10
+ processor(:logger) do |proc|
11
+ proc.stub(:log).and_return(log)
12
+ end.given('hi there', 'buddy').should emit(0).records
13
+ end
14
+
15
+ it "logs each event at the a desired level set with an argument" do
16
+ log = double("logger")
17
+ log.should_receive(:debug).with('hi there')
18
+ log.should_receive(:debug).with('buddy')
19
+ processor(:logger, level: :debug) do |proc|
20
+ proc.stub(:log).and_return(log)
21
+ end.given('hi there', 'buddy').should emit(0).records
22
+ end
23
+ end
@@ -0,0 +1,25 @@
1
+ require 'spec_helper'
2
+
3
+ describe "Operators" do
4
+
5
+ describe :map do
6
+ it_behaves_like 'a processor', :named => :map
7
+ it "performs an action on each input record" do
8
+ processor(:map, action: ->(input_record) { input_record.upcase }).given('snap', 'crackle', 'pop').should emit('SNAP', 'CRACKLE', 'POP')
9
+ end
10
+
11
+ it "can simultaneously filter out records" do
12
+ processor(:map, compact: true, action: ->(input_record) { input_record + 1 if input_record > 0 }).given(2, -4, 6).should emit(3, 7)
13
+ end
14
+ end
15
+
16
+ describe :flatten do
17
+ it_behaves_like 'a processor', :named => :flatten
18
+
19
+ it "yields each input record or its contents" do
20
+ processor(:flatten).given('foo', ['bar', 'baz'], 'bing').should emit('foo', 'bar', 'baz', 'bing')
21
+ end
22
+
23
+ end
24
+
25
+ end
@@ -0,0 +1,25 @@
1
+ require 'spec_helper'
2
+
3
+ describe "Reducers" do
4
+ describe :join_xml do
5
+ include_context "reducers"
6
+ it_behaves_like 'a processor', :named => :join_xml
7
+
8
+ it "joins XML spread out over multiple lines" do
9
+ processor(:join_xml).given('<xml>first line', 'second line', 'third line</xml>').should emit("<xml>first line\nsecond line\nthird line</xml>")
10
+ end
11
+
12
+ it "joins XML one-per-line" do
13
+ processor(:join_xml).given('<xml>first line</xml>', '<xml>second line</xml>', '<xml>third line</xml>').should emit('<xml>first line</xml>', '<xml>second line</xml>', '<xml>third line</xml>')
14
+ end
15
+
16
+ it "joins XML split in the middle of a line" do
17
+ processor(:join_xml).given('<xml>first line', 'second</xml><xml> line', 'third line</xml>').should emit("<xml>first line\nsecond</xml>", "<xml> line\nthird line</xml>")
18
+ end
19
+
20
+ it "joins XML with a custom tag" do
21
+ processor(:join_xml, root: 'foobar').given('<foobar>first line', 'second line', 'third line</foobar>').should emit("<foobar>first line\nsecond line\nthird line</foobar>")
22
+ end
23
+
24
+ end
25
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'wu-source' do
4
+
5
+ let(:input) { %w[1 2 3] }
6
+
7
+ context "without any arguments" do
8
+ subject { wu_source() }
9
+ it {should exit_with(:non_zero) }
10
+ it "displays help on STDERR" do
11
+ should have_stderr(/provide a processor.*to run/)
12
+ end
13
+ end
14
+
15
+ # FIXME -- it's hard to write an integration test for wu-source
16
+ # because it doesn't self-terminate under any conditions when run
17
+ # successfully.
18
+ #
19
+ # Options:
20
+ #
21
+ # 1) Add a --max (or similar) flag to wu-source allowing it to
22
+ # exit after some number of records which could then be checked
23
+ # by an integration test.
24
+ #
25
+ # 2) Launch it in a subprocess and wait a little while (how long?)
26
+ # and ensure that it's produced a bunch of output in the meantime.
27
+ # If the `per_sec` option is high, we shouldn't have to wait very
28
+ # long to see a whole bunch of output records. This is tricky b/c
29
+ # what if the system is under load and we don't wait long enough
30
+ # for the wu-source subprocess to boot up and start emitting?
31
+
32
+ end
@@ -0,0 +1,14 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'wu' do
4
+
5
+ let(:input) { %w[1 2 3] }
6
+
7
+ context "without any arguments" do
8
+ subject { wu() }
9
+ it {should exit_with(:non_zero) }
10
+ it "displays help on STDERR" do
11
+ should have_stderr(/provide a Wukong command to run/)
12
+ end
13
+ end
14
+ end
@@ -21,13 +21,12 @@ Gem::Specification.new do |gem|
21
21
  EOF
22
22
 
23
23
  gem.files = `git ls-files`.split("\n").reject { |path| path =~ /^(data|docpages|notes|old)/ }
24
- gem.executables = ['wu-local']
24
+ gem.executables = ['wu-local', 'wu-source', 'wu']
25
25
  gem.test_files = gem.files.grep(/^spec/)
26
26
  gem.require_paths = ['lib']
27
27
 
28
28
  gem.add_dependency('configliere', '>= 0.4.18')
29
29
  gem.add_dependency('multi_json', '>= 1.3.6')
30
- gem.add_dependency('vayacondios-client', '>= 0.1.2')
31
30
  gem.add_dependency('gorillib', '>= 0.4.2')
32
31
  gem.add_dependency('forgery')
33
32
  gem.add_dependency('uuidtools')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wukong
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.1
4
+ version: 4.0.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2013-03-07 00:00:00.000000000 Z
14
+ date: 2014-03-19 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: configliere
@@ -45,22 +45,6 @@ dependencies:
45
45
  - - ! '>='
46
46
  - !ruby/object:Gem::Version
47
47
  version: 1.3.6
48
- - !ruby/object:Gem::Dependency
49
- name: vayacondios-client
50
- requirement: !ruby/object:Gem::Requirement
51
- none: false
52
- requirements:
53
- - - ! '>='
54
- - !ruby/object:Gem::Version
55
- version: 0.1.2
56
- type: :runtime
57
- prerelease: false
58
- version_requirements: !ruby/object:Gem::Requirement
59
- none: false
60
- requirements:
61
- - - ! '>='
62
- - !ruby/object:Gem::Version
63
- version: 0.1.2
64
48
  - !ruby/object:Gem::Dependency
65
49
  name: gorillib
66
50
  requirement: !ruby/object:Gem::Requirement
@@ -149,6 +133,8 @@ description: ! " Treat your dataset like a:\n\n * stream of lines when it'
149
133
  email: coders@infochimps.com
150
134
  executables:
151
135
  - wu-local
136
+ - wu-source
137
+ - wu
152
138
  extensions: []
153
139
  extra_rdoc_files: []
154
140
  files:
@@ -174,6 +160,7 @@ files:
174
160
  - bin/tabchar
175
161
  - bin/uniq-ord
176
162
  - bin/uniqc
163
+ - bin/wu
177
164
  - bin/wu-clean-encoding
178
165
  - bin/wu-date
179
166
  - bin/wu-datetime
@@ -181,6 +168,7 @@ files:
181
168
  - bin/wu-lign
182
169
  - bin/wu-local
183
170
  - bin/wu-plus
171
+ - bin/wu-source
184
172
  - bin/wu-sum
185
173
  - diagrams/wu_local.dot
186
174
  - diagrams/wu_local.dot.png
@@ -215,6 +203,7 @@ files:
215
203
  - examples/geo/tile_fetcher.rb
216
204
  - examples/graph/implied_geolocation/README.md
217
205
  - examples/graph/minimum_spanning_tree/airfares_graphviz.rb
206
+ - examples/improver/tweet_summary.rb
218
207
  - examples/loadable.rb
219
208
  - examples/munging/airline_flights/airline_flights.rake
220
209
  - examples/munging/airline_flights/airplane.rb
@@ -266,6 +255,7 @@ files:
266
255
  - examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb
267
256
  - examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb
268
257
  - examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb
258
+ - examples/splitter.rb
269
259
  - examples/string_reverser.rb
270
260
  - examples/text/pig_latin/pig_latinizer.rb
271
261
  - examples/text/pig_latin/pig_latinizer_widget.rb
@@ -273,6 +263,7 @@ files:
273
263
  - examples/text/regional_flavor/article_wordbags.pig
274
264
  - examples/text/regional_flavor/j01-article_wordbags.rb
275
265
  - examples/text/regional_flavor/simple_pig_script.pig
266
+ - examples/twitter.rb
276
267
  - lib/hanuman.rb
277
268
  - lib/hanuman/graph.rb
278
269
  - lib/hanuman/graphvizzer.rb
@@ -281,6 +272,7 @@ files:
281
272
  - lib/hanuman/link.rb
282
273
  - lib/hanuman/registry.rb
283
274
  - lib/hanuman/stage.rb
275
+ - lib/hanuman/tree.rb
284
276
  - lib/wu/geo.rb
285
277
  - lib/wu/geo/geo_grids.numbers
286
278
  - lib/wu/geo/geolocated.rb
@@ -297,11 +289,11 @@ files:
297
289
  - lib/wukong/doc_helpers/field_handler.rb
298
290
  - lib/wukong/doc_helpers/processor_handler.rb
299
291
  - lib/wukong/driver.rb
292
+ - lib/wukong/driver/event_machine_driver.rb
293
+ - lib/wukong/driver/wiring.rb
300
294
  - lib/wukong/local.rb
301
- - lib/wukong/local/event_machine_driver.rb
302
295
  - lib/wukong/local/runner.rb
303
296
  - lib/wukong/local/stdio_driver.rb
304
- - lib/wukong/local/tcp_driver.rb
305
297
  - lib/wukong/logger.rb
306
298
  - lib/wukong/model/faker.rb
307
299
  - lib/wukong/model/flatpack_parser/flat.rb
@@ -315,8 +307,12 @@ files:
315
307
  - lib/wukong/runner.rb
316
308
  - lib/wukong/runner/boot_sequence.rb
317
309
  - lib/wukong/runner/code_loader.rb
310
+ - lib/wukong/runner/command_runner.rb
318
311
  - lib/wukong/runner/deploy_pack_loader.rb
319
312
  - lib/wukong/runner/help_message.rb
313
+ - lib/wukong/source.rb
314
+ - lib/wukong/source/source_driver.rb
315
+ - lib/wukong/source/source_runner.rb
320
316
  - lib/wukong/spec_helpers.rb
321
317
  - lib/wukong/spec_helpers/integration_tests.rb
322
318
  - lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb
@@ -327,14 +323,19 @@ files:
327
323
  - lib/wukong/spec_helpers/unit_tests/unit_test_matchers.rb
328
324
  - lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb
329
325
  - lib/wukong/version.rb
326
+ - lib/wukong/widget/echo.rb
327
+ - lib/wukong/widget/extract.rb
330
328
  - lib/wukong/widget/filters.rb
331
- - lib/wukong/widget/processors.rb
329
+ - lib/wukong/widget/logger.rb
330
+ - lib/wukong/widget/operators.rb
332
331
  - lib/wukong/widget/reducers.rb
333
332
  - lib/wukong/widget/reducers/accumulator.rb
334
333
  - lib/wukong/widget/reducers/bin.rb
335
334
  - lib/wukong/widget/reducers/count.rb
336
335
  - lib/wukong/widget/reducers/group.rb
337
336
  - lib/wukong/widget/reducers/group_concat.rb
337
+ - lib/wukong/widget/reducers/improver.rb
338
+ - lib/wukong/widget/reducers/join_xml.rb
338
339
  - lib/wukong/widget/reducers/moments.rb
339
340
  - lib/wukong/widget/reducers/sort.rb
340
341
  - lib/wukong/widget/reducers/uniq.rb
@@ -355,36 +356,44 @@ files:
355
356
  - spec/hanuman/hanuman_spec.rb
356
357
  - spec/hanuman/registry_spec.rb
357
358
  - spec/hanuman/stage_spec.rb
359
+ - spec/hanuman/tree_spec.rb
358
360
  - spec/spec.opts
359
361
  - spec/spec_helper.rb
360
362
  - spec/support/example_test_helpers.rb
361
363
  - spec/support/hanuman_test_helpers.rb
362
364
  - spec/support/integration_helper.rb
363
365
  - spec/support/model_test_helpers.rb
366
+ - spec/support/shared_context_for_graphs.rb
364
367
  - spec/support/shared_context_for_reducers.rb
365
368
  - spec/support/shared_examples_for_builders.rb
366
369
  - spec/support/shared_examples_for_shortcuts.rb
367
- - spec/wu/geo/geolocated_spec.rb
368
370
  - spec/wu/model/reconcilable_spec.rb
369
371
  - spec/wukong/dataflow_spec.rb
370
372
  - spec/wukong/driver_spec.rb
371
373
  - spec/wukong/local/runner_spec.rb
374
+ - spec/wukong/local/stdio_driver_spec.rb
372
375
  - spec/wukong/local_spec.rb
373
376
  - spec/wukong/logger_spec.rb
374
377
  - spec/wukong/model/faker_spec.rb
375
378
  - spec/wukong/processor_spec.rb
376
379
  - spec/wukong/runner_spec.rb
380
+ - spec/wukong/source_spec.rb
381
+ - spec/wukong/widget/extract_spec.rb
377
382
  - spec/wukong/widget/filters_spec.rb
378
- - spec/wukong/widget/processors_spec.rb
383
+ - spec/wukong/widget/logger_spec.rb
384
+ - spec/wukong/widget/operators_spec.rb
379
385
  - spec/wukong/widget/reducers/bin_spec.rb
380
386
  - spec/wukong/widget/reducers/count_spec.rb
381
387
  - spec/wukong/widget/reducers/group_spec.rb
388
+ - spec/wukong/widget/reducers/join_xml_spec.rb
382
389
  - spec/wukong/widget/reducers/moments_spec.rb
383
390
  - spec/wukong/widget/reducers/sort_spec.rb
384
391
  - spec/wukong/widget/serializers_spec.rb
385
392
  - spec/wukong/widget/sink_spec.rb
386
393
  - spec/wukong/widget/source_spec.rb
387
394
  - spec/wukong/wu-local_spec.rb
395
+ - spec/wukong/wu-source_spec.rb
396
+ - spec/wukong/wu_spec.rb
388
397
  - spec/wukong/wukong_spec.rb
389
398
  - wukong.gemspec
390
399
  homepage: https://github.com/infochimps-labs/wukong
@@ -402,7 +411,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
402
411
  version: '0'
403
412
  segments:
404
413
  - 0
405
- hash: 719389029987495852
414
+ hash: -3691227993438066896
406
415
  required_rubygems_version: !ruby/object:Gem::Requirement
407
416
  none: false
408
417
  requirements:
@@ -411,10 +420,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
411
420
  version: '0'
412
421
  segments:
413
422
  - 0
414
- hash: 719389029987495852
423
+ hash: -3691227993438066896
415
424
  requirements: []
416
425
  rubyforge_project:
417
- rubygems_version: 1.8.24
426
+ rubygems_version: 1.8.23
418
427
  signing_key:
419
428
  specification_version: 3
420
429
  summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use
@@ -434,35 +443,43 @@ test_files:
434
443
  - spec/hanuman/hanuman_spec.rb
435
444
  - spec/hanuman/registry_spec.rb
436
445
  - spec/hanuman/stage_spec.rb
446
+ - spec/hanuman/tree_spec.rb
437
447
  - spec/spec.opts
438
448
  - spec/spec_helper.rb
439
449
  - spec/support/example_test_helpers.rb
440
450
  - spec/support/hanuman_test_helpers.rb
441
451
  - spec/support/integration_helper.rb
442
452
  - spec/support/model_test_helpers.rb
453
+ - spec/support/shared_context_for_graphs.rb
443
454
  - spec/support/shared_context_for_reducers.rb
444
455
  - spec/support/shared_examples_for_builders.rb
445
456
  - spec/support/shared_examples_for_shortcuts.rb
446
- - spec/wu/geo/geolocated_spec.rb
447
457
  - spec/wu/model/reconcilable_spec.rb
448
458
  - spec/wukong/dataflow_spec.rb
449
459
  - spec/wukong/driver_spec.rb
450
460
  - spec/wukong/local/runner_spec.rb
461
+ - spec/wukong/local/stdio_driver_spec.rb
451
462
  - spec/wukong/local_spec.rb
452
463
  - spec/wukong/logger_spec.rb
453
464
  - spec/wukong/model/faker_spec.rb
454
465
  - spec/wukong/processor_spec.rb
455
466
  - spec/wukong/runner_spec.rb
467
+ - spec/wukong/source_spec.rb
468
+ - spec/wukong/widget/extract_spec.rb
456
469
  - spec/wukong/widget/filters_spec.rb
457
- - spec/wukong/widget/processors_spec.rb
470
+ - spec/wukong/widget/logger_spec.rb
471
+ - spec/wukong/widget/operators_spec.rb
458
472
  - spec/wukong/widget/reducers/bin_spec.rb
459
473
  - spec/wukong/widget/reducers/count_spec.rb
460
474
  - spec/wukong/widget/reducers/group_spec.rb
475
+ - spec/wukong/widget/reducers/join_xml_spec.rb
461
476
  - spec/wukong/widget/reducers/moments_spec.rb
462
477
  - spec/wukong/widget/reducers/sort_spec.rb
463
478
  - spec/wukong/widget/serializers_spec.rb
464
479
  - spec/wukong/widget/sink_spec.rb
465
480
  - spec/wukong/widget/source_spec.rb
466
481
  - spec/wukong/wu-local_spec.rb
482
+ - spec/wukong/wu-source_spec.rb
483
+ - spec/wukong/wu_spec.rb
467
484
  - spec/wukong/wukong_spec.rb
468
485
  has_rdoc:
@@ -1,47 +0,0 @@
1
- require_relative('event_machine_driver')
2
- module Wukong
3
- module Local
4
-
5
- # A class for driving processors over a TCP protocol.
6
- class TCPDriver < EM::P::LineAndTextProtocol
7
- include EventMachineDriver
8
- include Processor::BufferedProcessor
9
- include Logging
10
-
11
- def self.start(label, settings = {})
12
- host = (settings[:host] || Socket.gethostname) rescue 'localhost'
13
- port = (settings[:port] || 9000).to_i rescue 9000
14
- EM.start_server(host, port, self, label, settings)
15
- log.info "Server started on #{host} on port #{port}"
16
- add_signal_traps
17
- end
18
-
19
- def post_init
20
- port, ip = Socket.unpack_sockaddr_in(get_peername)
21
- log.info "Connected to #{ip} on #{port}"
22
- setup_dataflow
23
- end
24
-
25
- def receive_line line
26
- @buffer = []
27
- operation = proc { driver.send_through_dataflow(line) }
28
- callback = proc { flush_buffer @buffer }
29
- EM.defer(operation, callback)
30
- rescue => e
31
- EM.stop
32
- raise Wukong::Error.new(e)
33
- end
34
-
35
- def flush_buffer records
36
- send_data(records.join("\n") + "\n")
37
- records.clear
38
- end
39
-
40
- def unbind
41
- finalize_and_stop_dataflow
42
- EM.stop
43
- end
44
-
45
- end
46
- end
47
- end
@@ -1,247 +0,0 @@
1
- require 'gorillib/data_munging'
2
- require 'wu/geo/geolocated'
3
-
4
- describe Wukong::Geolocated do
5
- let(:aus_lng){ -97.759003 } # Austin, TX -- infochimps HQ
6
- let(:aus_lat){ 30.273884 }
7
- let(:sat_lng){ -98.486123 } # San Antonio, TX
8
- let(:sat_lat){ 29.42575 }
9
- let(:dpi){ 72 }
10
- #
11
- let(:aus_tile_x_3){ 1.82758 } # zoom level 3
12
- let(:aus_tile_y_3){ 3.29356 }
13
- let(:aus_pixel_x_3){ 468 }
14
- let(:aus_pixel_y_3){ 843 }
15
- #
16
- let(:aus_tile_x_8){ 58.48248675555555 } # zoom level 8
17
- let(:aus_tile_y_8){ 105.39405073699557 }
18
- let(:aus_tile_x_11){ 467 } # zoom level 11
19
- let(:aus_tile_y_11){ 843 }
20
- #
21
- let(:aus_quadkey ){ "0231301203311211" }
22
- let(:aus_quadkey_3){ "023" }
23
- let(:radius){ 1_000_000 } # 1,000 km
24
-
25
- context Wukong::Geolocated::ByCoordinates do
26
- let(:point_klass) do
27
- module Wukong
28
- class TestPoint
29
- include Gorillib::Model
30
- include Wukong::Geolocated::ByCoordinates
31
- field :name, String, position: 0, doc: "Name of this location"
32
- field :longitude, Float, position: 1, doc: "Longitude (X) of a point, in decimal degrees"
33
- field :latitude, Float, position: 2, doc: "Latitude (Y) of a point, in decimal degrees"
34
- end
35
- end
36
- Wukong::TestPoint
37
- end
38
- subject{ point_klass.new("Infochimps HQ", aus_lng, aus_lat) }
39
-
40
- context '#tile_xf' do
41
- it "tile X coordinate, as a float" do
42
- subject.tile_xf(3).should be_within(0.0001).of( 1.82758)
43
- subject.tile_xf(8).should be_within(0.0001).of(58.48248)
44
- subject.tile_xf(11).should be_within(0.0001).of(467.8598)
45
- end
46
- end
47
- context '#tile_x' do
48
- it "tile X coordinate, as an integer" do
49
- subject.tile_x(3).should == 1
50
- subject.tile_x(8).should == 58
51
- subject.tile_x(11).should == 467
52
- end
53
- end
54
- context '#tile_yf' do
55
- it "tile Y coordinate, as a float" do
56
- subject.tile_yf(3).should be_within(0.0001).of( 3.29356)
57
- subject.tile_yf(8).should be_within(0.0001).of(105.394051)
58
- subject.tile_yf(11).should be_within(0.0001).of(843.152406)
59
- end
60
- end
61
- context '#tile_x' do
62
- it "tile Y coordinate, as an integer" do
63
- subject.tile_y(3).should == 3
64
- subject.tile_y(8).should == 105
65
- subject.tile_y(11).should == 843
66
- end
67
- end
68
- context '#quadkey' do
69
- it "a string of 2-bit tile selectors" do
70
- subject.quadkey(3).should == "023"
71
- subject.quadkey(16).should == "0231301203311211"
72
- end
73
- end
74
- end
75
-
76
- context Wukong::Geolocated do
77
-
78
- it "gives private methods on including class as well as the methods on itself" do
79
- klass = Class.new{ include Wukong::Geolocated }
80
- klass.should be_private_method_defined(:lng_lat_zl_to_tile_xy)
81
- klass.should be_private_method_defined(:haversine_distance)
82
- end
83
-
84
- #
85
- # Tile coordinates
86
- #
87
-
88
- it "returns a map tile size given a zoom level" do
89
- Wukong::Geolocated.map_tile_size(3).should == 8
90
- end
91
-
92
- it "returns a tile_x, tile_y pair given a longitude, latitude and zoom level" do
93
- Wukong::Geolocated.lng_lat_zl_to_tile_xy(aus_lng, aus_lat, 8).should == [ 58, 105]
94
- Wukong::Geolocated.lng_lat_zl_to_tile_xy(aus_lng, aus_lat, 11).should == [467, 843]
95
- end
96
-
97
- it "returns a longitude, latitude pair given tile_x, tile_y and zoom level" do
98
- lng, lat = Wukong::Geolocated.tile_xy_zl_to_lng_lat(aus_tile_x_8, aus_tile_y_8, 8)
99
- lng.should be_within(0.0001).of(aus_lng)
100
- lat.should be_within(0.0001).of(aus_lat)
101
- end
102
-
103
- #
104
- # Pixel coordinates
105
- #
106
-
107
- it "returns a map pizel size given a zoom level" do
108
- Wukong::Geolocated.map_pixel_size(3).should == 2048
109
- end
110
-
111
- it "returns a pixel_x, pixel_y pair given a longitude, latitude and zoom level" do
112
- Wukong::Geolocated.lng_lat_zl_to_pixel_xy(aus_lng, aus_lat, 3).should == [468, 843]
113
- end
114
-
115
- it "returns a longitude, latitude pair given pixel_x, pixel_y and zoom level" do
116
- lng, lat = Wukong::Geolocated.pixel_xy_zl_to_lng_lat(aus_pixel_x_3, aus_pixel_y_3, 3)
117
- lat.round(4).should == 30.2970
118
- lng.round(4).should == -97.7344
119
- end
120
-
121
- it "returns a tile x-y pair given a pixel x-y pair" do
122
- Wukong::Geolocated.pixel_xy_to_tile_xy(aus_pixel_x_3, aus_pixel_y_3).should == [1,3]
123
- end
124
-
125
- it "returns a pixel x-y pair given a float tile x-y pair" do
126
- Wukong::Geolocated.tile_xy_to_pixel_xy(aus_tile_x_3, aus_tile_y_3 ).should == [467.86048, 843.15136]
127
- end
128
-
129
- it "returns a pixel x-y pair given an integer tile x-y pair" do
130
- Wukong::Geolocated.tile_xy_to_pixel_xy(aus_tile_x_3.to_i, aus_tile_y_3.to_i).should == [256, 768]
131
- end
132
-
133
- #
134
- # Quadkey coordinates
135
- #
136
-
137
- it "returns a quadkey given a tile x-y pair and a zoom level" do
138
- Wukong::Geolocated.tile_xy_zl_to_quadkey(aus_tile_x_3, aus_tile_y_3, 3).should == "023"
139
- Wukong::Geolocated.tile_xy_zl_to_quadkey(aus_tile_x_8, aus_tile_y_8, 8).should == "02313012"
140
- Wukong::Geolocated.tile_xy_zl_to_quadkey(aus_tile_x_11, aus_tile_y_11,11).should == "02313012033"
141
- end
142
-
143
- it "returns a quadkey given a longitude, latitude and a zoom level" do
144
- Wukong::Geolocated.lng_lat_zl_to_quadkey(aus_lng, aus_lat, 3).should == "023"
145
- Wukong::Geolocated.lng_lat_zl_to_quadkey(aus_lng, aus_lat, 8).should == "02313012"
146
- Wukong::Geolocated.lng_lat_zl_to_quadkey(aus_lng, aus_lat, 11).should == "02313012033"
147
- Wukong::Geolocated.lng_lat_zl_to_quadkey(aus_lng, aus_lat, 16).should == "0231301203311211"
148
- end
149
-
150
- it "returns a packed quadkey (an integer) given a tile xy and zoom level" do
151
- Wukong::Geolocated.tile_xy_zl_to_packed_qk(aus_tile_x_3.floor, aus_tile_y_3.floor, 3).should == "023".to_i(4)
152
- Wukong::Geolocated.tile_xy_zl_to_packed_qk(aus_tile_x_8.floor, aus_tile_y_8.floor, 8).should == "02313012".to_i(4)
153
- Wukong::Geolocated.tile_xy_zl_to_packed_qk(aus_tile_x_11.floor, aus_tile_y_11.floor,11).should == "02313012033".to_i(4)
154
- end
155
-
156
- context '.packed_qk_zl_to_tile_xy' do
157
- let(:packed_qk){ "0231301203311211".to_i(4) }
158
- it "returns a tile xy given a packed quadkey (integer)" do
159
- Wukong::Geolocated.packed_qk_zl_to_tile_xy(packed_qk >> 26, 3).should == [ 1, 3, 3]
160
- Wukong::Geolocated.packed_qk_zl_to_tile_xy(packed_qk >> 16, 8).should == [ 58, 105, 8]
161
- Wukong::Geolocated.packed_qk_zl_to_tile_xy(packed_qk >> 10, 11).should == [467, 843, 11]
162
- end
163
-
164
- it "defaults to zl=16 for packed quadkeys" do
165
- Wukong::Geolocated.packed_qk_zl_to_tile_xy(packed_qk ).should == [14971, 26980, 16]
166
- Wukong::Geolocated.packed_qk_zl_to_tile_xy(packed_qk, 16).should == [14971, 26980, 16]
167
- end
168
- end
169
-
170
- it "returns tile x-y pair and a zoom level given a quadkey" do
171
- Wukong::Geolocated.quadkey_to_tile_xy_zl(aus_quadkey[0..2] ).should == [1, 3, 3]
172
- Wukong::Geolocated.quadkey_to_tile_xy_zl(aus_quadkey[0..7] ).should == [aus_tile_x_8.floor, aus_tile_y_8.floor, 8]
173
- Wukong::Geolocated.quadkey_to_tile_xy_zl(aus_quadkey[0..10]).should == [aus_tile_x_11.floor, aus_tile_y_11.floor, 11]
174
- end
175
-
176
- it "allows '' to be a quadkey (whole map)" do
177
- Wukong::Geolocated.quadkey_to_tile_xy_zl("").should == [0, 0, 0]
178
- end
179
-
180
- it "maps tile xyz [0,0,0] to quadkey ''" do
181
- Wukong::Geolocated.tile_xy_zl_to_quadkey(0,0,0).should == ""
182
- end
183
-
184
- it "throws an error if a bad quadkey is given" do
185
- expect{ Wukong::Geolocated.quadkey_to_tile_xy_zl("bad_key") }.to raise_error(ArgumentError, /Quadkey.*characters/)
186
- end
187
-
188
- it "returns a bounding box given a quadkey" do
189
- left, btm, right, top = Wukong::Geolocated.quadkey_to_bbox(aus_quadkey_3)
190
- left.should be_within(0.0001).of(-135.0)
191
- right.should be_within(0.0001).of(- 90.0)
192
- btm.should be_within(0.0001).of( 0.0)
193
- top.should be_within(0.0001).of( 40.9799)
194
- end
195
-
196
- it "returns the smallest quadkey containing two points" do
197
- Wukong::Geolocated.quadkey_containing_bbox(aus_lng, aus_lat, sat_lng, sat_lat).should == "023130"
198
- end
199
-
200
- it "returns a bounding box given a point and radius" do
201
- left, btm, right, top = Wukong::Geolocated.lng_lat_rad_to_bbox(aus_lng, aus_lat, radius)
202
-
203
- left.should be_within(0.0001).of(-108.1723)
204
- right.should be_within(0.0001).of(- 87.3457)
205
- btm.should be_within(0.0001).of( 21.2807)
206
- top.should be_within(0.0001).of( 39.2671)
207
- end
208
-
209
- it "returns a centroid given a bounding box" do
210
- mid_lng, mid_lat = Wukong::Geolocated.bbox_centroid([aus_lng, sat_lat], [sat_lng, aus_lat])
211
- mid_lng.should be_within(0.0001).of(-98.1241)
212
- mid_lat.should be_within(0.0001).of( 29.8503)
213
- end
214
-
215
- it "returns a pixel resolution given a latitude and zoom level" do
216
- Wukong::Geolocated.pixel_resolution(aus_lat, 3).should be_within(0.0001).of(16880.4081)
217
- end
218
-
219
- it "returns a map scale given a latitude, zoom level and dpi" do
220
- Wukong::Geolocated.map_scale_for_dpi(aus_lat, 3, dpi).should be_within(0.0001).of(47849975.8302)
221
- end
222
-
223
- it "calculates the haversine distance between two points" do
224
- Wukong::Geolocated.haversine_distance(aus_lng, aus_lat, sat_lng, sat_lat).should be_within(0.0001).of(117522.1219)
225
- end
226
-
227
- it "calculates the haversine midpoint between two points" do
228
- lng, lat = Wukong::Geolocated.haversine_midpoint(aus_lng, sat_lat, sat_lng, aus_lat)
229
- lng.should be_within(0.0001).of(-98.1241)
230
- lat.should be_within(0.0001).of( 29.8503)
231
- end
232
-
233
- it "calculates the point a given distance directly north from a lat/lng" do
234
- lng, lat = Wukong::Geolocated.point_north(aus_lng, aus_lat, 1000000)
235
- lng.should be_within(0.0001).of(-97.7590)
236
- lat.should be_within(0.0001).of( 39.2671)
237
- end
238
-
239
- it "calculates the point a given distance directly east from a lat/lng" do
240
- lng, lat = Wukong::Geolocated.point_east(aus_lng, aus_lat, 1000000)
241
- lng.should be_within(0.0001).of(-87.3457)
242
- lat.should be_within(0.0001).of( 30.2739)
243
- end
244
-
245
-
246
- end # module methods
247
- end