wukong 3.0.0.pre → 3.0.0.pre2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (476) hide show
  1. data/.gitignore +46 -33
  2. data/.gitmodules +3 -0
  3. data/.rspec +1 -1
  4. data/.travis.yml +8 -1
  5. data/.yardopts +0 -13
  6. data/Guardfile +4 -6
  7. data/{LICENSE.textile → LICENSE.md} +43 -55
  8. data/README-old.md +422 -0
  9. data/README.md +279 -418
  10. data/Rakefile +21 -5
  11. data/TODO.md +6 -6
  12. data/bin/wu-clean-encoding +31 -0
  13. data/bin/wu-lign +2 -2
  14. data/bin/wu-local +69 -0
  15. data/bin/wu-server +70 -0
  16. data/examples/Gemfile +38 -0
  17. data/examples/README.md +9 -0
  18. data/examples/dataflow/apache_log_line.rb +64 -25
  19. data/examples/dataflow/fibonacci_series.rb +101 -0
  20. data/examples/dataflow/parse_apache_logs.rb +37 -7
  21. data/examples/{dataflow.rb → dataflow/scraper_macro_flow.rb} +0 -0
  22. data/examples/dataflow/simple.rb +4 -4
  23. data/examples/geo.rb +4 -0
  24. data/examples/geo/geo_grids.numbers +0 -0
  25. data/examples/geo/geolocated.rb +331 -0
  26. data/examples/geo/quadtile.rb +69 -0
  27. data/examples/geo/spec/geolocated_spec.rb +247 -0
  28. data/examples/geo/tile_fetcher.rb +77 -0
  29. data/examples/graph/minimum_spanning_tree.rb +61 -61
  30. data/examples/jabberwocky.txt +36 -0
  31. data/examples/models/wikipedia.rb +20 -0
  32. data/examples/munging/Gemfile +8 -0
  33. data/examples/munging/airline_flights/airline.rb +57 -0
  34. data/examples/munging/airline_flights/airline_flights.rake +83 -0
  35. data/{lib/wukong/settings.rb → examples/munging/airline_flights/airplane.rb} +0 -0
  36. data/examples/munging/airline_flights/airport.rb +211 -0
  37. data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
  38. data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
  39. data/examples/munging/airline_flights/flight.rb +156 -0
  40. data/examples/munging/airline_flights/models.rb +4 -0
  41. data/examples/munging/airline_flights/parse.rb +26 -0
  42. data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
  43. data/examples/munging/airline_flights/route.rb +35 -0
  44. data/examples/munging/airline_flights/tasks.rake +83 -0
  45. data/examples/munging/airline_flights/timezone_fixup.rb +62 -0
  46. data/examples/munging/airline_flights/topcities.rb +167 -0
  47. data/examples/munging/airports/40_wbans.txt +40 -0
  48. data/examples/munging/airports/filter_weather_reports.rb +37 -0
  49. data/examples/munging/airports/join.pig +31 -0
  50. data/examples/munging/airports/to_tsv.rb +33 -0
  51. data/examples/munging/airports/usa_wbans.pig +19 -0
  52. data/examples/munging/airports/usa_wbans.txt +2157 -0
  53. data/examples/munging/airports/wbans.pig +19 -0
  54. data/examples/munging/airports/wbans.txt +2310 -0
  55. data/examples/munging/geo/geo_json.rb +54 -0
  56. data/examples/munging/geo/geo_models.rb +69 -0
  57. data/examples/munging/geo/geonames_models.rb +78 -0
  58. data/examples/munging/geo/iso_codes.rb +172 -0
  59. data/examples/munging/geo/reconcile_countries.rb +124 -0
  60. data/examples/munging/geo/tasks.rake +71 -0
  61. data/examples/munging/rake_helper.rb +62 -0
  62. data/examples/munging/weather/.gitignore +1 -0
  63. data/examples/munging/weather/Gemfile +4 -0
  64. data/examples/munging/weather/Rakefile +28 -0
  65. data/examples/munging/weather/extract_ish.rb +13 -0
  66. data/examples/munging/weather/models/weather.rb +119 -0
  67. data/examples/munging/weather/utils/noaa_downloader.rb +46 -0
  68. data/examples/munging/wikipedia/README.md +34 -0
  69. data/examples/munging/wikipedia/Rakefile +193 -0
  70. data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
  71. data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
  72. data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
  73. data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
  74. data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
  75. data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
  76. data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
  77. data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +4 -0
  78. data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
  79. data/examples/munging/wikipedia/dbpedia/extract_links.rb +193 -0
  80. data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
  81. data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig +18 -0
  82. data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb +21 -0
  83. data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old +27 -0
  84. data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig +29 -0
  85. data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb +14 -0
  86. data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old +25 -0
  87. data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig +29 -0
  88. data/examples/munging/wikipedia/pageviews/augment_pageviews.pig +32 -0
  89. data/examples/munging/wikipedia/pageviews/extract_pageviews.rb +85 -0
  90. data/examples/munging/wikipedia/pig_style_guide.md +25 -0
  91. data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig +19 -0
  92. data/examples/munging/wikipedia/subuniverse/sub_articles.pig +23 -0
  93. data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig +24 -0
  94. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig +22 -0
  95. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig +22 -0
  96. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig +26 -0
  97. data/examples/munging/wikipedia/subuniverse/sub_pageviews.pig +29 -0
  98. data/examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig +24 -0
  99. data/examples/munging/wikipedia/utils/get_namespaces.rb +86 -0
  100. data/examples/munging/wikipedia/utils/munging_utils.rb +68 -0
  101. data/examples/munging/wikipedia/utils/namespaces.json +1 -0
  102. data/examples/rake_helper.rb +85 -0
  103. data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
  104. data/examples/server_logs/logline.rb +95 -0
  105. data/examples/server_logs/models.rb +66 -0
  106. data/examples/server_logs/page_counts.pig +48 -0
  107. data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
  108. data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
  109. data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
  110. data/{old/examples/server_logs/breadcrumbs.rb → examples/server_logs/server_logs-03-breadcrumbs-full.rb} +26 -30
  111. data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
  112. data/examples/string_reverser.rb +26 -0
  113. data/examples/text/pig_latin.rb +2 -2
  114. data/examples/text/regional_flavor/README.md +14 -0
  115. data/examples/text/regional_flavor/article_wordbags.pig +39 -0
  116. data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
  117. data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
  118. data/examples/word_count/accumulator.rb +26 -0
  119. data/examples/word_count/tokenizer.rb +13 -0
  120. data/examples/word_count/word_count.rb +6 -0
  121. data/examples/workflow/cherry_pie.dot +97 -0
  122. data/examples/workflow/cherry_pie.png +0 -0
  123. data/examples/workflow/cherry_pie.rb +61 -26
  124. data/lib/hanuman.rb +34 -7
  125. data/lib/hanuman/graph.rb +55 -31
  126. data/lib/hanuman/graphvizzer.rb +199 -178
  127. data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
  128. data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
  129. data/lib/hanuman/link.rb +35 -0
  130. data/lib/hanuman/registry.rb +46 -0
  131. data/lib/hanuman/stage.rb +76 -32
  132. data/lib/wukong.rb +23 -24
  133. data/lib/wukong/boot.rb +87 -0
  134. data/lib/wukong/configuration.rb +8 -0
  135. data/lib/wukong/dataflow.rb +45 -78
  136. data/lib/wukong/driver.rb +99 -0
  137. data/lib/wukong/emitter.rb +22 -0
  138. data/lib/wukong/model/faker.rb +24 -24
  139. data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
  140. data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
  141. data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
  142. data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
  143. data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
  144. data/lib/wukong/processor.rb +60 -114
  145. data/lib/wukong/spec_helpers.rb +81 -0
  146. data/lib/wukong/spec_helpers/integration_driver.rb +144 -0
  147. data/lib/wukong/spec_helpers/integration_driver_matchers.rb +219 -0
  148. data/lib/wukong/spec_helpers/processor_helpers.rb +95 -0
  149. data/lib/wukong/spec_helpers/processor_methods.rb +108 -0
  150. data/lib/wukong/spec_helpers/shared_examples.rb +15 -0
  151. data/lib/wukong/spec_helpers/spec_driver.rb +28 -0
  152. data/lib/wukong/spec_helpers/spec_driver_matchers.rb +195 -0
  153. data/lib/wukong/version.rb +2 -1
  154. data/lib/wukong/widget/filters.rb +311 -0
  155. data/lib/wukong/widget/processors.rb +156 -0
  156. data/lib/wukong/widget/reducers.rb +7 -0
  157. data/lib/wukong/widget/reducers/accumulator.rb +73 -0
  158. data/lib/wukong/widget/reducers/bin.rb +318 -0
  159. data/lib/wukong/widget/reducers/count.rb +61 -0
  160. data/lib/wukong/widget/reducers/group.rb +85 -0
  161. data/lib/wukong/widget/reducers/group_concat.rb +70 -0
  162. data/lib/wukong/widget/reducers/moments.rb +72 -0
  163. data/lib/wukong/widget/reducers/sort.rb +130 -0
  164. data/lib/wukong/widget/serializers.rb +287 -0
  165. data/lib/wukong/widget/sink.rb +10 -52
  166. data/lib/wukong/widget/source.rb +7 -113
  167. data/lib/wukong/widget/utils.rb +46 -0
  168. data/lib/wukong/widgets.rb +6 -0
  169. data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
  170. data/spec/examples/dataflow/parsing_spec.rb +12 -11
  171. data/spec/examples/dataflow/simple_spec.rb +32 -6
  172. data/spec/examples/dataflow/telegram_spec.rb +36 -36
  173. data/spec/examples/graph/minimum_spanning_tree_spec.rb +30 -31
  174. data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
  175. data/spec/examples/munging/airline_flights_spec.rb +202 -0
  176. data/spec/examples/text/pig_latin_spec.rb +13 -16
  177. data/spec/examples/workflow/cherry_pie_spec.rb +34 -4
  178. data/spec/hanuman/graph_spec.rb +27 -2
  179. data/spec/hanuman/hanuman_spec.rb +10 -0
  180. data/spec/hanuman/registry_spec.rb +123 -0
  181. data/spec/hanuman/stage_spec.rb +61 -7
  182. data/spec/spec_helper.rb +29 -19
  183. data/spec/support/hanuman_test_helpers.rb +14 -12
  184. data/spec/support/shared_context_for_reducers.rb +37 -0
  185. data/spec/support/shared_examples_for_builders.rb +101 -0
  186. data/spec/support/shared_examples_for_shortcuts.rb +57 -0
  187. data/spec/support/wukong_test_helpers.rb +37 -11
  188. data/spec/wukong/dataflow_spec.rb +77 -55
  189. data/spec/wukong/local_runner_spec.rb +24 -24
  190. data/spec/wukong/model/faker_spec.rb +132 -131
  191. data/spec/wukong/runner_spec.rb +8 -8
  192. data/spec/wukong/widget/filters_spec.rb +61 -0
  193. data/spec/wukong/widget/processors_spec.rb +126 -0
  194. data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
  195. data/spec/wukong/widget/reducers/count_spec.rb +11 -0
  196. data/spec/wukong/widget/reducers/group_spec.rb +20 -0
  197. data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
  198. data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
  199. data/spec/wukong/widget/serializers_spec.rb +92 -0
  200. data/spec/wukong/widget/sink_spec.rb +15 -15
  201. data/spec/wukong/widget/source_spec.rb +65 -41
  202. data/spec/wukong/wukong_spec.rb +10 -0
  203. data/wukong.gemspec +17 -10
  204. metadata +359 -335
  205. data/.document +0 -5
  206. data/VERSION +0 -1
  207. data/bin/hdp-bin +0 -44
  208. data/bin/hdp-bzip +0 -23
  209. data/bin/hdp-cat +0 -3
  210. data/bin/hdp-catd +0 -3
  211. data/bin/hdp-cp +0 -3
  212. data/bin/hdp-du +0 -86
  213. data/bin/hdp-get +0 -3
  214. data/bin/hdp-kill +0 -3
  215. data/bin/hdp-kill-task +0 -3
  216. data/bin/hdp-ls +0 -11
  217. data/bin/hdp-mkdir +0 -2
  218. data/bin/hdp-mkdirp +0 -12
  219. data/bin/hdp-mv +0 -3
  220. data/bin/hdp-parts_to_keys.rb +0 -77
  221. data/bin/hdp-ps +0 -3
  222. data/bin/hdp-put +0 -3
  223. data/bin/hdp-rm +0 -32
  224. data/bin/hdp-sort +0 -40
  225. data/bin/hdp-stream +0 -40
  226. data/bin/hdp-stream-flat +0 -22
  227. data/bin/hdp-stream2 +0 -39
  228. data/bin/hdp-sync +0 -17
  229. data/bin/hdp-wc +0 -67
  230. data/bin/wu-flow +0 -10
  231. data/bin/wu-map +0 -17
  232. data/bin/wu-red +0 -17
  233. data/bin/wukong +0 -17
  234. data/data/CREDITS.md +0 -355
  235. data/data/graph/airfares.tsv +0 -2174
  236. data/data/text/gift_of_the_magi.txt +0 -225
  237. data/data/text/jabberwocky.txt +0 -36
  238. data/data/text/rectification_of_names.txt +0 -33
  239. data/data/twitter/a_atsigns_b.tsv +0 -64
  240. data/data/twitter/a_follows_b.tsv +0 -53
  241. data/data/twitter/tweet.tsv +0 -167
  242. data/data/twitter/twitter_user.tsv +0 -55
  243. data/data/wikipedia/dbpedia-sentences.tsv +0 -1000
  244. data/docpages/INSTALL.textile +0 -92
  245. data/docpages/LICENSE.textile +0 -107
  246. data/docpages/README-elastic_map_reduce.textile +0 -377
  247. data/docpages/README-performance.textile +0 -90
  248. data/docpages/README-wulign.textile +0 -65
  249. data/docpages/UsingWukong-part1-get_ready.textile +0 -17
  250. data/docpages/UsingWukong-part2-ThinkingBigData.textile +0 -75
  251. data/docpages/UsingWukong-part3-parsing.textile +0 -138
  252. data/docpages/_config.yml +0 -39
  253. data/docpages/avro/avro_notes.textile +0 -56
  254. data/docpages/avro/performance.textile +0 -36
  255. data/docpages/avro/tethering.textile +0 -19
  256. data/docpages/bigdata-tips.textile +0 -143
  257. data/docpages/code/api_response_example.txt +0 -20
  258. data/docpages/code/parser_skeleton.rb +0 -38
  259. data/docpages/diagrams/MapReduceDiagram.graffle +0 -0
  260. data/docpages/favicon.ico +0 -0
  261. data/docpages/gem.css +0 -16
  262. data/docpages/hadoop-tips.textile +0 -83
  263. data/docpages/index.textile +0 -92
  264. data/docpages/intro.textile +0 -8
  265. data/docpages/moreinfo.textile +0 -174
  266. data/docpages/news.html +0 -24
  267. data/docpages/pig/PigLatinExpressionsList.txt +0 -122
  268. data/docpages/pig/PigLatinReferenceManual.txt +0 -1640
  269. data/docpages/pig/commandline_params.txt +0 -26
  270. data/docpages/pig/cookbook.html +0 -481
  271. data/docpages/pig/images/hadoop-logo.jpg +0 -0
  272. data/docpages/pig/images/instruction_arrow.png +0 -0
  273. data/docpages/pig/images/pig-logo.gif +0 -0
  274. data/docpages/pig/piglatin_ref1.html +0 -1103
  275. data/docpages/pig/piglatin_ref2.html +0 -14340
  276. data/docpages/pig/setup.html +0 -505
  277. data/docpages/pig/skin/basic.css +0 -166
  278. data/docpages/pig/skin/breadcrumbs.js +0 -237
  279. data/docpages/pig/skin/fontsize.js +0 -166
  280. data/docpages/pig/skin/getBlank.js +0 -40
  281. data/docpages/pig/skin/getMenu.js +0 -45
  282. data/docpages/pig/skin/images/chapter.gif +0 -0
  283. data/docpages/pig/skin/images/chapter_open.gif +0 -0
  284. data/docpages/pig/skin/images/current.gif +0 -0
  285. data/docpages/pig/skin/images/external-link.gif +0 -0
  286. data/docpages/pig/skin/images/header_white_line.gif +0 -0
  287. data/docpages/pig/skin/images/page.gif +0 -0
  288. data/docpages/pig/skin/images/pdfdoc.gif +0 -0
  289. data/docpages/pig/skin/images/rc-b-l-15-1body-2menu-3menu.png +0 -0
  290. data/docpages/pig/skin/images/rc-b-r-15-1body-2menu-3menu.png +0 -0
  291. data/docpages/pig/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png +0 -0
  292. data/docpages/pig/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png +0 -0
  293. data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png +0 -0
  294. data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png +0 -0
  295. data/docpages/pig/skin/images/rc-t-r-15-1body-2menu-3menu.png +0 -0
  296. data/docpages/pig/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png +0 -0
  297. data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png +0 -0
  298. data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png +0 -0
  299. data/docpages/pig/skin/print.css +0 -54
  300. data/docpages/pig/skin/profile.css +0 -181
  301. data/docpages/pig/skin/screen.css +0 -587
  302. data/docpages/pig/tutorial.html +0 -1059
  303. data/docpages/pig/udf.html +0 -1509
  304. data/docpages/tutorial.textile +0 -283
  305. data/docpages/usage.textile +0 -195
  306. data/docpages/wutils.textile +0 -263
  307. data/examples/dataflow/complex.rb +0 -11
  308. data/examples/dataflow/donuts.rb +0 -13
  309. data/examples/tiny_count/jabberwocky_output.tsv +0 -92
  310. data/examples/word_count.rb +0 -48
  311. data/examples/workflow/fiddle.rb +0 -24
  312. data/lib/away/escapement.rb +0 -129
  313. data/lib/away/exe.rb +0 -11
  314. data/lib/away/experimental.rb +0 -5
  315. data/lib/away/from_file.rb +0 -52
  316. data/lib/away/job.rb +0 -56
  317. data/lib/away/job/rake_compat.rb +0 -17
  318. data/lib/away/registry.rb +0 -79
  319. data/lib/away/runner.rb +0 -276
  320. data/lib/away/runner/execute.rb +0 -121
  321. data/lib/away/script.rb +0 -161
  322. data/lib/away/script/hadoop_command.rb +0 -240
  323. data/lib/away/source/file_list_source.rb +0 -15
  324. data/lib/away/source/looper.rb +0 -18
  325. data/lib/away/task.rb +0 -219
  326. data/lib/hanuman/action.rb +0 -21
  327. data/lib/hanuman/chain.rb +0 -4
  328. data/lib/hanuman/graphviz.rb +0 -74
  329. data/lib/hanuman/resource.rb +0 -6
  330. data/lib/hanuman/slot.rb +0 -87
  331. data/lib/hanuman/slottable.rb +0 -220
  332. data/lib/wukong/bad_record.rb +0 -15
  333. data/lib/wukong/event.rb +0 -44
  334. data/lib/wukong/local_runner.rb +0 -55
  335. data/lib/wukong/mapred.rb +0 -3
  336. data/lib/wukong/universe.rb +0 -48
  337. data/lib/wukong/widget/filter.rb +0 -81
  338. data/lib/wukong/widget/gibberish.rb +0 -123
  339. data/lib/wukong/widget/monitor.rb +0 -26
  340. data/lib/wukong/widget/reducer.rb +0 -66
  341. data/lib/wukong/widget/stringifier.rb +0 -50
  342. data/lib/wukong/workflow.rb +0 -22
  343. data/lib/wukong/workflow/command.rb +0 -42
  344. data/old/config/emr-example.yaml +0 -48
  345. data/old/examples/README.txt +0 -17
  346. data/old/examples/contrib/jeans/README.markdown +0 -165
  347. data/old/examples/contrib/jeans/data/normalized_sizes +0 -3
  348. data/old/examples/contrib/jeans/data/orders.tsv +0 -1302
  349. data/old/examples/contrib/jeans/data/sizes +0 -3
  350. data/old/examples/contrib/jeans/normalize.rb +0 -20
  351. data/old/examples/contrib/jeans/sizes.rb +0 -55
  352. data/old/examples/corpus/bnc_word_freq.rb +0 -44
  353. data/old/examples/corpus/bucket_counter.rb +0 -47
  354. data/old/examples/corpus/dbpedia_abstract_to_sentences.rb +0 -86
  355. data/old/examples/corpus/sentence_bigrams.rb +0 -53
  356. data/old/examples/corpus/sentence_coocurrence.rb +0 -66
  357. data/old/examples/corpus/stopwords.rb +0 -138
  358. data/old/examples/corpus/words_to_bigrams.rb +0 -53
  359. data/old/examples/emr/README.textile +0 -110
  360. data/old/examples/emr/dot_wukong_dir/credentials.json +0 -7
  361. data/old/examples/emr/dot_wukong_dir/emr.yaml +0 -69
  362. data/old/examples/emr/dot_wukong_dir/emr_bootstrap.sh +0 -33
  363. data/old/examples/emr/elastic_mapreduce_example.rb +0 -28
  364. data/old/examples/network_graph/adjacency_list.rb +0 -74
  365. data/old/examples/network_graph/breadth_first_search.rb +0 -72
  366. data/old/examples/network_graph/gen_2paths.rb +0 -68
  367. data/old/examples/network_graph/gen_multi_edge.rb +0 -112
  368. data/old/examples/network_graph/gen_symmetric_links.rb +0 -64
  369. data/old/examples/pagerank/README.textile +0 -6
  370. data/old/examples/pagerank/gen_initial_pagerank_graph.pig +0 -57
  371. data/old/examples/pagerank/pagerank.rb +0 -72
  372. data/old/examples/pagerank/pagerank_initialize.rb +0 -42
  373. data/old/examples/pagerank/run_pagerank.sh +0 -21
  374. data/old/examples/sample_records.rb +0 -33
  375. data/old/examples/server_logs/apache_log_parser.rb +0 -15
  376. data/old/examples/server_logs/nook.rb +0 -48
  377. data/old/examples/server_logs/nook/faraday_dummy_adapter.rb +0 -94
  378. data/old/examples/server_logs/user_agent.rb +0 -40
  379. data/old/examples/simple_word_count.rb +0 -82
  380. data/old/examples/size.rb +0 -61
  381. data/old/examples/stats/avg_value_frequency.rb +0 -86
  382. data/old/examples/stats/binning_percentile_estimator.rb +0 -140
  383. data/old/examples/stats/data/avg_value_frequency.tsv +0 -3
  384. data/old/examples/stats/rank_and_bin.rb +0 -173
  385. data/old/examples/stupidly_simple_filter.rb +0 -40
  386. data/old/examples/word_count.rb +0 -75
  387. data/old/graph/graphviz_builder.rb +0 -580
  388. data/old/graph_easy/Attributes.pm +0 -4181
  389. data/old/graph_easy/Graphviz.pm +0 -2232
  390. data/old/wukong.rb +0 -18
  391. data/old/wukong/and_pig.rb +0 -38
  392. data/old/wukong/bad_record.rb +0 -18
  393. data/old/wukong/datatypes.rb +0 -24
  394. data/old/wukong/datatypes/enum.rb +0 -127
  395. data/old/wukong/datatypes/fake_types.rb +0 -17
  396. data/old/wukong/decorator.rb +0 -28
  397. data/old/wukong/encoding/asciize.rb +0 -108
  398. data/old/wukong/extensions.rb +0 -16
  399. data/old/wukong/extensions/array.rb +0 -18
  400. data/old/wukong/extensions/blank.rb +0 -93
  401. data/old/wukong/extensions/class.rb +0 -189
  402. data/old/wukong/extensions/date_time.rb +0 -53
  403. data/old/wukong/extensions/emittable.rb +0 -69
  404. data/old/wukong/extensions/enumerable.rb +0 -79
  405. data/old/wukong/extensions/hash.rb +0 -167
  406. data/old/wukong/extensions/hash_keys.rb +0 -16
  407. data/old/wukong/extensions/hash_like.rb +0 -150
  408. data/old/wukong/extensions/hashlike_class.rb +0 -47
  409. data/old/wukong/extensions/module.rb +0 -2
  410. data/old/wukong/extensions/pathname.rb +0 -27
  411. data/old/wukong/extensions/string.rb +0 -65
  412. data/old/wukong/extensions/struct.rb +0 -17
  413. data/old/wukong/extensions/symbol.rb +0 -11
  414. data/old/wukong/filename_pattern.rb +0 -74
  415. data/old/wukong/helper.rb +0 -7
  416. data/old/wukong/helper/stopwords.rb +0 -195
  417. data/old/wukong/helper/tokenize.rb +0 -35
  418. data/old/wukong/logger.rb +0 -38
  419. data/old/wukong/periodic_monitor.rb +0 -72
  420. data/old/wukong/schema.rb +0 -269
  421. data/old/wukong/script.rb +0 -286
  422. data/old/wukong/script/avro_command.rb +0 -5
  423. data/old/wukong/script/cassandra_loader_script.rb +0 -40
  424. data/old/wukong/script/emr_command.rb +0 -168
  425. data/old/wukong/script/hadoop_command.rb +0 -237
  426. data/old/wukong/script/local_command.rb +0 -41
  427. data/old/wukong/store.rb +0 -10
  428. data/old/wukong/store/base.rb +0 -27
  429. data/old/wukong/store/cassandra.rb +0 -10
  430. data/old/wukong/store/cassandra/streaming.rb +0 -75
  431. data/old/wukong/store/cassandra/struct_loader.rb +0 -21
  432. data/old/wukong/store/cassandra_model.rb +0 -91
  433. data/old/wukong/store/chh_chunked_flat_file_store.rb +0 -37
  434. data/old/wukong/store/chunked_flat_file_store.rb +0 -48
  435. data/old/wukong/store/conditional_store.rb +0 -57
  436. data/old/wukong/store/factory.rb +0 -8
  437. data/old/wukong/store/flat_file_store.rb +0 -89
  438. data/old/wukong/store/key_store.rb +0 -51
  439. data/old/wukong/store/null_store.rb +0 -15
  440. data/old/wukong/store/read_thru_store.rb +0 -22
  441. data/old/wukong/store/tokyo_tdb_key_store.rb +0 -33
  442. data/old/wukong/store/tyrant_rdb_key_store.rb +0 -57
  443. data/old/wukong/store/tyrant_tdb_key_store.rb +0 -20
  444. data/old/wukong/streamer.rb +0 -30
  445. data/old/wukong/streamer/accumulating_reducer.rb +0 -83
  446. data/old/wukong/streamer/base.rb +0 -126
  447. data/old/wukong/streamer/counting_reducer.rb +0 -25
  448. data/old/wukong/streamer/filter.rb +0 -20
  449. data/old/wukong/streamer/instance_streamer.rb +0 -15
  450. data/old/wukong/streamer/json_streamer.rb +0 -21
  451. data/old/wukong/streamer/line_streamer.rb +0 -12
  452. data/old/wukong/streamer/list_reducer.rb +0 -31
  453. data/old/wukong/streamer/rank_and_bin_reducer.rb +0 -145
  454. data/old/wukong/streamer/record_streamer.rb +0 -14
  455. data/old/wukong/streamer/reducer.rb +0 -11
  456. data/old/wukong/streamer/set_reducer.rb +0 -14
  457. data/old/wukong/streamer/struct_streamer.rb +0 -48
  458. data/old/wukong/streamer/summing_reducer.rb +0 -29
  459. data/old/wukong/streamer/uniq_by_last_reducer.rb +0 -51
  460. data/old/wukong/typed_struct.rb +0 -12
  461. data/spec/away/encoding_spec.rb +0 -32
  462. data/spec/away/exe_spec.rb +0 -20
  463. data/spec/away/flow_spec.rb +0 -82
  464. data/spec/away/graph_spec.rb +0 -6
  465. data/spec/away/job_spec.rb +0 -15
  466. data/spec/away/rake_compat_spec.rb +0 -9
  467. data/spec/away/script_spec.rb +0 -81
  468. data/spec/hanuman/graphviz_spec.rb +0 -29
  469. data/spec/hanuman/slot_spec.rb +0 -2
  470. data/spec/support/examples_helper.rb +0 -10
  471. data/spec/support/streamer_test_helpers.rb +0 -6
  472. data/spec/support/wukong_widget_helpers.rb +0 -66
  473. data/spec/wukong/processor_spec.rb +0 -109
  474. data/spec/wukong/widget/filter_spec.rb +0 -99
  475. data/spec/wukong/widget/stringifier_spec.rb +0 -51
  476. data/spec/wukong/workflow/command_spec.rb +0 -5
@@ -1,189 +0,0 @@
1
- #
2
- # This is taken in whole from the extlib gem. Thanks y'all.
3
- #
4
-
5
-
6
- # Copyright (c) 2004-2008 David Heinemeier Hansson
7
- #
8
- # Permission is hereby granted, free of charge, to any person obtaining
9
- # a copy of this software and associated documentation files (the
10
- # "Software"), to deal in the Software without restriction, including
11
- # without limitation the rights to use, copy, modify, merge, publish,
12
- # distribute, sublicense, and/or sell copies of the Software, and to
13
- # permit persons to whom the Software is furnished to do so, subject to
14
- # the following conditions:
15
- #
16
- # The above copyright notice and this permission notice shall be
17
- # included in all copies or substantial portions of the Software.
18
- #
19
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22
- # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
23
- # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
24
- # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
25
- # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
-
27
- # Allows attributes to be shared within an inheritance hierarchy, but where
28
- # each descendant gets a copy of their parents' attributes, instead of just a
29
- # pointer to the same. This means that the child can add elements to, for
30
- # example, an array without those additions being shared with either their
31
- # parent, siblings, or children, which is unlike the regular class-level
32
- # attributes that are shared across the entire hierarchy.
33
- class Class
34
- # Defines class-level and instance-level attribute reader.
35
- #
36
- # @param *syms<Array> Array of attributes to define reader for.
37
- # @return <Array[#to_s]> List of attributes that were made into cattr_readers
38
- #
39
- # @api public
40
- #
41
- # @todo Is this inconsistent in that it does not allow you to prevent
42
- # an instance_reader via :instance_reader => false
43
- def cattr_reader(*syms)
44
- syms.flatten.each do |sym|
45
- next if sym.is_a?(Hash)
46
- class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
47
- unless defined? @@#{sym}
48
- @@#{sym} = nil
49
- end
50
-
51
- def self.#{sym}
52
- @@#{sym}
53
- end
54
-
55
- def #{sym}
56
- @@#{sym}
57
- end
58
- RUBY
59
- end
60
- end
61
-
62
- # Defines class-level (and optionally instance-level) attribute writer.
63
- #
64
- # @param <Array[*#to_s, Hash{:instance_writer => Boolean}]> Array of attributes to define writer for.
65
- # @option syms :instance_writer<Boolean> if true, instance-level attribute writer is defined.
66
- # @return <Array[#to_s]> List of attributes that were made into cattr_writers
67
- #
68
- # @api public
69
- def cattr_writer(*syms)
70
- options = syms.last.is_a?(Hash) ? syms.pop : {}
71
- syms.flatten.each do |sym|
72
- class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
73
- unless defined? @@#{sym}
74
- @@#{sym} = nil
75
- end
76
-
77
- def self.#{sym}=(obj)
78
- @@#{sym} = obj
79
- end
80
- RUBY
81
-
82
- unless options[:instance_writer] == false
83
- class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
84
- def #{sym}=(obj)
85
- @@#{sym} = obj
86
- end
87
- RUBY
88
- end
89
- end
90
- end
91
-
92
- # Defines class-level (and optionally instance-level) attribute accessor.
93
- #
94
- # @param *syms<Array[*#to_s, Hash{:instance_writer => Boolean}]> Array of attributes to define accessor for.
95
- # @option syms :instance_writer<Boolean> if true, instance-level attribute writer is defined.
96
- # @return <Array[#to_s]> List of attributes that were made into accessors
97
- #
98
- # @api public
99
- def cattr_accessor(*syms)
100
- cattr_reader(*syms)
101
- cattr_writer(*syms)
102
- end
103
-
104
- # Defines class-level inheritable attribute reader. Attributes are available to subclasses,
105
- # each subclass has a copy of parent's attribute.
106
- #
107
- # @param *syms<Array[#to_s]> Array of attributes to define inheritable reader for.
108
- # @return <Array[#to_s]> Array of attributes converted into inheritable_readers.
109
- #
110
- # @api public
111
- #
112
- # @todo Do we want to block instance_reader via :instance_reader => false
113
- # @todo It would be preferable that we do something with a Hash passed in
114
- # (error out or do the same as other methods above) instead of silently
115
- # moving on). In particular, this makes the return value of this function
116
- # less useful.
117
- def class_inheritable_reader(*ivars)
118
- instance_reader = ivars.pop[:reader] if ivars.last.is_a?(Hash)
119
-
120
- ivars.each do |ivar|
121
- self.class_eval <<-RUBY, __FILE__, __LINE__ + 1
122
- def self.#{ivar}
123
- return @#{ivar} if defined?(@#{ivar})
124
- return nil if self.object_id == #{self.object_id}
125
- ivar = superclass.#{ivar}
126
- return nil if ivar.nil?
127
- @#{ivar} = ivar.try_dup
128
- end
129
- RUBY
130
-
131
- unless instance_reader == false
132
- self.class_eval <<-RUBY, __FILE__, __LINE__ + 1
133
- def #{ivar}
134
- self.class.#{ivar}
135
- end
136
- RUBY
137
- end
138
- end
139
- end
140
-
141
- # Defines class-level inheritable attribute writer. Attributes are available to subclasses,
142
- # each subclass has a copy of parent's attribute.
143
- #
144
- # @param *syms<Array[*#to_s, Hash{:instance_writer => Boolean}]> Array of attributes to
145
- # define inheritable writer for.
146
- # @option syms :instance_writer<Boolean> if true, instance-level inheritable attribute writer is defined.
147
- # @return <Array[#to_s]> An Array of the attributes that were made into inheritable writers.
148
- #
149
- # @api public
150
- #
151
- # @todo We need a style for class_eval <<-HEREDOC. I'd like to make it
152
- # class_eval(<<-RUBY, __FILE__, __LINE__), but we should codify it somewhere.
153
- def class_inheritable_writer(*ivars)
154
- instance_writer = ivars.pop[:instance_writer] if ivars.last.is_a?(Hash)
155
- ivars.each do |ivar|
156
- self.class_eval <<-RUBY, __FILE__, __LINE__ + 1
157
- def self.#{ivar}=(obj)
158
- @#{ivar} = obj
159
- end
160
- RUBY
161
- unless instance_writer == false
162
- self.class_eval <<-RUBY, __FILE__, __LINE__ + 1
163
- def #{ivar}=(obj) self.class.#{ivar} = obj end
164
- RUBY
165
- end
166
- end
167
- end
168
-
169
- # Defines class-level inheritable attribute accessor. Attributes are available to subclasses,
170
- # each subclass has a copy of parent's attribute.
171
- #
172
- # @param *syms<Array[*#to_s, Hash{:instance_writer => Boolean}]> Array of attributes to
173
- # define inheritable accessor for.
174
- # @option syms :instance_writer<Boolean> if true, instance-level inheritable attribute writer is defined.
175
- # @return <Array[#to_s]> An Array of attributes turned into inheritable accessors.
176
- #
177
- # @api public
178
- def class_inheritable_accessor(*syms)
179
- class_inheritable_reader(*syms)
180
- class_inheritable_writer(*syms)
181
- end
182
- end
183
-
184
-
185
- class Fixnum
186
- def try_dup
187
- self
188
- end
189
- end
@@ -1,53 +0,0 @@
1
- require 'time'
2
- require 'date'
3
-
4
- class Time
5
- # strftime() format to flatten a date
6
- FLAT_FORMAT = "%Y%m%d%H%M%S"
7
- # Flatten
8
- def to_flat
9
- utc.strftime(FLAT_FORMAT)
10
- end
11
-
12
- #
13
- # Parses the time but never fails.
14
- # Return value is always in the UTC time zone.
15
- #
16
- # A flattened datetime -- a 14-digit YYYYmmddHHMMMSS -- is fixed to the UTC
17
- # time zone by parsing it as YYYYmmddHHMMMSSZ <- 'Z' at end
18
- #
19
- def self.parse_safely dt
20
- return nil if dt.blank?
21
- begin
22
- case
23
- when dt.is_a?(Time) then dt.utc
24
- when (dt.to_s =~ /\A\d{14}\z/) then parse(dt.to_s+'Z', true)
25
- else parse(dt.to_s, true).utc
26
- end
27
- rescue StandardError => e
28
- Log.debug e
29
- end
30
- end
31
-
32
- def self.parse_and_flatten str
33
- parse_safely(str).to_flat
34
- end
35
- end
36
-
37
- class DateTime < Date
38
- # strftime() format to flatten a date
39
- FLAT_FORMAT = "%Y%m%d%H%M%S"
40
- # Flatten
41
- def to_flat
42
- strftime(FLAT_FORMAT)
43
- end
44
- end
45
-
46
- class Date
47
- # strftime() format to flatten a date
48
- FLAT_FORMAT = "%Y%m%d"
49
- # Flatten
50
- def to_flat
51
- strftime(FLAT_FORMAT)
52
- end
53
- end
@@ -1,69 +0,0 @@
1
- Object.class_eval do
2
- def to_flat() [to_s] end
3
- end
4
-
5
- module Enumerable
6
- alias_method :to_flat, :to_a
7
- end
8
-
9
- Struct.class_eval do
10
- #
11
- # The last portion of the class in underscored form
12
- # note memoization
13
- #
14
- def self.resource_name
15
- @resource_name ||= self.to_s.gsub(%r{.*::}, '').underscore.to_sym
16
- end
17
- #
18
- # Flatten for packing as resource name followed by all fields
19
- #
20
- def to_flat include_key=false
21
- if include_key.is_a? Proc
22
- sort_key = include_key.call(self)
23
- elsif (! include_key.blank?) && respond_to?(:key)
24
- sort_key = [self.class.resource_name, key].flatten.join("-")
25
- else
26
- sort_key = self.class.resource_name
27
- end
28
- [sort_key, *to_a] # .map(&:to_flat).flatten
29
- end
30
- end
31
-
32
- module HashLike
33
- #
34
- # Flatten for packing as resource name followed by all fields
35
- #
36
- def to_flat include_key=true
37
- if include_key.is_a? Proc
38
- sort_key = include_key.call(self)
39
- elsif include_key && respond_to?(:key)
40
- sort_key = [self.class.resource_name, key].flatten.join("-")
41
- else
42
- sort_key = self.class.resource_name
43
- end
44
- [sort_key, *to_a] # .map(&:to_flat).flatten
45
- end
46
- end
47
-
48
- Hash.class_eval do
49
- def to_flat
50
- map do |k, v|
51
- [k.to_flat, v.to_flat].join(":")
52
- end
53
- end
54
- end
55
-
56
- class Integer
57
- #
58
- # Express boolean as 1 (true) or 0 (false). In contravention of typical ruby
59
- # semantics (but in a way that is more robust for wukong-like batch
60
- # processing), the number 0, the string '0', nil and false are all considered
61
- # false. (This also makes the method idempotent: repeated calls give same result.)
62
- #
63
- def self.unbooleanize bool
64
- case bool
65
- when 0, '0', false, nil then 0
66
- else 1
67
- end
68
- end
69
- end
@@ -1,79 +0,0 @@
1
- module Enumerable
2
-
3
- # Calculates a sum from the elements. Examples:
4
- #
5
- # payments.sum { |p| p.price * p.tax_rate }
6
- # payments.sum(&:price)
7
- #
8
- # The latter is a shortcut for:
9
- #
10
- # payments.inject { |sum, p| sum + p.price }
11
- #
12
- # It can also calculate the sum without the use of a block.
13
- #
14
- # [5, 15, 10].sum # => 30
15
- # ["foo", "bar"].sum # => "foobar"
16
- # [[1, 2], [3, 1, 5]].sum => [1, 2, 3, 1, 5]
17
- #
18
- # The default sum of an empty list is zero. You can override this default:
19
- #
20
- # [].sum(Payment.new(0)) { |i| i.amount } # => Payment.new(0)
21
- #
22
- def sum(identity = 0, &block)
23
- if block_given?
24
- map(&block).sum(identity)
25
- else
26
- inject { |sum, element| sum + element } || identity
27
- end
28
- end
29
-
30
- # Iterates over a collection, passing the current element *and* the
31
- # +memo+ to the block. Handy for building up hashes or
32
- # reducing collections down to one object. Examples:
33
- #
34
- # %w(foo bar).each_with_object({}) { |str, hsh| hsh[str] = str.upcase } #=> {'foo' => 'FOO', 'bar' => 'BAR'}
35
- #
36
- # *Note* that you can't use immutable objects like numbers, true or false as
37
- # the memo. You would think the following returns 120, but since the memo is
38
- # never changed, it does not.
39
- #
40
- # (1..5).each_with_object(1) { |value, memo| memo *= value } # => 1
41
- #
42
- def each_with_object(memo, &block)
43
- returning memo do |m|
44
- each do |element|
45
- block.call(element, m)
46
- end
47
- end
48
- end unless [].respond_to?(:each_with_object)
49
-
50
- # Convert an enumerable to a hash. Examples:
51
- #
52
- # people.index_by(&:login)
53
- # => { "nextangle" => <Person ...>, "chade-" => <Person ...>, ...}
54
- # people.index_by { |person| "#{person.first_name} #{person.last_name}" }
55
- # => { "Chade- Fowlersburg-e" => <Person ...>, "David Heinemeier Hansson" => <Person ...>, ...}
56
- #
57
- def index_by
58
- inject({}) do |accum, elem|
59
- accum[yield(elem)] = elem
60
- accum
61
- end
62
- end
63
-
64
- # Returns true if the collection has more than 1 element. Functionally equivalent to collection.size > 1.
65
- # Works with a block too ala any?, so people.many? { |p| p.age > 26 } # => returns true if more than 1 person is over 26.
66
- def many?(&block)
67
- size = block_given? ? select(&block).size : self.size
68
- size > 1
69
- end
70
-
71
- # Returns true if none of the elements match the given block.
72
- #
73
- # success = responses.none? {|r| r.status / 100 == 5 }
74
- #
75
- # This is a builtin method in Ruby 1.8.7 and later.
76
- def none?(&block)
77
- !any?(&block)
78
- end unless [].respond_to?(:none?)
79
- end
@@ -1,167 +0,0 @@
1
- #
2
- # h2. extensions/hash.rb -- hash extensions
3
- #
4
-
5
- require 'set'
6
- class Hash
7
-
8
- # Slice a hash to include only the given keys. This is useful for
9
- # limiting an options hash to valid keys before passing to a method:
10
- #
11
- # def search(criteria = {})
12
- # assert_valid_keys(:mass, :velocity, :time)
13
- # end
14
- #
15
- # search(options.slice(:mass, :velocity, :time))
16
- # Returns a new hash with only the given keys.
17
- def slice(*keys)
18
- allowed = Set.new(respond_to?(:convert_key) ? keys.map { |key| convert_key(key) } : keys)
19
- reject{|key,| !allowed.include?(key) }
20
- end
21
- #
22
- # Replace the hash with only the given keys.
23
- #
24
- def slice!(*keys)
25
- replace(slice(*keys))
26
- end
27
- #
28
- # #values_of is an alias for #values_at, but can be called on a Hash, a
29
- # Struct, or an instance of a class that includes HashLike
30
- #
31
- alias_method :values_of, :values_at
32
-
33
- #
34
- # Create a hash from an array of keys and corresponding values.
35
- #
36
- def self.zip(keys, values, default=nil, &block)
37
- hash = block_given? ? Hash.new(&block) : Hash.new(default)
38
- keys.zip(values){|key,val| hash[key]=val }
39
- hash
40
- end
41
-
42
- # lambda for recursive merges
43
- ::Hash::DEEP_MERGER = proc do |key,v1,v2|
44
- (v1.respond_to?(:merge) && v2.respond_to?(:merge)) ? v1.merge(v2.compact, &Hash::DEEP_MERGER) : (v2.nil? ? v1 : v2)
45
- end unless defined?(::Hash::DEEP_MERGER)
46
-
47
- #
48
- # Merge hashes recursively.
49
- # Nothing special happens to array values
50
- #
51
- # x = { :subhash => { 1 => :val_from_x, 222 => :only_in_x, 333 => :only_in_x }, :scalar => :scalar_from_x}
52
- # y = { :subhash => { 1 => :val_from_y, 999 => :only_in_y }, :scalar => :scalar_from_y }
53
- # x.deep_merge y
54
- # => {:subhash=>{1=>:val_from_y, 222=>:only_in_x, 333=>:only_in_x, 999=>:only_in_y}, :scalar=>:scalar_from_y}
55
- # y.deep_merge x
56
- # => {:subhash=>{1=>:val_from_x, 222=>:only_in_x, 333=>:only_in_x, 999=>:only_in_y}, :scalar=>:scalar_from_x}
57
- #
58
- # Nil values always lose.
59
- #
60
- # x = {:subhash=>{:nil_in_x=>nil, 1=>:val1,}, :nil_in_x=>nil}
61
- # y = {:subhash=>{:nil_in_x=>5}, :nil_in_x=>5}
62
- # y.deep_merge x
63
- # => {:subhash=>{1=>:val1, :nil_in_x=>5}, :nil_in_x=>5}
64
- # x.deep_merge y
65
- # => {:subhash=>{1=>:val1, :nil_in_x=>5}, :nil_in_x=>5}
66
- #
67
- def deep_merge hsh2
68
- merge hsh2, &Hash::DEEP_MERGER
69
- end unless method_defined?(:deep_merge)
70
-
71
- def deep_merge! hsh2
72
- merge! hsh2, &Hash::DEEP_MERGER
73
- end unless method_defined?(:deep_merge!)
74
-
75
- #
76
- # Treat hash as tree of hashes:
77
- #
78
- # x = { 1 => :val, :subhash => { 1 => :val1 } }
79
- # x.deep_set(:subhash, :cat, :hat)
80
- # # => { 1 => :val, :subhash => { 1 => :val1, :cat => :hat } }
81
- # x.deep_set(:subhash, 1, :newval)
82
- # # => { 1 => :val, :subhash => { 1 => :newval, :cat => :hat } }
83
- #
84
- #
85
- def deep_set *args
86
- val = args.pop
87
- last_key = args.pop
88
- # dig down to last subtree (building out if necessary)
89
- hsh = args.empty? ? self : args.inject(self){|h, k| h[k] ||= {} }
90
- # set leaf value
91
- hsh[last_key] = val
92
- end unless method_defined?(:deep_set)
93
-
94
- #
95
- # Treat hash as tree of hashes:
96
- #
97
- # x = { 1 => :val, :subhash => { 1 => :val1 } }
98
- # x.deep_get(:subhash, 1)
99
- # # => :val
100
- # x.deep_get(:subhash, 2)
101
- # # => nil
102
- # x.deep_get(:subhash, 2, 3)
103
- # # => nil
104
- # x.deep_get(:subhash, 2)
105
- # # => nil
106
- #
107
- def deep_get *args
108
- last_key = args.pop
109
- # dig down to last subtree (building out if necessary)
110
- hsh = args.inject(self){|h, k| h[k] || {} }
111
- # get leaf value
112
- hsh[last_key]
113
- end unless method_defined?(:deep_get)
114
-
115
-
116
- #
117
- # Treat hash as tree of hashes:
118
- #
119
- # x = { 1 => :val, :subhash => { 1 => :val1, 2 => :val2 } }
120
- # x.deep_delete(:subhash, 1)
121
- # #=> :val
122
- # x
123
- # #=> { 1 => :val, :subhash => { 2 => :val2 } }
124
- #
125
- def deep_delete *args
126
- last_key = args.pop
127
- last_hsh = args.empty? ? self : (deep_get(*args)||{})
128
- last_hsh.delete(last_key)
129
- end unless method_defined?(:deep_delete)
130
-
131
- #
132
- # remove all key-value pairs where the value is nil
133
- #
134
- def compact
135
- reject{|key,val| val.nil? }
136
- end unless method_defined?(:compact)
137
- #
138
- # Replace the hash with its compacted self
139
- #
140
- def compact!
141
- replace(compact)
142
- end unless method_defined?(:compact!)
143
-
144
- #
145
- # remove all key-value pairs where the value is blank
146
- #
147
- def compact_blank
148
- reject{|key,val| val.blank? }
149
- end
150
- #
151
- # Replace the hash with its compact_blank'ed self
152
- #
153
- def compact_blank!
154
- replace(compact_blank)
155
- end
156
-
157
- # Stolen from ActiveSupport::CoreExtensions::Hash::ReverseMerge.
158
- def reverse_merge(other_hash)
159
- other_hash.merge(self)
160
- end
161
-
162
- # Stolen from ActiveSupport::CoreExtensions::Hash::ReverseMerge.
163
- def reverse_merge!(other_hash)
164
- replace(reverse_merge(other_hash))
165
- end
166
-
167
- end