wukong 3.0.0.pre → 3.0.0.pre2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (476) hide show
  1. data/.gitignore +46 -33
  2. data/.gitmodules +3 -0
  3. data/.rspec +1 -1
  4. data/.travis.yml +8 -1
  5. data/.yardopts +0 -13
  6. data/Guardfile +4 -6
  7. data/{LICENSE.textile → LICENSE.md} +43 -55
  8. data/README-old.md +422 -0
  9. data/README.md +279 -418
  10. data/Rakefile +21 -5
  11. data/TODO.md +6 -6
  12. data/bin/wu-clean-encoding +31 -0
  13. data/bin/wu-lign +2 -2
  14. data/bin/wu-local +69 -0
  15. data/bin/wu-server +70 -0
  16. data/examples/Gemfile +38 -0
  17. data/examples/README.md +9 -0
  18. data/examples/dataflow/apache_log_line.rb +64 -25
  19. data/examples/dataflow/fibonacci_series.rb +101 -0
  20. data/examples/dataflow/parse_apache_logs.rb +37 -7
  21. data/examples/{dataflow.rb → dataflow/scraper_macro_flow.rb} +0 -0
  22. data/examples/dataflow/simple.rb +4 -4
  23. data/examples/geo.rb +4 -0
  24. data/examples/geo/geo_grids.numbers +0 -0
  25. data/examples/geo/geolocated.rb +331 -0
  26. data/examples/geo/quadtile.rb +69 -0
  27. data/examples/geo/spec/geolocated_spec.rb +247 -0
  28. data/examples/geo/tile_fetcher.rb +77 -0
  29. data/examples/graph/minimum_spanning_tree.rb +61 -61
  30. data/examples/jabberwocky.txt +36 -0
  31. data/examples/models/wikipedia.rb +20 -0
  32. data/examples/munging/Gemfile +8 -0
  33. data/examples/munging/airline_flights/airline.rb +57 -0
  34. data/examples/munging/airline_flights/airline_flights.rake +83 -0
  35. data/{lib/wukong/settings.rb → examples/munging/airline_flights/airplane.rb} +0 -0
  36. data/examples/munging/airline_flights/airport.rb +211 -0
  37. data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
  38. data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
  39. data/examples/munging/airline_flights/flight.rb +156 -0
  40. data/examples/munging/airline_flights/models.rb +4 -0
  41. data/examples/munging/airline_flights/parse.rb +26 -0
  42. data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
  43. data/examples/munging/airline_flights/route.rb +35 -0
  44. data/examples/munging/airline_flights/tasks.rake +83 -0
  45. data/examples/munging/airline_flights/timezone_fixup.rb +62 -0
  46. data/examples/munging/airline_flights/topcities.rb +167 -0
  47. data/examples/munging/airports/40_wbans.txt +40 -0
  48. data/examples/munging/airports/filter_weather_reports.rb +37 -0
  49. data/examples/munging/airports/join.pig +31 -0
  50. data/examples/munging/airports/to_tsv.rb +33 -0
  51. data/examples/munging/airports/usa_wbans.pig +19 -0
  52. data/examples/munging/airports/usa_wbans.txt +2157 -0
  53. data/examples/munging/airports/wbans.pig +19 -0
  54. data/examples/munging/airports/wbans.txt +2310 -0
  55. data/examples/munging/geo/geo_json.rb +54 -0
  56. data/examples/munging/geo/geo_models.rb +69 -0
  57. data/examples/munging/geo/geonames_models.rb +78 -0
  58. data/examples/munging/geo/iso_codes.rb +172 -0
  59. data/examples/munging/geo/reconcile_countries.rb +124 -0
  60. data/examples/munging/geo/tasks.rake +71 -0
  61. data/examples/munging/rake_helper.rb +62 -0
  62. data/examples/munging/weather/.gitignore +1 -0
  63. data/examples/munging/weather/Gemfile +4 -0
  64. data/examples/munging/weather/Rakefile +28 -0
  65. data/examples/munging/weather/extract_ish.rb +13 -0
  66. data/examples/munging/weather/models/weather.rb +119 -0
  67. data/examples/munging/weather/utils/noaa_downloader.rb +46 -0
  68. data/examples/munging/wikipedia/README.md +34 -0
  69. data/examples/munging/wikipedia/Rakefile +193 -0
  70. data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
  71. data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
  72. data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
  73. data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
  74. data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
  75. data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
  76. data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
  77. data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +4 -0
  78. data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
  79. data/examples/munging/wikipedia/dbpedia/extract_links.rb +193 -0
  80. data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
  81. data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig +18 -0
  82. data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb +21 -0
  83. data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old +27 -0
  84. data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig +29 -0
  85. data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb +14 -0
  86. data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old +25 -0
  87. data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig +29 -0
  88. data/examples/munging/wikipedia/pageviews/augment_pageviews.pig +32 -0
  89. data/examples/munging/wikipedia/pageviews/extract_pageviews.rb +85 -0
  90. data/examples/munging/wikipedia/pig_style_guide.md +25 -0
  91. data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig +19 -0
  92. data/examples/munging/wikipedia/subuniverse/sub_articles.pig +23 -0
  93. data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig +24 -0
  94. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig +22 -0
  95. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig +22 -0
  96. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig +26 -0
  97. data/examples/munging/wikipedia/subuniverse/sub_pageviews.pig +29 -0
  98. data/examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig +24 -0
  99. data/examples/munging/wikipedia/utils/get_namespaces.rb +86 -0
  100. data/examples/munging/wikipedia/utils/munging_utils.rb +68 -0
  101. data/examples/munging/wikipedia/utils/namespaces.json +1 -0
  102. data/examples/rake_helper.rb +85 -0
  103. data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
  104. data/examples/server_logs/logline.rb +95 -0
  105. data/examples/server_logs/models.rb +66 -0
  106. data/examples/server_logs/page_counts.pig +48 -0
  107. data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
  108. data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
  109. data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
  110. data/{old/examples/server_logs/breadcrumbs.rb → examples/server_logs/server_logs-03-breadcrumbs-full.rb} +26 -30
  111. data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
  112. data/examples/string_reverser.rb +26 -0
  113. data/examples/text/pig_latin.rb +2 -2
  114. data/examples/text/regional_flavor/README.md +14 -0
  115. data/examples/text/regional_flavor/article_wordbags.pig +39 -0
  116. data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
  117. data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
  118. data/examples/word_count/accumulator.rb +26 -0
  119. data/examples/word_count/tokenizer.rb +13 -0
  120. data/examples/word_count/word_count.rb +6 -0
  121. data/examples/workflow/cherry_pie.dot +97 -0
  122. data/examples/workflow/cherry_pie.png +0 -0
  123. data/examples/workflow/cherry_pie.rb +61 -26
  124. data/lib/hanuman.rb +34 -7
  125. data/lib/hanuman/graph.rb +55 -31
  126. data/lib/hanuman/graphvizzer.rb +199 -178
  127. data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
  128. data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
  129. data/lib/hanuman/link.rb +35 -0
  130. data/lib/hanuman/registry.rb +46 -0
  131. data/lib/hanuman/stage.rb +76 -32
  132. data/lib/wukong.rb +23 -24
  133. data/lib/wukong/boot.rb +87 -0
  134. data/lib/wukong/configuration.rb +8 -0
  135. data/lib/wukong/dataflow.rb +45 -78
  136. data/lib/wukong/driver.rb +99 -0
  137. data/lib/wukong/emitter.rb +22 -0
  138. data/lib/wukong/model/faker.rb +24 -24
  139. data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
  140. data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
  141. data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
  142. data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
  143. data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
  144. data/lib/wukong/processor.rb +60 -114
  145. data/lib/wukong/spec_helpers.rb +81 -0
  146. data/lib/wukong/spec_helpers/integration_driver.rb +144 -0
  147. data/lib/wukong/spec_helpers/integration_driver_matchers.rb +219 -0
  148. data/lib/wukong/spec_helpers/processor_helpers.rb +95 -0
  149. data/lib/wukong/spec_helpers/processor_methods.rb +108 -0
  150. data/lib/wukong/spec_helpers/shared_examples.rb +15 -0
  151. data/lib/wukong/spec_helpers/spec_driver.rb +28 -0
  152. data/lib/wukong/spec_helpers/spec_driver_matchers.rb +195 -0
  153. data/lib/wukong/version.rb +2 -1
  154. data/lib/wukong/widget/filters.rb +311 -0
  155. data/lib/wukong/widget/processors.rb +156 -0
  156. data/lib/wukong/widget/reducers.rb +7 -0
  157. data/lib/wukong/widget/reducers/accumulator.rb +73 -0
  158. data/lib/wukong/widget/reducers/bin.rb +318 -0
  159. data/lib/wukong/widget/reducers/count.rb +61 -0
  160. data/lib/wukong/widget/reducers/group.rb +85 -0
  161. data/lib/wukong/widget/reducers/group_concat.rb +70 -0
  162. data/lib/wukong/widget/reducers/moments.rb +72 -0
  163. data/lib/wukong/widget/reducers/sort.rb +130 -0
  164. data/lib/wukong/widget/serializers.rb +287 -0
  165. data/lib/wukong/widget/sink.rb +10 -52
  166. data/lib/wukong/widget/source.rb +7 -113
  167. data/lib/wukong/widget/utils.rb +46 -0
  168. data/lib/wukong/widgets.rb +6 -0
  169. data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
  170. data/spec/examples/dataflow/parsing_spec.rb +12 -11
  171. data/spec/examples/dataflow/simple_spec.rb +32 -6
  172. data/spec/examples/dataflow/telegram_spec.rb +36 -36
  173. data/spec/examples/graph/minimum_spanning_tree_spec.rb +30 -31
  174. data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
  175. data/spec/examples/munging/airline_flights_spec.rb +202 -0
  176. data/spec/examples/text/pig_latin_spec.rb +13 -16
  177. data/spec/examples/workflow/cherry_pie_spec.rb +34 -4
  178. data/spec/hanuman/graph_spec.rb +27 -2
  179. data/spec/hanuman/hanuman_spec.rb +10 -0
  180. data/spec/hanuman/registry_spec.rb +123 -0
  181. data/spec/hanuman/stage_spec.rb +61 -7
  182. data/spec/spec_helper.rb +29 -19
  183. data/spec/support/hanuman_test_helpers.rb +14 -12
  184. data/spec/support/shared_context_for_reducers.rb +37 -0
  185. data/spec/support/shared_examples_for_builders.rb +101 -0
  186. data/spec/support/shared_examples_for_shortcuts.rb +57 -0
  187. data/spec/support/wukong_test_helpers.rb +37 -11
  188. data/spec/wukong/dataflow_spec.rb +77 -55
  189. data/spec/wukong/local_runner_spec.rb +24 -24
  190. data/spec/wukong/model/faker_spec.rb +132 -131
  191. data/spec/wukong/runner_spec.rb +8 -8
  192. data/spec/wukong/widget/filters_spec.rb +61 -0
  193. data/spec/wukong/widget/processors_spec.rb +126 -0
  194. data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
  195. data/spec/wukong/widget/reducers/count_spec.rb +11 -0
  196. data/spec/wukong/widget/reducers/group_spec.rb +20 -0
  197. data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
  198. data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
  199. data/spec/wukong/widget/serializers_spec.rb +92 -0
  200. data/spec/wukong/widget/sink_spec.rb +15 -15
  201. data/spec/wukong/widget/source_spec.rb +65 -41
  202. data/spec/wukong/wukong_spec.rb +10 -0
  203. data/wukong.gemspec +17 -10
  204. metadata +359 -335
  205. data/.document +0 -5
  206. data/VERSION +0 -1
  207. data/bin/hdp-bin +0 -44
  208. data/bin/hdp-bzip +0 -23
  209. data/bin/hdp-cat +0 -3
  210. data/bin/hdp-catd +0 -3
  211. data/bin/hdp-cp +0 -3
  212. data/bin/hdp-du +0 -86
  213. data/bin/hdp-get +0 -3
  214. data/bin/hdp-kill +0 -3
  215. data/bin/hdp-kill-task +0 -3
  216. data/bin/hdp-ls +0 -11
  217. data/bin/hdp-mkdir +0 -2
  218. data/bin/hdp-mkdirp +0 -12
  219. data/bin/hdp-mv +0 -3
  220. data/bin/hdp-parts_to_keys.rb +0 -77
  221. data/bin/hdp-ps +0 -3
  222. data/bin/hdp-put +0 -3
  223. data/bin/hdp-rm +0 -32
  224. data/bin/hdp-sort +0 -40
  225. data/bin/hdp-stream +0 -40
  226. data/bin/hdp-stream-flat +0 -22
  227. data/bin/hdp-stream2 +0 -39
  228. data/bin/hdp-sync +0 -17
  229. data/bin/hdp-wc +0 -67
  230. data/bin/wu-flow +0 -10
  231. data/bin/wu-map +0 -17
  232. data/bin/wu-red +0 -17
  233. data/bin/wukong +0 -17
  234. data/data/CREDITS.md +0 -355
  235. data/data/graph/airfares.tsv +0 -2174
  236. data/data/text/gift_of_the_magi.txt +0 -225
  237. data/data/text/jabberwocky.txt +0 -36
  238. data/data/text/rectification_of_names.txt +0 -33
  239. data/data/twitter/a_atsigns_b.tsv +0 -64
  240. data/data/twitter/a_follows_b.tsv +0 -53
  241. data/data/twitter/tweet.tsv +0 -167
  242. data/data/twitter/twitter_user.tsv +0 -55
  243. data/data/wikipedia/dbpedia-sentences.tsv +0 -1000
  244. data/docpages/INSTALL.textile +0 -92
  245. data/docpages/LICENSE.textile +0 -107
  246. data/docpages/README-elastic_map_reduce.textile +0 -377
  247. data/docpages/README-performance.textile +0 -90
  248. data/docpages/README-wulign.textile +0 -65
  249. data/docpages/UsingWukong-part1-get_ready.textile +0 -17
  250. data/docpages/UsingWukong-part2-ThinkingBigData.textile +0 -75
  251. data/docpages/UsingWukong-part3-parsing.textile +0 -138
  252. data/docpages/_config.yml +0 -39
  253. data/docpages/avro/avro_notes.textile +0 -56
  254. data/docpages/avro/performance.textile +0 -36
  255. data/docpages/avro/tethering.textile +0 -19
  256. data/docpages/bigdata-tips.textile +0 -143
  257. data/docpages/code/api_response_example.txt +0 -20
  258. data/docpages/code/parser_skeleton.rb +0 -38
  259. data/docpages/diagrams/MapReduceDiagram.graffle +0 -0
  260. data/docpages/favicon.ico +0 -0
  261. data/docpages/gem.css +0 -16
  262. data/docpages/hadoop-tips.textile +0 -83
  263. data/docpages/index.textile +0 -92
  264. data/docpages/intro.textile +0 -8
  265. data/docpages/moreinfo.textile +0 -174
  266. data/docpages/news.html +0 -24
  267. data/docpages/pig/PigLatinExpressionsList.txt +0 -122
  268. data/docpages/pig/PigLatinReferenceManual.txt +0 -1640
  269. data/docpages/pig/commandline_params.txt +0 -26
  270. data/docpages/pig/cookbook.html +0 -481
  271. data/docpages/pig/images/hadoop-logo.jpg +0 -0
  272. data/docpages/pig/images/instruction_arrow.png +0 -0
  273. data/docpages/pig/images/pig-logo.gif +0 -0
  274. data/docpages/pig/piglatin_ref1.html +0 -1103
  275. data/docpages/pig/piglatin_ref2.html +0 -14340
  276. data/docpages/pig/setup.html +0 -505
  277. data/docpages/pig/skin/basic.css +0 -166
  278. data/docpages/pig/skin/breadcrumbs.js +0 -237
  279. data/docpages/pig/skin/fontsize.js +0 -166
  280. data/docpages/pig/skin/getBlank.js +0 -40
  281. data/docpages/pig/skin/getMenu.js +0 -45
  282. data/docpages/pig/skin/images/chapter.gif +0 -0
  283. data/docpages/pig/skin/images/chapter_open.gif +0 -0
  284. data/docpages/pig/skin/images/current.gif +0 -0
  285. data/docpages/pig/skin/images/external-link.gif +0 -0
  286. data/docpages/pig/skin/images/header_white_line.gif +0 -0
  287. data/docpages/pig/skin/images/page.gif +0 -0
  288. data/docpages/pig/skin/images/pdfdoc.gif +0 -0
  289. data/docpages/pig/skin/images/rc-b-l-15-1body-2menu-3menu.png +0 -0
  290. data/docpages/pig/skin/images/rc-b-r-15-1body-2menu-3menu.png +0 -0
  291. data/docpages/pig/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png +0 -0
  292. data/docpages/pig/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png +0 -0
  293. data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png +0 -0
  294. data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png +0 -0
  295. data/docpages/pig/skin/images/rc-t-r-15-1body-2menu-3menu.png +0 -0
  296. data/docpages/pig/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png +0 -0
  297. data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png +0 -0
  298. data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png +0 -0
  299. data/docpages/pig/skin/print.css +0 -54
  300. data/docpages/pig/skin/profile.css +0 -181
  301. data/docpages/pig/skin/screen.css +0 -587
  302. data/docpages/pig/tutorial.html +0 -1059
  303. data/docpages/pig/udf.html +0 -1509
  304. data/docpages/tutorial.textile +0 -283
  305. data/docpages/usage.textile +0 -195
  306. data/docpages/wutils.textile +0 -263
  307. data/examples/dataflow/complex.rb +0 -11
  308. data/examples/dataflow/donuts.rb +0 -13
  309. data/examples/tiny_count/jabberwocky_output.tsv +0 -92
  310. data/examples/word_count.rb +0 -48
  311. data/examples/workflow/fiddle.rb +0 -24
  312. data/lib/away/escapement.rb +0 -129
  313. data/lib/away/exe.rb +0 -11
  314. data/lib/away/experimental.rb +0 -5
  315. data/lib/away/from_file.rb +0 -52
  316. data/lib/away/job.rb +0 -56
  317. data/lib/away/job/rake_compat.rb +0 -17
  318. data/lib/away/registry.rb +0 -79
  319. data/lib/away/runner.rb +0 -276
  320. data/lib/away/runner/execute.rb +0 -121
  321. data/lib/away/script.rb +0 -161
  322. data/lib/away/script/hadoop_command.rb +0 -240
  323. data/lib/away/source/file_list_source.rb +0 -15
  324. data/lib/away/source/looper.rb +0 -18
  325. data/lib/away/task.rb +0 -219
  326. data/lib/hanuman/action.rb +0 -21
  327. data/lib/hanuman/chain.rb +0 -4
  328. data/lib/hanuman/graphviz.rb +0 -74
  329. data/lib/hanuman/resource.rb +0 -6
  330. data/lib/hanuman/slot.rb +0 -87
  331. data/lib/hanuman/slottable.rb +0 -220
  332. data/lib/wukong/bad_record.rb +0 -15
  333. data/lib/wukong/event.rb +0 -44
  334. data/lib/wukong/local_runner.rb +0 -55
  335. data/lib/wukong/mapred.rb +0 -3
  336. data/lib/wukong/universe.rb +0 -48
  337. data/lib/wukong/widget/filter.rb +0 -81
  338. data/lib/wukong/widget/gibberish.rb +0 -123
  339. data/lib/wukong/widget/monitor.rb +0 -26
  340. data/lib/wukong/widget/reducer.rb +0 -66
  341. data/lib/wukong/widget/stringifier.rb +0 -50
  342. data/lib/wukong/workflow.rb +0 -22
  343. data/lib/wukong/workflow/command.rb +0 -42
  344. data/old/config/emr-example.yaml +0 -48
  345. data/old/examples/README.txt +0 -17
  346. data/old/examples/contrib/jeans/README.markdown +0 -165
  347. data/old/examples/contrib/jeans/data/normalized_sizes +0 -3
  348. data/old/examples/contrib/jeans/data/orders.tsv +0 -1302
  349. data/old/examples/contrib/jeans/data/sizes +0 -3
  350. data/old/examples/contrib/jeans/normalize.rb +0 -20
  351. data/old/examples/contrib/jeans/sizes.rb +0 -55
  352. data/old/examples/corpus/bnc_word_freq.rb +0 -44
  353. data/old/examples/corpus/bucket_counter.rb +0 -47
  354. data/old/examples/corpus/dbpedia_abstract_to_sentences.rb +0 -86
  355. data/old/examples/corpus/sentence_bigrams.rb +0 -53
  356. data/old/examples/corpus/sentence_coocurrence.rb +0 -66
  357. data/old/examples/corpus/stopwords.rb +0 -138
  358. data/old/examples/corpus/words_to_bigrams.rb +0 -53
  359. data/old/examples/emr/README.textile +0 -110
  360. data/old/examples/emr/dot_wukong_dir/credentials.json +0 -7
  361. data/old/examples/emr/dot_wukong_dir/emr.yaml +0 -69
  362. data/old/examples/emr/dot_wukong_dir/emr_bootstrap.sh +0 -33
  363. data/old/examples/emr/elastic_mapreduce_example.rb +0 -28
  364. data/old/examples/network_graph/adjacency_list.rb +0 -74
  365. data/old/examples/network_graph/breadth_first_search.rb +0 -72
  366. data/old/examples/network_graph/gen_2paths.rb +0 -68
  367. data/old/examples/network_graph/gen_multi_edge.rb +0 -112
  368. data/old/examples/network_graph/gen_symmetric_links.rb +0 -64
  369. data/old/examples/pagerank/README.textile +0 -6
  370. data/old/examples/pagerank/gen_initial_pagerank_graph.pig +0 -57
  371. data/old/examples/pagerank/pagerank.rb +0 -72
  372. data/old/examples/pagerank/pagerank_initialize.rb +0 -42
  373. data/old/examples/pagerank/run_pagerank.sh +0 -21
  374. data/old/examples/sample_records.rb +0 -33
  375. data/old/examples/server_logs/apache_log_parser.rb +0 -15
  376. data/old/examples/server_logs/nook.rb +0 -48
  377. data/old/examples/server_logs/nook/faraday_dummy_adapter.rb +0 -94
  378. data/old/examples/server_logs/user_agent.rb +0 -40
  379. data/old/examples/simple_word_count.rb +0 -82
  380. data/old/examples/size.rb +0 -61
  381. data/old/examples/stats/avg_value_frequency.rb +0 -86
  382. data/old/examples/stats/binning_percentile_estimator.rb +0 -140
  383. data/old/examples/stats/data/avg_value_frequency.tsv +0 -3
  384. data/old/examples/stats/rank_and_bin.rb +0 -173
  385. data/old/examples/stupidly_simple_filter.rb +0 -40
  386. data/old/examples/word_count.rb +0 -75
  387. data/old/graph/graphviz_builder.rb +0 -580
  388. data/old/graph_easy/Attributes.pm +0 -4181
  389. data/old/graph_easy/Graphviz.pm +0 -2232
  390. data/old/wukong.rb +0 -18
  391. data/old/wukong/and_pig.rb +0 -38
  392. data/old/wukong/bad_record.rb +0 -18
  393. data/old/wukong/datatypes.rb +0 -24
  394. data/old/wukong/datatypes/enum.rb +0 -127
  395. data/old/wukong/datatypes/fake_types.rb +0 -17
  396. data/old/wukong/decorator.rb +0 -28
  397. data/old/wukong/encoding/asciize.rb +0 -108
  398. data/old/wukong/extensions.rb +0 -16
  399. data/old/wukong/extensions/array.rb +0 -18
  400. data/old/wukong/extensions/blank.rb +0 -93
  401. data/old/wukong/extensions/class.rb +0 -189
  402. data/old/wukong/extensions/date_time.rb +0 -53
  403. data/old/wukong/extensions/emittable.rb +0 -69
  404. data/old/wukong/extensions/enumerable.rb +0 -79
  405. data/old/wukong/extensions/hash.rb +0 -167
  406. data/old/wukong/extensions/hash_keys.rb +0 -16
  407. data/old/wukong/extensions/hash_like.rb +0 -150
  408. data/old/wukong/extensions/hashlike_class.rb +0 -47
  409. data/old/wukong/extensions/module.rb +0 -2
  410. data/old/wukong/extensions/pathname.rb +0 -27
  411. data/old/wukong/extensions/string.rb +0 -65
  412. data/old/wukong/extensions/struct.rb +0 -17
  413. data/old/wukong/extensions/symbol.rb +0 -11
  414. data/old/wukong/filename_pattern.rb +0 -74
  415. data/old/wukong/helper.rb +0 -7
  416. data/old/wukong/helper/stopwords.rb +0 -195
  417. data/old/wukong/helper/tokenize.rb +0 -35
  418. data/old/wukong/logger.rb +0 -38
  419. data/old/wukong/periodic_monitor.rb +0 -72
  420. data/old/wukong/schema.rb +0 -269
  421. data/old/wukong/script.rb +0 -286
  422. data/old/wukong/script/avro_command.rb +0 -5
  423. data/old/wukong/script/cassandra_loader_script.rb +0 -40
  424. data/old/wukong/script/emr_command.rb +0 -168
  425. data/old/wukong/script/hadoop_command.rb +0 -237
  426. data/old/wukong/script/local_command.rb +0 -41
  427. data/old/wukong/store.rb +0 -10
  428. data/old/wukong/store/base.rb +0 -27
  429. data/old/wukong/store/cassandra.rb +0 -10
  430. data/old/wukong/store/cassandra/streaming.rb +0 -75
  431. data/old/wukong/store/cassandra/struct_loader.rb +0 -21
  432. data/old/wukong/store/cassandra_model.rb +0 -91
  433. data/old/wukong/store/chh_chunked_flat_file_store.rb +0 -37
  434. data/old/wukong/store/chunked_flat_file_store.rb +0 -48
  435. data/old/wukong/store/conditional_store.rb +0 -57
  436. data/old/wukong/store/factory.rb +0 -8
  437. data/old/wukong/store/flat_file_store.rb +0 -89
  438. data/old/wukong/store/key_store.rb +0 -51
  439. data/old/wukong/store/null_store.rb +0 -15
  440. data/old/wukong/store/read_thru_store.rb +0 -22
  441. data/old/wukong/store/tokyo_tdb_key_store.rb +0 -33
  442. data/old/wukong/store/tyrant_rdb_key_store.rb +0 -57
  443. data/old/wukong/store/tyrant_tdb_key_store.rb +0 -20
  444. data/old/wukong/streamer.rb +0 -30
  445. data/old/wukong/streamer/accumulating_reducer.rb +0 -83
  446. data/old/wukong/streamer/base.rb +0 -126
  447. data/old/wukong/streamer/counting_reducer.rb +0 -25
  448. data/old/wukong/streamer/filter.rb +0 -20
  449. data/old/wukong/streamer/instance_streamer.rb +0 -15
  450. data/old/wukong/streamer/json_streamer.rb +0 -21
  451. data/old/wukong/streamer/line_streamer.rb +0 -12
  452. data/old/wukong/streamer/list_reducer.rb +0 -31
  453. data/old/wukong/streamer/rank_and_bin_reducer.rb +0 -145
  454. data/old/wukong/streamer/record_streamer.rb +0 -14
  455. data/old/wukong/streamer/reducer.rb +0 -11
  456. data/old/wukong/streamer/set_reducer.rb +0 -14
  457. data/old/wukong/streamer/struct_streamer.rb +0 -48
  458. data/old/wukong/streamer/summing_reducer.rb +0 -29
  459. data/old/wukong/streamer/uniq_by_last_reducer.rb +0 -51
  460. data/old/wukong/typed_struct.rb +0 -12
  461. data/spec/away/encoding_spec.rb +0 -32
  462. data/spec/away/exe_spec.rb +0 -20
  463. data/spec/away/flow_spec.rb +0 -82
  464. data/spec/away/graph_spec.rb +0 -6
  465. data/spec/away/job_spec.rb +0 -15
  466. data/spec/away/rake_compat_spec.rb +0 -9
  467. data/spec/away/script_spec.rb +0 -81
  468. data/spec/hanuman/graphviz_spec.rb +0 -29
  469. data/spec/hanuman/slot_spec.rb +0 -2
  470. data/spec/support/examples_helper.rb +0 -10
  471. data/spec/support/streamer_test_helpers.rb +0 -6
  472. data/spec/support/wukong_widget_helpers.rb +0 -66
  473. data/spec/wukong/processor_spec.rb +0 -109
  474. data/spec/wukong/widget/filter_spec.rb +0 -99
  475. data/spec/wukong/widget/stringifier_spec.rb +0 -51
  476. data/spec/wukong/workflow/command_spec.rb +0 -5
data/docpages/pig/setup.html DELETED
@@ -1,505 +0,0 @@
1
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
2
- <html>
3
- <head>
4
- <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
5
- <meta content="Apache Forrest" name="Generator">
6
- <meta name="Forrest-version" content="0.8">
7
- <meta name="Forrest-skin-name" content="pelt">
8
- <title>Pig Setup</title>
9
- <link type="text/css" href="skin/basic.css" rel="stylesheet">
10
- <link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
11
- <link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
12
- <link type="text/css" href="skin/profile.css" rel="stylesheet">
13
- <script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
14
- <link rel="shortcut icon" href="">
15
- </head>
16
- <body onload="init()">
17
- <script type="text/javascript">ndeSetTextSize();</script>
18
- <div id="top">
19
- <!--+
20
- |breadtrail
21
- +-->
22
- <div class="breadtrail">
23
- <a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/pig/">Pig</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
24
- </div>
25
- <!--+
26
- |header
27
- +-->
28
- <div class="header">
29
- <!--+
30
- |start group logo
31
- +-->
32
- <div class="grouplogo">
33
- <a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
34
- </div>
35
- <!--+
36
- |end group logo
37
- +-->
38
- <!--+
39
- |start Project Logo
40
- +-->
41
- <div class="projectlogo">
42
- <a href="http://hadoop.apache.org/pig/"><img class="logoImage" alt="Pig" src="images/pig-logo.gif" title="A platform for analyzing large datasets."></a>
43
- </div>
44
- <!--+
45
- |end Project Logo
46
- +-->
47
- <!--+
48
- |start Search
49
- +-->
50
- <div class="searchbox">
51
- <form action="http://www.google.com/search" method="get" class="roundtopsmall">
52
- <input value="" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
53
- <input name="Search" value="Search" type="submit">
54
- </form>
55
- </div>
56
- <!--+
57
- |end search
58
- +-->
59
- <!--+
60
- |start Tabs
61
- +-->
62
- <ul id="tabs">
63
- <li>
64
- <a class="unselected" href="http://hadoop.apache.org/pig/">Project</a>
65
- </li>
66
- <li>
67
- <a class="unselected" href="http://wiki.apache.org/pig/">Wiki</a>
68
- </li>
69
- <li class="current">
70
- <a class="selected" href="index.html">Pig 0.7.0 Documentation</a>
71
- </li>
72
- </ul>
73
- <!--+
74
- |end Tabs
75
- +-->
76
- </div>
77
- </div>
78
- <div id="main">
79
- <div id="publishedStrip">
80
- <!--+
81
- |start Subtabs
82
- +-->
83
- <div id="level2tabs"></div>
84
- <!--+
85
- |end Endtabs
86
- +-->
87
- <script type="text/javascript"><!--
88
- document.write("Last Published: " + document.lastModified);
89
- // --></script>
90
- </div>
91
- <!--+
92
- |breadtrail
93
- +-->
94
- <div class="breadtrail">
95
-
96
- &nbsp;
97
- </div>
98
- <!--+
99
- |start Menu, mainarea
100
- +-->
101
- <!--+
102
- |start Menu
103
- +-->
104
- <div id="menu">
105
- <div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Pig</div>
106
- <div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
107
- <div class="menuitem">
108
- <a href="index.html">Overview</a>
109
- </div>
110
- <div class="menupage">
111
- <div class="menupagetitle">Setup</div>
112
- </div>
113
- <div class="menuitem">
114
- <a href="tutorial.html">Tutorial</a>
115
- </div>
116
- <div class="menuitem">
117
- <a href="piglatin_ref1.html">Pig Latin 1</a>
118
- </div>
119
- <div class="menuitem">
120
- <a href="piglatin_ref2.html">Pig Latin 2</a>
121
- </div>
122
- <div class="menuitem">
123
- <a href="cookbook.html">Cookbook</a>
124
- </div>
125
- <div class="menuitem">
126
- <a href="udf.html">UDFs</a>
127
- </div>
128
- </div>
129
- <div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Zebra</div>
130
- <div id="menu_1.2" class="menuitemgroup">
131
- <div class="menuitem">
132
- <a href="zebra_overview.html">Zebra Overview </a>
133
- </div>
134
- <div class="menuitem">
135
- <a href="zebra_users.html">Zebra Users </a>
136
- </div>
137
- <div class="menuitem">
138
- <a href="zebra_reference.html">Zebra Reference </a>
139
- </div>
140
- <div class="menuitem">
141
- <a href="zebra_mapreduce.html">Zebra MapReduce </a>
142
- </div>
143
- <div class="menuitem">
144
- <a href="zebra_pig.html">Zebra Pig </a>
145
- </div>
146
- <div class="menuitem">
147
- <a href="zebra_stream.html">Zebra Streaming </a>
148
- </div>
149
- </div>
150
- <div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Miscellaneous</div>
151
- <div id="menu_1.3" class="menuitemgroup">
152
- <div class="menuitem">
153
- <a href="api/">API Docs</a>
154
- </div>
155
- <div class="menuitem">
156
- <a href="http://wiki.apache.org/pig/">Wiki</a>
157
- </div>
158
- <div class="menuitem">
159
- <a href="http://wiki.apache.org/pig/FAQ">FAQ</a>
160
- </div>
161
- <div class="menuitem">
162
- <a href="http://hadoop.apache.org/pig/releases.html">Release Notes</a>
163
- </div>
164
- </div>
165
- <div id="credit"></div>
166
- <div id="roundbottom">
167
- <img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
168
- <!--+
169
- |alternative credits
170
- +-->
171
- <div id="credit2"></div>
172
- </div>
173
- <!--+
174
- |end Menu
175
- +-->
176
- <!--+
177
- |start content
178
- +-->
179
- <div id="content">
180
- <div title="Portable Document Format" class="pdflink">
181
- <a class="dida" href="setup.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
182
- PDF</a>
183
- </div>
184
- <h1>Pig Setup</h1>
185
- <div id="minitoc-area">
186
- <ul class="minitoc">
187
- <li>
188
- <a href="#Overview">Overview</a>
189
- <ul class="minitoc">
190
- <li>
191
- <a href="#req">Requirements</a>
192
- </li>
193
- </ul>
194
- </li>
195
- <li>
196
- <a href="#Beginning+Pig">Beginning Pig</a>
197
- <ul class="minitoc">
198
- <li>
199
- <a href="#Download+Pig">Download Pig</a>
200
- </li>
201
- <li>
202
- <a href="#Run+Modes">Run Modes</a>
203
- </li>
204
- <li>
205
- <a href="#Grunt+Shell">Grunt Shell</a>
206
- </li>
207
- <li>
208
- <a href="#Script+Files">Script Files</a>
209
- </li>
210
- </ul>
211
- </li>
212
- <li>
213
- <a href="#Advanced+Pig">Advanced Pig</a>
214
- <ul class="minitoc">
215
- <li>
216
- <a href="#Build+Pig">Build Pig</a>
217
- </li>
218
- <li>
219
- <a href="#Environment+Variables+and+Properties">Environment Variables and Properties</a>
220
- </li>
221
- <li>
222
- <a href="#Run+Modes-N1012B">Run Modes</a>
223
- </li>
224
- <li>
225
- <a href="#Embedded+Programs">Embedded Programs</a>
226
- </li>
227
- </ul>
228
- </li>
229
- <li>
230
- <a href="#Sample+Code">Sample Code</a>
231
- </li>
232
- </ul>
233
- </div>
234
-
235
-
236
- <a name="N1000D"></a><a name="Overview"></a>
237
- <h2 class="h3">Overview</h2>
238
- <div class="section">
239
- <a name="N10013"></a><a name="req"></a>
240
- <h3 class="h4">Requirements</h3>
241
- <p>
242
- <strong>Unix</strong> and <strong>Windows</strong> users need the following:</p>
243
- <ol>
244
-
245
- <li>
246
- <strong>Hadoop 0.20.2</strong> - <a href="http://hadoop.apache.org/common/releases.html">http://hadoop.apache.org/common/releases.html</a>
247
- </li>
248
-
249
- <li>
250
- <strong>Java 1.6</strong> - <a href="http://java.sun.com/javase/downloads/index.jsp">http://java.sun.com/javase/downloads/index.jsp</a> (set JAVA_HOME to the root of your Java installation)</li>
251
-
252
- <li>
253
- <strong>Ant 1.7</strong> - <a href="http://ant.apache.org/">http://ant.apache.org/</a> (optional, for builds) </li>
254
-
255
- <li>
256
- <strong>JUnit 4.5</strong> - <a href="http://junit.sourceforge.net/">http://junit.sourceforge.net/</a> (optional, for unit tests) </li>
257
-
258
- </ol>
259
- <p>
260
- <strong>Windows</strong> users need to install Cygwin and the Perl package: <a href="http://www.cygwin.com/"> http://www.cygwin.com/</a>
261
- </p>
262
- </div>
263
-
264
-
265
-
266
- <a name="N10055"></a><a name="Beginning+Pig"></a>
267
- <h2 class="h3">Beginning Pig</h2>
268
- <div class="section">
269
- <a name="N1005B"></a><a name="Download+Pig"></a>
270
- <h3 class="h4">Download Pig</h3>
271
- <p>To get a Pig distribution, download a recent stable release from one of the Apache Download Mirrors (see <a href="http://hadoop.apache.org/pig/releases.html"> Pig Releases</a>).</p>
272
- <p>Unpack the downloaded Pig distribution. The Pig script is located in the bin directory (/pig-n.n.n/bin/pig).</p>
273
- <p>Add /pig-n.n.n/bin to your path. Use export (bash,sh,ksh) or setenv (tcsh,csh). For example: </p>
274
- <pre class="code">
275
- $ export PATH=/&lt;my-path-to-pig&gt;/pig-n.n.n/bin:$PATH
276
- </pre>
277
- <p>Try the following command, to get a list of Pig commands: </p>
278
- <pre class="code">
279
- $ pig -help
280
- </pre>
281
- <p>Try the following command, to start the Grunt shell:</p>
282
- <pre class="code">
283
- $ pig
284
- </pre>
285
- <a name="N10081"></a><a name="Run+Modes"></a>
286
- <h3 class="h4">Run Modes</h3>
287
- <p>Pig has two run modes or exectypes: </p>
288
- <ul>
289
-
290
- <li>
291
- <p> Local Mode - To run Pig in local mode, you need access to a single machine. </p>
292
- </li>
293
-
294
- <li>
295
- <p> Mapreduce Mode - To run Pig in mapreduce mode, you need access to a Hadoop cluster and HDFS installation.
296
- Pig will automatically allocate and deallocate a 15-node cluster.</p>
297
- </li>
298
-
299
- </ul>
300
- <p>You can run the Grunt shell, Pig scripts, or embedded programs using either mode.</p>
301
- <a name="N10099"></a><a name="Grunt+Shell"></a>
302
- <h3 class="h4">Grunt Shell</h3>
303
- <p>Use Pig's interactive shell, Grunt, to enter pig commands manually. See the <a href="setup.html#Sample+Code">Sample Code</a> for instructions about the passwd file used in the example.</p>
304
- <p>You can also run or execute script files from the Grunt shell.
305
- See the <a href="piglatin_ref2.html#run">run</a> and <a href="piglatin_ref2.html#exec">exec</a> commands. </p>
306
- <p>
307
- <strong>Local Mode</strong>
308
- </p>
309
- <pre class="code">
310
- $ pig -x local
311
- </pre>
312
- <p>
313
- <strong>Mapreduce Mode</strong>
314
- </p>
315
- <pre class="code">
316
- $ pig
317
- or
318
- $ pig -x mapreduce
319
- </pre>
320
- <p>For either mode, the Grunt shell is invoked and you can enter commands at the prompt. The results are displayed to your terminal screen (if DUMP is used) or to a file (if STORE is used).
321
- </p>
322
- <pre class="code">
323
- grunt&gt; A = load 'passwd' using PigStorage(':');
324
- grunt&gt; B = foreach A generate $0 as id;
325
- grunt&gt; dump B;
326
- grunt&gt; store B;
327
- </pre>
328
- <a name="N100CA"></a><a name="Script+Files"></a>
329
- <h3 class="h4">Script Files</h3>
330
- <p>Use script files to run Pig commands as batch jobs. See the <a href="setup.html#Sample+Code">Sample Code</a> for instructions about the passwd file and the script file (id.pig) used in the example.</p>
331
- <p>
332
- <strong>Local Mode</strong>
333
- </p>
334
- <pre class="code">
335
- $ pig -x local id.pig
336
- </pre>
337
- <p>
338
- <strong>Mapreduce Mode</strong>
339
- </p>
340
- <pre class="code">
341
- $ pig id.pig
342
- or
343
- $ pig -x mapreduce id.pig
344
- </pre>
345
- <p>For either mode, the Pig Latin statements are executed and the results are displayed to your terminal screen (if DUMP is used) or to a file (if STORE is used).</p>
346
- </div>
347
-
348
-
349
-
350
- <a name="N100ED"></a><a name="Advanced+Pig"></a>
351
- <h2 class="h3">Advanced Pig</h2>
352
- <div class="section">
353
- <a name="N100F3"></a><a name="Build+Pig"></a>
354
- <h3 class="h4">Build Pig</h3>
355
- <p>To build pig, do the following:</p>
356
- <ol>
357
-
358
- <li> Check out the Pig code from SVN: <em>svn co http://svn.apache.org/repos/asf/hadoop/pig/trunk</em>. </li>
359
-
360
- <li> Build the code from the top directory: <em>ant</em>. If the build is successful, you should see the <em>pig.jar</em> created in that directory. </li>
361
-
362
- <li> Validate your <em>pig.jar</em> by running a unit test: <em>ant test</em>
363
- </li>
364
-
365
- </ol>
366
- <a name="N10117"></a><a name="Environment+Variables+and+Properties"></a>
367
- <h3 class="h4">Environment Variables and Properties</h3>
368
- <p>See <a href="setup.html#Download+Pig">Download Pig</a>.</p>
369
- <p>The Pig environment variables are described in the Pig script file, located in the /pig-n.n.n/bin directory.</p>
370
- <p>The Pig properties file, pig.properties, is located in the /pig-n.n.n/conf directory. You can specify an alternate location using the PIG_CONF_DIR environment variable.</p>
371
- <a name="N1012B"></a><a name="Run+Modes-N1012B"></a>
372
- <h3 class="h4">Run Modes</h3>
373
- <p>See <a href="setup.html#Run+Modes">Run Modes</a>. </p>
374
- <a name="N10139"></a><a name="Embedded+Programs"></a>
375
- <h3 class="h4">Embedded Programs</h3>
376
- <p>Used the embedded option to embed Pig commands in a host language and run the program.
377
- See the <a href="setup.html#Sample+Code">Sample Code</a> for instructions about the passwd file and java files (idlocal.java, idmapreduce.java) used in the examples.</p>
378
- <p>
379
- <strong>Local Mode</strong>
380
- </p>
381
- <p>From your current working directory, compile the program: </p>
382
- <pre class="code">
383
- $ javac -cp pig.jar idlocal.java
384
- </pre>
385
- <p>Note: idlocal.class is written to your current working directory. Include &ldquo;.&rdquo; in the class path when you run the program. </p>
386
- <p>From your current working directory, run the program:
387
- </p>
388
- <pre class="code">
389
- Unix: $ java -cp pig.jar:. idlocal
390
- Cygwin: $ java &ndash;cp &lsquo;.;pig.jar&rsquo; idlocal
391
- </pre>
392
- <p>To view the results, check the output file, id.out. </p>
393
- <p>
394
- <strong>Mapreduce Mode</strong>
395
- </p>
396
- <p>Point $HADOOPDIR to the directory that contains the hadoop-site.xml file. Example:
397
- </p>
398
- <pre class="code">
399
- $ export HADOOPDIR=/yourHADOOPsite/conf
400
- </pre>
401
- <p>From your current working directory, compile the program:
402
- </p>
403
- <pre class="code">
404
- $ javac -cp pig.jar idmapreduce.java
405
- </pre>
406
- <p>Note: idmapreduce.class is written to your current working directory. Include &ldquo;.&rdquo; in the class path when you run the program. </p>
407
- <p>From your current working directory, run the program:
408
- </p>
409
- <pre class="code">
410
- Unix: $ java -cp pig.jar:.:$HADOOPDIR idmapreduce
411
- Cygwin: $ java &ndash;cp &lsquo;.;pig.jar;$HADOOPDIR&rsquo; idmapreduce
412
- </pre>
413
- <p>To view the results, check the idout directory on your Hadoop system. </p>
414
- </div>
415
-
416
-
417
-
418
- <a name="N1017F"></a><a name="Sample+Code"></a>
419
- <h2 class="h3">Sample Code</h2>
420
- <div class="section">
421
- <p>The sample code is based on Pig Latin statements that extract all user IDs from the /etc/passwd file. </p>
422
- <p>Copy the /etc/passwd file to your local working directory.</p>
423
- <p>
424
- <strong>id.pig</strong>
425
- </p>
426
- <p>For the Grunt Shell and script files. </p>
427
- <pre class="code">
428
- A = load 'passwd' using PigStorage(':');
429
- B = foreach A generate $0 as id;
430
- dump B;
431
- store B into &lsquo;id.out&rsquo;;
432
- </pre>
433
- <p>
434
- <strong>idlocal.java</strong>
435
- </p>
436
- <p>For embedded programs. </p>
437
- <pre class="code">
438
- import java.io.IOException;
439
- import org.apache.pig.PigServer;
440
- public class idlocal{
441
- public static void main(String[] args) {
442
- try {
443
- PigServer pigServer = new PigServer("local");
444
- runIdQuery(pigServer, "passwd");
445
- }
446
- catch(Exception e) {
447
- }
448
- }
449
- public static void runIdQuery(PigServer pigServer, String inputFile) throws IOException {
450
- pigServer.registerQuery("A = load '" + inputFile + "' using PigStorage(':');");
451
- pigServer.registerQuery("B = foreach A generate $0 as id;");
452
- pigServer.store("B", "id.out");
453
- }
454
- }
455
- </pre>
456
- <p>
457
- <strong>idmapreduce.java</strong>
458
- </p>
459
- <p>For embedded programs. </p>
460
- <pre class="code">
461
- import java.io.IOException;
462
- import org.apache.pig.PigServer;
463
- public class idmapreduce{
464
- public static void main(String[] args) {
465
- try {
466
- PigServer pigServer = new PigServer("mapreduce");
467
- runIdQuery(pigServer, "passwd");
468
- }
469
- catch(Exception e) {
470
- }
471
- }
472
- public static void runIdQuery(PigServer pigServer, String inputFile) throws IOException {
473
- pigServer.registerQuery("A = load '" + inputFile + "' using PigStorage(':');")
474
- pigServer.registerQuery("B = foreach A generate $0 as id;");
475
- pigServer.store("B", "idout");
476
- }
477
- }
478
- </pre>
479
- </div>
480
-
481
- </div>
482
- <!--+
483
- |end content
484
- +-->
485
- <div class="clearboth">&nbsp;</div>
486
- </div>
487
- <div id="footer">
488
- <!--+
489
- |start bottomstrip
490
- +-->
491
- <div class="lastmodified">
492
- <script type="text/javascript"><!--
493
- document.write("Last Published: " + document.lastModified);
494
- // --></script>
495
- </div>
496
- <div class="copyright">
497
- Copyright &copy;
498
- 2007-2010 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
499
- </div>
500
- <!--+
501
- |end bottomstrip
502
- +-->
503
- </div>
504
- </body>
505
- </html>