wukong 3.0.0.pre → 3.0.0.pre2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (476) hide show
  1. data/.gitignore +46 -33
  2. data/.gitmodules +3 -0
  3. data/.rspec +1 -1
  4. data/.travis.yml +8 -1
  5. data/.yardopts +0 -13
  6. data/Guardfile +4 -6
  7. data/{LICENSE.textile → LICENSE.md} +43 -55
  8. data/README-old.md +422 -0
  9. data/README.md +279 -418
  10. data/Rakefile +21 -5
  11. data/TODO.md +6 -6
  12. data/bin/wu-clean-encoding +31 -0
  13. data/bin/wu-lign +2 -2
  14. data/bin/wu-local +69 -0
  15. data/bin/wu-server +70 -0
  16. data/examples/Gemfile +38 -0
  17. data/examples/README.md +9 -0
  18. data/examples/dataflow/apache_log_line.rb +64 -25
  19. data/examples/dataflow/fibonacci_series.rb +101 -0
  20. data/examples/dataflow/parse_apache_logs.rb +37 -7
  21. data/examples/{dataflow.rb → dataflow/scraper_macro_flow.rb} +0 -0
  22. data/examples/dataflow/simple.rb +4 -4
  23. data/examples/geo.rb +4 -0
  24. data/examples/geo/geo_grids.numbers +0 -0
  25. data/examples/geo/geolocated.rb +331 -0
  26. data/examples/geo/quadtile.rb +69 -0
  27. data/examples/geo/spec/geolocated_spec.rb +247 -0
  28. data/examples/geo/tile_fetcher.rb +77 -0
  29. data/examples/graph/minimum_spanning_tree.rb +61 -61
  30. data/examples/jabberwocky.txt +36 -0
  31. data/examples/models/wikipedia.rb +20 -0
  32. data/examples/munging/Gemfile +8 -0
  33. data/examples/munging/airline_flights/airline.rb +57 -0
  34. data/examples/munging/airline_flights/airline_flights.rake +83 -0
  35. data/{lib/wukong/settings.rb → examples/munging/airline_flights/airplane.rb} +0 -0
  36. data/examples/munging/airline_flights/airport.rb +211 -0
  37. data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
  38. data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
  39. data/examples/munging/airline_flights/flight.rb +156 -0
  40. data/examples/munging/airline_flights/models.rb +4 -0
  41. data/examples/munging/airline_flights/parse.rb +26 -0
  42. data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
  43. data/examples/munging/airline_flights/route.rb +35 -0
  44. data/examples/munging/airline_flights/tasks.rake +83 -0
  45. data/examples/munging/airline_flights/timezone_fixup.rb +62 -0
  46. data/examples/munging/airline_flights/topcities.rb +167 -0
  47. data/examples/munging/airports/40_wbans.txt +40 -0
  48. data/examples/munging/airports/filter_weather_reports.rb +37 -0
  49. data/examples/munging/airports/join.pig +31 -0
  50. data/examples/munging/airports/to_tsv.rb +33 -0
  51. data/examples/munging/airports/usa_wbans.pig +19 -0
  52. data/examples/munging/airports/usa_wbans.txt +2157 -0
  53. data/examples/munging/airports/wbans.pig +19 -0
  54. data/examples/munging/airports/wbans.txt +2310 -0
  55. data/examples/munging/geo/geo_json.rb +54 -0
  56. data/examples/munging/geo/geo_models.rb +69 -0
  57. data/examples/munging/geo/geonames_models.rb +78 -0
  58. data/examples/munging/geo/iso_codes.rb +172 -0
  59. data/examples/munging/geo/reconcile_countries.rb +124 -0
  60. data/examples/munging/geo/tasks.rake +71 -0
  61. data/examples/munging/rake_helper.rb +62 -0
  62. data/examples/munging/weather/.gitignore +1 -0
  63. data/examples/munging/weather/Gemfile +4 -0
  64. data/examples/munging/weather/Rakefile +28 -0
  65. data/examples/munging/weather/extract_ish.rb +13 -0
  66. data/examples/munging/weather/models/weather.rb +119 -0
  67. data/examples/munging/weather/utils/noaa_downloader.rb +46 -0
  68. data/examples/munging/wikipedia/README.md +34 -0
  69. data/examples/munging/wikipedia/Rakefile +193 -0
  70. data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
  71. data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
  72. data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
  73. data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
  74. data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
  75. data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
  76. data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
  77. data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +4 -0
  78. data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
  79. data/examples/munging/wikipedia/dbpedia/extract_links.rb +193 -0
  80. data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
  81. data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig +18 -0
  82. data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb +21 -0
  83. data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old +27 -0
  84. data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig +29 -0
  85. data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb +14 -0
  86. data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old +25 -0
  87. data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig +29 -0
  88. data/examples/munging/wikipedia/pageviews/augment_pageviews.pig +32 -0
  89. data/examples/munging/wikipedia/pageviews/extract_pageviews.rb +85 -0
  90. data/examples/munging/wikipedia/pig_style_guide.md +25 -0
  91. data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig +19 -0
  92. data/examples/munging/wikipedia/subuniverse/sub_articles.pig +23 -0
  93. data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig +24 -0
  94. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig +22 -0
  95. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig +22 -0
  96. data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig +26 -0
  97. data/examples/munging/wikipedia/subuniverse/sub_pageviews.pig +29 -0
  98. data/examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig +24 -0
  99. data/examples/munging/wikipedia/utils/get_namespaces.rb +86 -0
  100. data/examples/munging/wikipedia/utils/munging_utils.rb +68 -0
  101. data/examples/munging/wikipedia/utils/namespaces.json +1 -0
  102. data/examples/rake_helper.rb +85 -0
  103. data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
  104. data/examples/server_logs/logline.rb +95 -0
  105. data/examples/server_logs/models.rb +66 -0
  106. data/examples/server_logs/page_counts.pig +48 -0
  107. data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
  108. data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
  109. data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
  110. data/{old/examples/server_logs/breadcrumbs.rb → examples/server_logs/server_logs-03-breadcrumbs-full.rb} +26 -30
  111. data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
  112. data/examples/string_reverser.rb +26 -0
  113. data/examples/text/pig_latin.rb +2 -2
  114. data/examples/text/regional_flavor/README.md +14 -0
  115. data/examples/text/regional_flavor/article_wordbags.pig +39 -0
  116. data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
  117. data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
  118. data/examples/word_count/accumulator.rb +26 -0
  119. data/examples/word_count/tokenizer.rb +13 -0
  120. data/examples/word_count/word_count.rb +6 -0
  121. data/examples/workflow/cherry_pie.dot +97 -0
  122. data/examples/workflow/cherry_pie.png +0 -0
  123. data/examples/workflow/cherry_pie.rb +61 -26
  124. data/lib/hanuman.rb +34 -7
  125. data/lib/hanuman/graph.rb +55 -31
  126. data/lib/hanuman/graphvizzer.rb +199 -178
  127. data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
  128. data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
  129. data/lib/hanuman/link.rb +35 -0
  130. data/lib/hanuman/registry.rb +46 -0
  131. data/lib/hanuman/stage.rb +76 -32
  132. data/lib/wukong.rb +23 -24
  133. data/lib/wukong/boot.rb +87 -0
  134. data/lib/wukong/configuration.rb +8 -0
  135. data/lib/wukong/dataflow.rb +45 -78
  136. data/lib/wukong/driver.rb +99 -0
  137. data/lib/wukong/emitter.rb +22 -0
  138. data/lib/wukong/model/faker.rb +24 -24
  139. data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
  140. data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
  141. data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
  142. data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
  143. data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
  144. data/lib/wukong/processor.rb +60 -114
  145. data/lib/wukong/spec_helpers.rb +81 -0
  146. data/lib/wukong/spec_helpers/integration_driver.rb +144 -0
  147. data/lib/wukong/spec_helpers/integration_driver_matchers.rb +219 -0
  148. data/lib/wukong/spec_helpers/processor_helpers.rb +95 -0
  149. data/lib/wukong/spec_helpers/processor_methods.rb +108 -0
  150. data/lib/wukong/spec_helpers/shared_examples.rb +15 -0
  151. data/lib/wukong/spec_helpers/spec_driver.rb +28 -0
  152. data/lib/wukong/spec_helpers/spec_driver_matchers.rb +195 -0
  153. data/lib/wukong/version.rb +2 -1
  154. data/lib/wukong/widget/filters.rb +311 -0
  155. data/lib/wukong/widget/processors.rb +156 -0
  156. data/lib/wukong/widget/reducers.rb +7 -0
  157. data/lib/wukong/widget/reducers/accumulator.rb +73 -0
  158. data/lib/wukong/widget/reducers/bin.rb +318 -0
  159. data/lib/wukong/widget/reducers/count.rb +61 -0
  160. data/lib/wukong/widget/reducers/group.rb +85 -0
  161. data/lib/wukong/widget/reducers/group_concat.rb +70 -0
  162. data/lib/wukong/widget/reducers/moments.rb +72 -0
  163. data/lib/wukong/widget/reducers/sort.rb +130 -0
  164. data/lib/wukong/widget/serializers.rb +287 -0
  165. data/lib/wukong/widget/sink.rb +10 -52
  166. data/lib/wukong/widget/source.rb +7 -113
  167. data/lib/wukong/widget/utils.rb +46 -0
  168. data/lib/wukong/widgets.rb +6 -0
  169. data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
  170. data/spec/examples/dataflow/parsing_spec.rb +12 -11
  171. data/spec/examples/dataflow/simple_spec.rb +32 -6
  172. data/spec/examples/dataflow/telegram_spec.rb +36 -36
  173. data/spec/examples/graph/minimum_spanning_tree_spec.rb +30 -31
  174. data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
  175. data/spec/examples/munging/airline_flights_spec.rb +202 -0
  176. data/spec/examples/text/pig_latin_spec.rb +13 -16
  177. data/spec/examples/workflow/cherry_pie_spec.rb +34 -4
  178. data/spec/hanuman/graph_spec.rb +27 -2
  179. data/spec/hanuman/hanuman_spec.rb +10 -0
  180. data/spec/hanuman/registry_spec.rb +123 -0
  181. data/spec/hanuman/stage_spec.rb +61 -7
  182. data/spec/spec_helper.rb +29 -19
  183. data/spec/support/hanuman_test_helpers.rb +14 -12
  184. data/spec/support/shared_context_for_reducers.rb +37 -0
  185. data/spec/support/shared_examples_for_builders.rb +101 -0
  186. data/spec/support/shared_examples_for_shortcuts.rb +57 -0
  187. data/spec/support/wukong_test_helpers.rb +37 -11
  188. data/spec/wukong/dataflow_spec.rb +77 -55
  189. data/spec/wukong/local_runner_spec.rb +24 -24
  190. data/spec/wukong/model/faker_spec.rb +132 -131
  191. data/spec/wukong/runner_spec.rb +8 -8
  192. data/spec/wukong/widget/filters_spec.rb +61 -0
  193. data/spec/wukong/widget/processors_spec.rb +126 -0
  194. data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
  195. data/spec/wukong/widget/reducers/count_spec.rb +11 -0
  196. data/spec/wukong/widget/reducers/group_spec.rb +20 -0
  197. data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
  198. data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
  199. data/spec/wukong/widget/serializers_spec.rb +92 -0
  200. data/spec/wukong/widget/sink_spec.rb +15 -15
  201. data/spec/wukong/widget/source_spec.rb +65 -41
  202. data/spec/wukong/wukong_spec.rb +10 -0
  203. data/wukong.gemspec +17 -10
  204. metadata +359 -335
  205. data/.document +0 -5
  206. data/VERSION +0 -1
  207. data/bin/hdp-bin +0 -44
  208. data/bin/hdp-bzip +0 -23
  209. data/bin/hdp-cat +0 -3
  210. data/bin/hdp-catd +0 -3
  211. data/bin/hdp-cp +0 -3
  212. data/bin/hdp-du +0 -86
  213. data/bin/hdp-get +0 -3
  214. data/bin/hdp-kill +0 -3
  215. data/bin/hdp-kill-task +0 -3
  216. data/bin/hdp-ls +0 -11
  217. data/bin/hdp-mkdir +0 -2
  218. data/bin/hdp-mkdirp +0 -12
  219. data/bin/hdp-mv +0 -3
  220. data/bin/hdp-parts_to_keys.rb +0 -77
  221. data/bin/hdp-ps +0 -3
  222. data/bin/hdp-put +0 -3
  223. data/bin/hdp-rm +0 -32
  224. data/bin/hdp-sort +0 -40
  225. data/bin/hdp-stream +0 -40
  226. data/bin/hdp-stream-flat +0 -22
  227. data/bin/hdp-stream2 +0 -39
  228. data/bin/hdp-sync +0 -17
  229. data/bin/hdp-wc +0 -67
  230. data/bin/wu-flow +0 -10
  231. data/bin/wu-map +0 -17
  232. data/bin/wu-red +0 -17
  233. data/bin/wukong +0 -17
  234. data/data/CREDITS.md +0 -355
  235. data/data/graph/airfares.tsv +0 -2174
  236. data/data/text/gift_of_the_magi.txt +0 -225
  237. data/data/text/jabberwocky.txt +0 -36
  238. data/data/text/rectification_of_names.txt +0 -33
  239. data/data/twitter/a_atsigns_b.tsv +0 -64
  240. data/data/twitter/a_follows_b.tsv +0 -53
  241. data/data/twitter/tweet.tsv +0 -167
  242. data/data/twitter/twitter_user.tsv +0 -55
  243. data/data/wikipedia/dbpedia-sentences.tsv +0 -1000
  244. data/docpages/INSTALL.textile +0 -92
  245. data/docpages/LICENSE.textile +0 -107
  246. data/docpages/README-elastic_map_reduce.textile +0 -377
  247. data/docpages/README-performance.textile +0 -90
  248. data/docpages/README-wulign.textile +0 -65
  249. data/docpages/UsingWukong-part1-get_ready.textile +0 -17
  250. data/docpages/UsingWukong-part2-ThinkingBigData.textile +0 -75
  251. data/docpages/UsingWukong-part3-parsing.textile +0 -138
  252. data/docpages/_config.yml +0 -39
  253. data/docpages/avro/avro_notes.textile +0 -56
  254. data/docpages/avro/performance.textile +0 -36
  255. data/docpages/avro/tethering.textile +0 -19
  256. data/docpages/bigdata-tips.textile +0 -143
  257. data/docpages/code/api_response_example.txt +0 -20
  258. data/docpages/code/parser_skeleton.rb +0 -38
  259. data/docpages/diagrams/MapReduceDiagram.graffle +0 -0
  260. data/docpages/favicon.ico +0 -0
  261. data/docpages/gem.css +0 -16
  262. data/docpages/hadoop-tips.textile +0 -83
  263. data/docpages/index.textile +0 -92
  264. data/docpages/intro.textile +0 -8
  265. data/docpages/moreinfo.textile +0 -174
  266. data/docpages/news.html +0 -24
  267. data/docpages/pig/PigLatinExpressionsList.txt +0 -122
  268. data/docpages/pig/PigLatinReferenceManual.txt +0 -1640
  269. data/docpages/pig/commandline_params.txt +0 -26
  270. data/docpages/pig/cookbook.html +0 -481
  271. data/docpages/pig/images/hadoop-logo.jpg +0 -0
  272. data/docpages/pig/images/instruction_arrow.png +0 -0
  273. data/docpages/pig/images/pig-logo.gif +0 -0
  274. data/docpages/pig/piglatin_ref1.html +0 -1103
  275. data/docpages/pig/piglatin_ref2.html +0 -14340
  276. data/docpages/pig/setup.html +0 -505
  277. data/docpages/pig/skin/basic.css +0 -166
  278. data/docpages/pig/skin/breadcrumbs.js +0 -237
  279. data/docpages/pig/skin/fontsize.js +0 -166
  280. data/docpages/pig/skin/getBlank.js +0 -40
  281. data/docpages/pig/skin/getMenu.js +0 -45
  282. data/docpages/pig/skin/images/chapter.gif +0 -0
  283. data/docpages/pig/skin/images/chapter_open.gif +0 -0
  284. data/docpages/pig/skin/images/current.gif +0 -0
  285. data/docpages/pig/skin/images/external-link.gif +0 -0
  286. data/docpages/pig/skin/images/header_white_line.gif +0 -0
  287. data/docpages/pig/skin/images/page.gif +0 -0
  288. data/docpages/pig/skin/images/pdfdoc.gif +0 -0
  289. data/docpages/pig/skin/images/rc-b-l-15-1body-2menu-3menu.png +0 -0
  290. data/docpages/pig/skin/images/rc-b-r-15-1body-2menu-3menu.png +0 -0
  291. data/docpages/pig/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png +0 -0
  292. data/docpages/pig/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png +0 -0
  293. data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png +0 -0
  294. data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png +0 -0
  295. data/docpages/pig/skin/images/rc-t-r-15-1body-2menu-3menu.png +0 -0
  296. data/docpages/pig/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png +0 -0
  297. data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png +0 -0
  298. data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png +0 -0
  299. data/docpages/pig/skin/print.css +0 -54
  300. data/docpages/pig/skin/profile.css +0 -181
  301. data/docpages/pig/skin/screen.css +0 -587
  302. data/docpages/pig/tutorial.html +0 -1059
  303. data/docpages/pig/udf.html +0 -1509
  304. data/docpages/tutorial.textile +0 -283
  305. data/docpages/usage.textile +0 -195
  306. data/docpages/wutils.textile +0 -263
  307. data/examples/dataflow/complex.rb +0 -11
  308. data/examples/dataflow/donuts.rb +0 -13
  309. data/examples/tiny_count/jabberwocky_output.tsv +0 -92
  310. data/examples/word_count.rb +0 -48
  311. data/examples/workflow/fiddle.rb +0 -24
  312. data/lib/away/escapement.rb +0 -129
  313. data/lib/away/exe.rb +0 -11
  314. data/lib/away/experimental.rb +0 -5
  315. data/lib/away/from_file.rb +0 -52
  316. data/lib/away/job.rb +0 -56
  317. data/lib/away/job/rake_compat.rb +0 -17
  318. data/lib/away/registry.rb +0 -79
  319. data/lib/away/runner.rb +0 -276
  320. data/lib/away/runner/execute.rb +0 -121
  321. data/lib/away/script.rb +0 -161
  322. data/lib/away/script/hadoop_command.rb +0 -240
  323. data/lib/away/source/file_list_source.rb +0 -15
  324. data/lib/away/source/looper.rb +0 -18
  325. data/lib/away/task.rb +0 -219
  326. data/lib/hanuman/action.rb +0 -21
  327. data/lib/hanuman/chain.rb +0 -4
  328. data/lib/hanuman/graphviz.rb +0 -74
  329. data/lib/hanuman/resource.rb +0 -6
  330. data/lib/hanuman/slot.rb +0 -87
  331. data/lib/hanuman/slottable.rb +0 -220
  332. data/lib/wukong/bad_record.rb +0 -15
  333. data/lib/wukong/event.rb +0 -44
  334. data/lib/wukong/local_runner.rb +0 -55
  335. data/lib/wukong/mapred.rb +0 -3
  336. data/lib/wukong/universe.rb +0 -48
  337. data/lib/wukong/widget/filter.rb +0 -81
  338. data/lib/wukong/widget/gibberish.rb +0 -123
  339. data/lib/wukong/widget/monitor.rb +0 -26
  340. data/lib/wukong/widget/reducer.rb +0 -66
  341. data/lib/wukong/widget/stringifier.rb +0 -50
  342. data/lib/wukong/workflow.rb +0 -22
  343. data/lib/wukong/workflow/command.rb +0 -42
  344. data/old/config/emr-example.yaml +0 -48
  345. data/old/examples/README.txt +0 -17
  346. data/old/examples/contrib/jeans/README.markdown +0 -165
  347. data/old/examples/contrib/jeans/data/normalized_sizes +0 -3
  348. data/old/examples/contrib/jeans/data/orders.tsv +0 -1302
  349. data/old/examples/contrib/jeans/data/sizes +0 -3
  350. data/old/examples/contrib/jeans/normalize.rb +0 -20
  351. data/old/examples/contrib/jeans/sizes.rb +0 -55
  352. data/old/examples/corpus/bnc_word_freq.rb +0 -44
  353. data/old/examples/corpus/bucket_counter.rb +0 -47
  354. data/old/examples/corpus/dbpedia_abstract_to_sentences.rb +0 -86
  355. data/old/examples/corpus/sentence_bigrams.rb +0 -53
  356. data/old/examples/corpus/sentence_coocurrence.rb +0 -66
  357. data/old/examples/corpus/stopwords.rb +0 -138
  358. data/old/examples/corpus/words_to_bigrams.rb +0 -53
  359. data/old/examples/emr/README.textile +0 -110
  360. data/old/examples/emr/dot_wukong_dir/credentials.json +0 -7
  361. data/old/examples/emr/dot_wukong_dir/emr.yaml +0 -69
  362. data/old/examples/emr/dot_wukong_dir/emr_bootstrap.sh +0 -33
  363. data/old/examples/emr/elastic_mapreduce_example.rb +0 -28
  364. data/old/examples/network_graph/adjacency_list.rb +0 -74
  365. data/old/examples/network_graph/breadth_first_search.rb +0 -72
  366. data/old/examples/network_graph/gen_2paths.rb +0 -68
  367. data/old/examples/network_graph/gen_multi_edge.rb +0 -112
  368. data/old/examples/network_graph/gen_symmetric_links.rb +0 -64
  369. data/old/examples/pagerank/README.textile +0 -6
  370. data/old/examples/pagerank/gen_initial_pagerank_graph.pig +0 -57
  371. data/old/examples/pagerank/pagerank.rb +0 -72
  372. data/old/examples/pagerank/pagerank_initialize.rb +0 -42
  373. data/old/examples/pagerank/run_pagerank.sh +0 -21
  374. data/old/examples/sample_records.rb +0 -33
  375. data/old/examples/server_logs/apache_log_parser.rb +0 -15
  376. data/old/examples/server_logs/nook.rb +0 -48
  377. data/old/examples/server_logs/nook/faraday_dummy_adapter.rb +0 -94
  378. data/old/examples/server_logs/user_agent.rb +0 -40
  379. data/old/examples/simple_word_count.rb +0 -82
  380. data/old/examples/size.rb +0 -61
  381. data/old/examples/stats/avg_value_frequency.rb +0 -86
  382. data/old/examples/stats/binning_percentile_estimator.rb +0 -140
  383. data/old/examples/stats/data/avg_value_frequency.tsv +0 -3
  384. data/old/examples/stats/rank_and_bin.rb +0 -173
  385. data/old/examples/stupidly_simple_filter.rb +0 -40
  386. data/old/examples/word_count.rb +0 -75
  387. data/old/graph/graphviz_builder.rb +0 -580
  388. data/old/graph_easy/Attributes.pm +0 -4181
  389. data/old/graph_easy/Graphviz.pm +0 -2232
  390. data/old/wukong.rb +0 -18
  391. data/old/wukong/and_pig.rb +0 -38
  392. data/old/wukong/bad_record.rb +0 -18
  393. data/old/wukong/datatypes.rb +0 -24
  394. data/old/wukong/datatypes/enum.rb +0 -127
  395. data/old/wukong/datatypes/fake_types.rb +0 -17
  396. data/old/wukong/decorator.rb +0 -28
  397. data/old/wukong/encoding/asciize.rb +0 -108
  398. data/old/wukong/extensions.rb +0 -16
  399. data/old/wukong/extensions/array.rb +0 -18
  400. data/old/wukong/extensions/blank.rb +0 -93
  401. data/old/wukong/extensions/class.rb +0 -189
  402. data/old/wukong/extensions/date_time.rb +0 -53
  403. data/old/wukong/extensions/emittable.rb +0 -69
  404. data/old/wukong/extensions/enumerable.rb +0 -79
  405. data/old/wukong/extensions/hash.rb +0 -167
  406. data/old/wukong/extensions/hash_keys.rb +0 -16
  407. data/old/wukong/extensions/hash_like.rb +0 -150
  408. data/old/wukong/extensions/hashlike_class.rb +0 -47
  409. data/old/wukong/extensions/module.rb +0 -2
  410. data/old/wukong/extensions/pathname.rb +0 -27
  411. data/old/wukong/extensions/string.rb +0 -65
  412. data/old/wukong/extensions/struct.rb +0 -17
  413. data/old/wukong/extensions/symbol.rb +0 -11
  414. data/old/wukong/filename_pattern.rb +0 -74
  415. data/old/wukong/helper.rb +0 -7
  416. data/old/wukong/helper/stopwords.rb +0 -195
  417. data/old/wukong/helper/tokenize.rb +0 -35
  418. data/old/wukong/logger.rb +0 -38
  419. data/old/wukong/periodic_monitor.rb +0 -72
  420. data/old/wukong/schema.rb +0 -269
  421. data/old/wukong/script.rb +0 -286
  422. data/old/wukong/script/avro_command.rb +0 -5
  423. data/old/wukong/script/cassandra_loader_script.rb +0 -40
  424. data/old/wukong/script/emr_command.rb +0 -168
  425. data/old/wukong/script/hadoop_command.rb +0 -237
  426. data/old/wukong/script/local_command.rb +0 -41
  427. data/old/wukong/store.rb +0 -10
  428. data/old/wukong/store/base.rb +0 -27
  429. data/old/wukong/store/cassandra.rb +0 -10
  430. data/old/wukong/store/cassandra/streaming.rb +0 -75
  431. data/old/wukong/store/cassandra/struct_loader.rb +0 -21
  432. data/old/wukong/store/cassandra_model.rb +0 -91
  433. data/old/wukong/store/chh_chunked_flat_file_store.rb +0 -37
  434. data/old/wukong/store/chunked_flat_file_store.rb +0 -48
  435. data/old/wukong/store/conditional_store.rb +0 -57
  436. data/old/wukong/store/factory.rb +0 -8
  437. data/old/wukong/store/flat_file_store.rb +0 -89
  438. data/old/wukong/store/key_store.rb +0 -51
  439. data/old/wukong/store/null_store.rb +0 -15
  440. data/old/wukong/store/read_thru_store.rb +0 -22
  441. data/old/wukong/store/tokyo_tdb_key_store.rb +0 -33
  442. data/old/wukong/store/tyrant_rdb_key_store.rb +0 -57
  443. data/old/wukong/store/tyrant_tdb_key_store.rb +0 -20
  444. data/old/wukong/streamer.rb +0 -30
  445. data/old/wukong/streamer/accumulating_reducer.rb +0 -83
  446. data/old/wukong/streamer/base.rb +0 -126
  447. data/old/wukong/streamer/counting_reducer.rb +0 -25
  448. data/old/wukong/streamer/filter.rb +0 -20
  449. data/old/wukong/streamer/instance_streamer.rb +0 -15
  450. data/old/wukong/streamer/json_streamer.rb +0 -21
  451. data/old/wukong/streamer/line_streamer.rb +0 -12
  452. data/old/wukong/streamer/list_reducer.rb +0 -31
  453. data/old/wukong/streamer/rank_and_bin_reducer.rb +0 -145
  454. data/old/wukong/streamer/record_streamer.rb +0 -14
  455. data/old/wukong/streamer/reducer.rb +0 -11
  456. data/old/wukong/streamer/set_reducer.rb +0 -14
  457. data/old/wukong/streamer/struct_streamer.rb +0 -48
  458. data/old/wukong/streamer/summing_reducer.rb +0 -29
  459. data/old/wukong/streamer/uniq_by_last_reducer.rb +0 -51
  460. data/old/wukong/typed_struct.rb +0 -12
  461. data/spec/away/encoding_spec.rb +0 -32
  462. data/spec/away/exe_spec.rb +0 -20
  463. data/spec/away/flow_spec.rb +0 -82
  464. data/spec/away/graph_spec.rb +0 -6
  465. data/spec/away/job_spec.rb +0 -15
  466. data/spec/away/rake_compat_spec.rb +0 -9
  467. data/spec/away/script_spec.rb +0 -81
  468. data/spec/hanuman/graphviz_spec.rb +0 -29
  469. data/spec/hanuman/slot_spec.rb +0 -2
  470. data/spec/support/examples_helper.rb +0 -10
  471. data/spec/support/streamer_test_helpers.rb +0 -6
  472. data/spec/support/wukong_widget_helpers.rb +0 -66
  473. data/spec/wukong/processor_spec.rb +0 -109
  474. data/spec/wukong/widget/filter_spec.rb +0 -99
  475. data/spec/wukong/widget/stringifier_spec.rb +0 -51
  476. data/spec/wukong/workflow/command_spec.rb +0 -5
data/docpages/pig/setup.html DELETED
@@ -1,505 +0,0 @@
1
- <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
2
- <html>
3
- <head>
4
- <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
5
- <meta content="Apache Forrest" name="Generator">
6
- <meta name="Forrest-version" content="0.8">
7
- <meta name="Forrest-skin-name" content="pelt">
8
- <title>Pig Setup</title>
9
- <link type="text/css" href="skin/basic.css" rel="stylesheet">
10
- <link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
11
- <link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
12
- <link type="text/css" href="skin/profile.css" rel="stylesheet">
13
- <script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
14
- <link rel="shortcut icon" href="">
15
- </head>
16
- <body onload="init()">
17
- <script type="text/javascript">ndeSetTextSize();</script>
18
- <div id="top">
19
- <!--+
20
- |breadtrail
21
- +-->
22
- <div class="breadtrail">
23
- <a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/pig/">Pig</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
24
- </div>
25
- <!--+
26
- |header
27
- +-->
28
- <div class="header">
29
- <!--+
30
- |start group logo
31
- +-->
32
- <div class="grouplogo">
33
- <a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
34
- </div>
35
- <!--+
36
- |end group logo
37
- +-->
38
- <!--+
39
- |start Project Logo
40
- +-->
41
- <div class="projectlogo">
42
- <a href="http://hadoop.apache.org/pig/"><img class="logoImage" alt="Pig" src="images/pig-logo.gif" title="A platform for analyzing large datasets."></a>
43
- </div>
44
- <!--+
45
- |end Project Logo
46
- +-->
47
- <!--+
48
- |start Search
49
- +-->
50
- <div class="searchbox">
51
- <form action="http://www.google.com/search" method="get" class="roundtopsmall">
52
- <input value="" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
53
- <input name="Search" value="Search" type="submit">
54
- </form>
55
- </div>
56
- <!--+
57
- |end search
58
- +-->
59
- <!--+
60
- |start Tabs
61
- +-->
62
- <ul id="tabs">
63
- <li>
64
- <a class="unselected" href="http://hadoop.apache.org/pig/">Project</a>
65
- </li>
66
- <li>
67
- <a class="unselected" href="http://wiki.apache.org/pig/">Wiki</a>
68
- </li>
69
- <li class="current">
70
- <a class="selected" href="index.html">Pig 0.7.0 Documentation</a>
71
- </li>
72
- </ul>
73
- <!--+
74
- |end Tabs
75
- +-->
76
- </div>
77
- </div>
78
- <div id="main">
79
- <div id="publishedStrip">
80
- <!--+
81
- |start Subtabs
82
- +-->
83
- <div id="level2tabs"></div>
84
- <!--+
85
- |end Endtabs
86
- +-->
87
- <script type="text/javascript"><!--
88
- document.write("Last Published: " + document.lastModified);
89
- // --></script>
90
- </div>
91
- <!--+
92
- |breadtrail
93
- +-->
94
- <div class="breadtrail">
95
-
96
- &nbsp;
97
- </div>
98
- <!--+
99
- |start Menu, mainarea
100
- +-->
101
- <!--+
102
- |start Menu
103
- +-->
104
- <div id="menu">
105
- <div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Pig</div>
106
- <div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
107
- <div class="menuitem">
108
- <a href="index.html">Overview</a>
109
- </div>
110
- <div class="menupage">
111
- <div class="menupagetitle">Setup</div>
112
- </div>
113
- <div class="menuitem">
114
- <a href="tutorial.html">Tutorial</a>
115
- </div>
116
- <div class="menuitem">
117
- <a href="piglatin_ref1.html">Pig Latin 1</a>
118
- </div>
119
- <div class="menuitem">
120
- <a href="piglatin_ref2.html">Pig Latin 2</a>
121
- </div>
122
- <div class="menuitem">
123
- <a href="cookbook.html">Cookbook</a>
124
- </div>
125
- <div class="menuitem">
126
- <a href="udf.html">UDFs</a>
127
- </div>
128
- </div>
129
- <div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Zebra</div>
130
- <div id="menu_1.2" class="menuitemgroup">
131
- <div class="menuitem">
132
- <a href="zebra_overview.html">Zebra Overview </a>
133
- </div>
134
- <div class="menuitem">
135
- <a href="zebra_users.html">Zebra Users </a>
136
- </div>
137
- <div class="menuitem">
138
- <a href="zebra_reference.html">Zebra Reference </a>
139
- </div>
140
- <div class="menuitem">
141
- <a href="zebra_mapreduce.html">Zebra MapReduce </a>
142
- </div>
143
- <div class="menuitem">
144
- <a href="zebra_pig.html">Zebra Pig </a>
145
- </div>
146
- <div class="menuitem">
147
- <a href="zebra_stream.html">Zebra Streaming </a>
148
- </div>
149
- </div>
150
- <div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Miscellaneous</div>
151
- <div id="menu_1.3" class="menuitemgroup">
152
- <div class="menuitem">
153
- <a href="api/">API Docs</a>
154
- </div>
155
- <div class="menuitem">
156
- <a href="http://wiki.apache.org/pig/">Wiki</a>
157
- </div>
158
- <div class="menuitem">
159
- <a href="http://wiki.apache.org/pig/FAQ">FAQ</a>
160
- </div>
161
- <div class="menuitem">
162
- <a href="http://hadoop.apache.org/pig/releases.html">Release Notes</a>
163
- </div>
164
- </div>
165
- <div id="credit"></div>
166
- <div id="roundbottom">
167
- <img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
168
- <!--+
169
- |alternative credits
170
- +-->
171
- <div id="credit2"></div>
172
- </div>
173
- <!--+
174
- |end Menu
175
- +-->
176
- <!--+
177
- |start content
178
- +-->
179
- <div id="content">
180
- <div title="Portable Document Format" class="pdflink">
181
- <a class="dida" href="setup.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
182
- PDF</a>
183
- </div>
184
- <h1>Pig Setup</h1>
185
- <div id="minitoc-area">
186
- <ul class="minitoc">
187
- <li>
188
- <a href="#Overview">Overview</a>
189
- <ul class="minitoc">
190
- <li>
191
- <a href="#req">Requirements</a>
192
- </li>
193
- </ul>
194
- </li>
195
- <li>
196
- <a href="#Beginning+Pig">Beginning Pig</a>
197
- <ul class="minitoc">
198
- <li>
199
- <a href="#Download+Pig">Download Pig</a>
200
- </li>
201
- <li>
202
- <a href="#Run+Modes">Run Modes</a>
203
- </li>
204
- <li>
205
- <a href="#Grunt+Shell">Grunt Shell</a>
206
- </li>
207
- <li>
208
- <a href="#Script+Files">Script Files</a>
209
- </li>
210
- </ul>
211
- </li>
212
- <li>
213
- <a href="#Advanced+Pig">Advanced Pig</a>
214
- <ul class="minitoc">
215
- <li>
216
- <a href="#Build+Pig">Build Pig</a>
217
- </li>
218
- <li>
219
- <a href="#Environment+Variables+and+Properties">Environment Variables and Properties</a>
220
- </li>
221
- <li>
222
- <a href="#Run+Modes-N1012B">Run Modes</a>
223
- </li>
224
- <li>
225
- <a href="#Embedded+Programs">Embedded Programs</a>
226
- </li>
227
- </ul>
228
- </li>
229
- <li>
230
- <a href="#Sample+Code">Sample Code</a>
231
- </li>
232
- </ul>
233
- </div>
234
-
235
-
236
- <a name="N1000D"></a><a name="Overview"></a>
237
- <h2 class="h3">Overview</h2>
238
- <div class="section">
239
- <a name="N10013"></a><a name="req"></a>
240
- <h3 class="h4">Requirements</h3>
241
- <p>
242
- <strong>Unix</strong> and <strong>Windows</strong> users need the following:</p>
243
- <ol>
244
-
245
- <li>
246
- <strong>Hadoop 0.20.2</strong> - <a href="http://hadoop.apache.org/common/releases.html">http://hadoop.apache.org/common/releases.html</a>
247
- </li>
248
-
249
- <li>
250
- <strong>Java 1.6</strong> - <a href="http://java.sun.com/javase/downloads/index.jsp">http://java.sun.com/javase/downloads/index.jsp</a> (set JAVA_HOME to the root of your Java installation)</li>
251
-
252
- <li>
253
- <strong>Ant 1.7</strong> - <a href="http://ant.apache.org/">http://ant.apache.org/</a> (optional, for builds) </li>
254
-
255
- <li>
256
- <strong>JUnit 4.5</strong> - <a href="http://junit.sourceforge.net/">http://junit.sourceforge.net/</a> (optional, for unit tests) </li>
257
-
258
- </ol>
259
- <p>
260
- <strong>Windows</strong> users need to install Cygwin and the Perl package: <a href="http://www.cygwin.com/"> http://www.cygwin.com/</a>
261
- </p>
262
- </div>
263
-
264
-
265
-
266
- <a name="N10055"></a><a name="Beginning+Pig"></a>
267
- <h2 class="h3">Beginning Pig</h2>
268
- <div class="section">
269
- <a name="N1005B"></a><a name="Download+Pig"></a>
270
- <h3 class="h4">Download Pig</h3>
271
- <p>To get a Pig distribution, download a recent stable release from one of the Apache Download Mirrors (see <a href="http://hadoop.apache.org/pig/releases.html"> Pig Releases</a>).</p>
272
- <p>Unpack the downloaded Pig distribution. The Pig script is located in the bin directory (/pig-n.n.n/bin/pig).</p>
273
- <p>Add /pig-n.n.n/bin to your path. Use export (bash,sh,ksh) or setenv (tcsh,csh). For example: </p>
274
- <pre class="code">
275
- $ export PATH=/&lt;my-path-to-pig&gt;/pig-n.n.n/bin:$PATH
276
- </pre>
277
- <p>Try the following command, to get a list of Pig commands: </p>
278
- <pre class="code">
279
- $ pig -help
280
- </pre>
281
- <p>Try the following command, to start the Grunt shell:</p>
282
- <pre class="code">
283
- $ pig
284
- </pre>
285
- <a name="N10081"></a><a name="Run+Modes"></a>
286
- <h3 class="h4">Run Modes</h3>
287
- <p>Pig has two run modes or exectypes: </p>
288
- <ul>
289
-
290
- <li>
291
- <p> Local Mode - To run Pig in local mode, you need access to a single machine. </p>
292
- </li>
293
-
294
- <li>
295
- <p> Mapreduce Mode - To run Pig in mapreduce mode, you need access to a Hadoop cluster and HDFS installation.
296
- Pig will automatically allocate and deallocate a 15-node cluster.</p>
297
- </li>
298
-
299
- </ul>
300
- <p>You can run the Grunt shell, Pig scripts, or embedded programs using either mode.</p>
301
- <a name="N10099"></a><a name="Grunt+Shell"></a>
302
- <h3 class="h4">Grunt Shell</h3>
303
- <p>Use Pig's interactive shell, Grunt, to enter pig commands manually. See the <a href="setup.html#Sample+Code">Sample Code</a> for instructions about the passwd file used in the example.</p>
304
- <p>You can also run or execute script files from the Grunt shell.
305
- See the <a href="piglatin_ref2.html#run">run</a> and <a href="piglatin_ref2.html#exec">exec</a> commands. </p>
306
- <p>
307
- <strong>Local Mode</strong>
308
- </p>
309
- <pre class="code">
310
- $ pig -x local
311
- </pre>
312
- <p>
313
- <strong>Mapreduce Mode</strong>
314
- </p>
315
- <pre class="code">
316
- $ pig
317
- or
318
- $ pig -x mapreduce
319
- </pre>
320
- <p>For either mode, the Grunt shell is invoked and you can enter commands at the prompt. The results are displayed to your terminal screen (if DUMP is used) or to a file (if STORE is used).
321
- </p>
322
- <pre class="code">
323
- grunt&gt; A = load 'passwd' using PigStorage(':');
324
- grunt&gt; B = foreach A generate $0 as id;
325
- grunt&gt; dump B;
326
- grunt&gt; store B;
327
- </pre>
328
- <a name="N100CA"></a><a name="Script+Files"></a>
329
- <h3 class="h4">Script Files</h3>
330
- <p>Use script files to run Pig commands as batch jobs. See the <a href="setup.html#Sample+Code">Sample Code</a> for instructions about the passwd file and the script file (id.pig) used in the example.</p>
331
- <p>
332
- <strong>Local Mode</strong>
333
- </p>
334
- <pre class="code">
335
- $ pig -x local id.pig
336
- </pre>
337
- <p>
338
- <strong>Mapreduce Mode</strong>
339
- </p>
340
- <pre class="code">
341
- $ pig id.pig
342
- or
343
- $ pig -x mapreduce id.pig
344
- </pre>
345
- <p>For either mode, the Pig Latin statements are executed and the results are displayed to your terminal screen (if DUMP is used) or to a file (if STORE is used).</p>
346
- </div>
347
-
348
-
349
-
350
- <a name="N100ED"></a><a name="Advanced+Pig"></a>
351
- <h2 class="h3">Advanced Pig</h2>
352
- <div class="section">
353
- <a name="N100F3"></a><a name="Build+Pig"></a>
354
- <h3 class="h4">Build Pig</h3>
355
- <p>To build pig, do the following:</p>
356
- <ol>
357
-
358
- <li> Check out the Pig code from SVN: <em>svn co http://svn.apache.org/repos/asf/hadoop/pig/trunk</em>. </li>
359
-
360
- <li> Build the code from the top directory: <em>ant</em>. If the build is successful, you should see the <em>pig.jar</em> created in that directory. </li>
361
-
362
- <li> Validate your <em>pig.jar</em> by running a unit test: <em>ant test</em>
363
- </li>
364
-
365
- </ol>
366
- <a name="N10117"></a><a name="Environment+Variables+and+Properties"></a>
367
- <h3 class="h4">Environment Variables and Properties</h3>
368
- <p>See <a href="setup.html#Download+Pig">Download Pig</a>.</p>
369
- <p>The Pig environment variables are described in the Pig script file, located in the /pig-n.n.n/bin directory.</p>
370
- <p>The Pig properties file, pig.properties, is located in the /pig-n.n.n/conf directory. You can specify an alternate location using the PIG_CONF_DIR environment variable.</p>
371
- <a name="N1012B"></a><a name="Run+Modes-N1012B"></a>
372
- <h3 class="h4">Run Modes</h3>
373
- <p>See <a href="setup.html#Run+Modes">Run Modes</a>. </p>
374
- <a name="N10139"></a><a name="Embedded+Programs"></a>
375
- <h3 class="h4">Embedded Programs</h3>
376
- <p>Used the embedded option to embed Pig commands in a host language and run the program.
377
- See the <a href="setup.html#Sample+Code">Sample Code</a> for instructions about the passwd file and java files (idlocal.java, idmapreduce.java) used in the examples.</p>
378
- <p>
379
- <strong>Local Mode</strong>
380
- </p>
381
- <p>From your current working directory, compile the program: </p>
382
- <pre class="code">
383
- $ javac -cp pig.jar idlocal.java
384
- </pre>
385
- <p>Note: idlocal.class is written to your current working directory. Include &ldquo;.&rdquo; in the class path when you run the program. </p>
386
- <p>From your current working directory, run the program:
387
- </p>
388
- <pre class="code">
389
- Unix: $ java -cp pig.jar:. idlocal
390
- Cygwin: $ java &ndash;cp &lsquo;.;pig.jar&rsquo; idlocal
391
- </pre>
392
- <p>To view the results, check the output file, id.out. </p>
393
- <p>
394
- <strong>Mapreduce Mode</strong>
395
- </p>
396
- <p>Point $HADOOPDIR to the directory that contains the hadoop-site.xml file. Example:
397
- </p>
398
- <pre class="code">
399
- $ export HADOOPDIR=/yourHADOOPsite/conf
400
- </pre>
401
- <p>From your current working directory, compile the program:
402
- </p>
403
- <pre class="code">
404
- $ javac -cp pig.jar idmapreduce.java
405
- </pre>
406
- <p>Note: idmapreduce.class is written to your current working directory. Include &ldquo;.&rdquo; in the class path when you run the program. </p>
407
- <p>From your current working directory, run the program:
408
- </p>
409
- <pre class="code">
410
- Unix: $ java -cp pig.jar:.:$HADOOPDIR idmapreduce
411
- Cygwin: $ java &ndash;cp &lsquo;.;pig.jar;$HADOOPDIR&rsquo; idmapreduce
412
- </pre>
413
- <p>To view the results, check the idout directory on your Hadoop system. </p>
414
- </div>
415
-
416
-
417
-
418
- <a name="N1017F"></a><a name="Sample+Code"></a>
419
- <h2 class="h3">Sample Code</h2>
420
- <div class="section">
421
- <p>The sample code is based on Pig Latin statements that extract all user IDs from the /etc/passwd file. </p>
422
- <p>Copy the /etc/passwd file to your local working directory.</p>
423
- <p>
424
- <strong>id.pig</strong>
425
- </p>
426
- <p>For the Grunt Shell and script files. </p>
427
- <pre class="code">
428
- A = load 'passwd' using PigStorage(':');
429
- B = foreach A generate $0 as id;
430
- dump B;
431
- store B into &lsquo;id.out&rsquo;;
432
- </pre>
433
- <p>
434
- <strong>idlocal.java</strong>
435
- </p>
436
- <p>For embedded programs. </p>
437
- <pre class="code">
438
- import java.io.IOException;
439
- import org.apache.pig.PigServer;
440
- public class idlocal{
441
- public static void main(String[] args) {
442
- try {
443
- PigServer pigServer = new PigServer("local");
444
- runIdQuery(pigServer, "passwd");
445
- }
446
- catch(Exception e) {
447
- }
448
- }
449
- public static void runIdQuery(PigServer pigServer, String inputFile) throws IOException {
450
- pigServer.registerQuery("A = load '" + inputFile + "' using PigStorage(':');");
451
- pigServer.registerQuery("B = foreach A generate $0 as id;");
452
- pigServer.store("B", "id.out");
453
- }
454
- }
455
- </pre>
456
- <p>
457
- <strong>idmapreduce.java</strong>
458
- </p>
459
- <p>For embedded programs. </p>
460
- <pre class="code">
461
- import java.io.IOException;
462
- import org.apache.pig.PigServer;
463
- public class idmapreduce{
464
- public static void main(String[] args) {
465
- try {
466
- PigServer pigServer = new PigServer("mapreduce");
467
- runIdQuery(pigServer, "passwd");
468
- }
469
- catch(Exception e) {
470
- }
471
- }
472
- public static void runIdQuery(PigServer pigServer, String inputFile) throws IOException {
473
- pigServer.registerQuery("A = load '" + inputFile + "' using PigStorage(':');")
474
- pigServer.registerQuery("B = foreach A generate $0 as id;");
475
- pigServer.store("B", "idout");
476
- }
477
- }
478
- </pre>
479
- </div>
480
-
481
- </div>
482
- <!--+
483
- |end content
484
- +-->
485
- <div class="clearboth">&nbsp;</div>
486
- </div>
487
- <div id="footer">
488
- <!--+
489
- |start bottomstrip
490
- +-->
491
- <div class="lastmodified">
492
- <script type="text/javascript"><!--
493
- document.write("Last Published: " + document.lastModified);
494
- // --></script>
495
- </div>
496
- <div class="copyright">
497
- Copyright &copy;
498
- 2007-2010 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
499
- </div>
500
- <!--+
501
- |end bottomstrip
502
- +-->
503
- </div>
504
- </body>
505
- </html>