RubyGems - wukong - Versions diffs - 3.0.0.pre → 3.0.0.pre2 - Mend

wukong 3.0.0.pre → 3.0.0.pre2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (476) hide show

data/.gitignore +46 -33
data/.gitmodules +3 -0
data/.rspec +1 -1
data/.travis.yml +8 -1
data/.yardopts +0 -13
data/Guardfile +4 -6
data/{LICENSE.textile → LICENSE.md} +43 -55
data/README-old.md +422 -0
data/README.md +279 -418
data/Rakefile +21 -5
data/TODO.md +6 -6
data/bin/wu-clean-encoding +31 -0
data/bin/wu-lign +2 -2
data/bin/wu-local +69 -0
data/bin/wu-server +70 -0
data/examples/Gemfile +38 -0
data/examples/README.md +9 -0
data/examples/dataflow/apache_log_line.rb +64 -25
data/examples/dataflow/fibonacci_series.rb +101 -0
data/examples/dataflow/parse_apache_logs.rb +37 -7
data/examples/{dataflow.rb → dataflow/scraper_macro_flow.rb} +0 -0
data/examples/dataflow/simple.rb +4 -4
data/examples/geo.rb +4 -0
data/examples/geo/geo_grids.numbers +0 -0
data/examples/geo/geolocated.rb +331 -0
data/examples/geo/quadtile.rb +69 -0
data/examples/geo/spec/geolocated_spec.rb +247 -0
data/examples/geo/tile_fetcher.rb +77 -0
data/examples/graph/minimum_spanning_tree.rb +61 -61
data/examples/jabberwocky.txt +36 -0
data/examples/models/wikipedia.rb +20 -0
data/examples/munging/Gemfile +8 -0
data/examples/munging/airline_flights/airline.rb +57 -0
data/examples/munging/airline_flights/airline_flights.rake +83 -0
data/{lib/wukong/settings.rb → examples/munging/airline_flights/airplane.rb} +0 -0
data/examples/munging/airline_flights/airport.rb +211 -0
data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
data/examples/munging/airline_flights/flight.rb +156 -0
data/examples/munging/airline_flights/models.rb +4 -0
data/examples/munging/airline_flights/parse.rb +26 -0
data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
data/examples/munging/airline_flights/route.rb +35 -0
data/examples/munging/airline_flights/tasks.rake +83 -0
data/examples/munging/airline_flights/timezone_fixup.rb +62 -0
data/examples/munging/airline_flights/topcities.rb +167 -0
data/examples/munging/airports/40_wbans.txt +40 -0
data/examples/munging/airports/filter_weather_reports.rb +37 -0
data/examples/munging/airports/join.pig +31 -0
data/examples/munging/airports/to_tsv.rb +33 -0
data/examples/munging/airports/usa_wbans.pig +19 -0
data/examples/munging/airports/usa_wbans.txt +2157 -0
data/examples/munging/airports/wbans.pig +19 -0
data/examples/munging/airports/wbans.txt +2310 -0
data/examples/munging/geo/geo_json.rb +54 -0
data/examples/munging/geo/geo_models.rb +69 -0
data/examples/munging/geo/geonames_models.rb +78 -0
data/examples/munging/geo/iso_codes.rb +172 -0
data/examples/munging/geo/reconcile_countries.rb +124 -0
data/examples/munging/geo/tasks.rake +71 -0
data/examples/munging/rake_helper.rb +62 -0
data/examples/munging/weather/.gitignore +1 -0
data/examples/munging/weather/Gemfile +4 -0
data/examples/munging/weather/Rakefile +28 -0
data/examples/munging/weather/extract_ish.rb +13 -0
data/examples/munging/weather/models/weather.rb +119 -0
data/examples/munging/weather/utils/noaa_downloader.rb +46 -0
data/examples/munging/wikipedia/README.md +34 -0
data/examples/munging/wikipedia/Rakefile +193 -0
data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +4 -0
data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
data/examples/munging/wikipedia/dbpedia/extract_links.rb +193 -0
data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig +18 -0
data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb +21 -0
data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old +27 -0
data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig +29 -0
data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb +14 -0
data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old +25 -0
data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig +29 -0
data/examples/munging/wikipedia/pageviews/augment_pageviews.pig +32 -0
data/examples/munging/wikipedia/pageviews/extract_pageviews.rb +85 -0
data/examples/munging/wikipedia/pig_style_guide.md +25 -0
data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig +19 -0
data/examples/munging/wikipedia/subuniverse/sub_articles.pig +23 -0
data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig +24 -0
data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig +22 -0
data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig +22 -0
data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig +26 -0
data/examples/munging/wikipedia/subuniverse/sub_pageviews.pig +29 -0
data/examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig +24 -0
data/examples/munging/wikipedia/utils/get_namespaces.rb +86 -0
data/examples/munging/wikipedia/utils/munging_utils.rb +68 -0
data/examples/munging/wikipedia/utils/namespaces.json +1 -0
data/examples/rake_helper.rb +85 -0
data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
data/examples/server_logs/logline.rb +95 -0
data/examples/server_logs/models.rb +66 -0
data/examples/server_logs/page_counts.pig +48 -0
data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
data/{old/examples/server_logs/breadcrumbs.rb → examples/server_logs/server_logs-03-breadcrumbs-full.rb} +26 -30
data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
data/examples/string_reverser.rb +26 -0
data/examples/text/pig_latin.rb +2 -2
data/examples/text/regional_flavor/README.md +14 -0
data/examples/text/regional_flavor/article_wordbags.pig +39 -0
data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
data/examples/word_count/accumulator.rb +26 -0
data/examples/word_count/tokenizer.rb +13 -0
data/examples/word_count/word_count.rb +6 -0
data/examples/workflow/cherry_pie.dot +97 -0
data/examples/workflow/cherry_pie.png +0 -0
data/examples/workflow/cherry_pie.rb +61 -26
data/lib/hanuman.rb +34 -7
data/lib/hanuman/graph.rb +55 -31
data/lib/hanuman/graphvizzer.rb +199 -178
data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
data/lib/hanuman/link.rb +35 -0
data/lib/hanuman/registry.rb +46 -0
data/lib/hanuman/stage.rb +76 -32
data/lib/wukong.rb +23 -24
data/lib/wukong/boot.rb +87 -0
data/lib/wukong/configuration.rb +8 -0
data/lib/wukong/dataflow.rb +45 -78
data/lib/wukong/driver.rb +99 -0
data/lib/wukong/emitter.rb +22 -0
data/lib/wukong/model/faker.rb +24 -24
data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
data/lib/wukong/processor.rb +60 -114
data/lib/wukong/spec_helpers.rb +81 -0
data/lib/wukong/spec_helpers/integration_driver.rb +144 -0
data/lib/wukong/spec_helpers/integration_driver_matchers.rb +219 -0
data/lib/wukong/spec_helpers/processor_helpers.rb +95 -0
data/lib/wukong/spec_helpers/processor_methods.rb +108 -0
data/lib/wukong/spec_helpers/shared_examples.rb +15 -0
data/lib/wukong/spec_helpers/spec_driver.rb +28 -0
data/lib/wukong/spec_helpers/spec_driver_matchers.rb +195 -0
data/lib/wukong/version.rb +2 -1
data/lib/wukong/widget/filters.rb +311 -0
data/lib/wukong/widget/processors.rb +156 -0
data/lib/wukong/widget/reducers.rb +7 -0
data/lib/wukong/widget/reducers/accumulator.rb +73 -0
data/lib/wukong/widget/reducers/bin.rb +318 -0
data/lib/wukong/widget/reducers/count.rb +61 -0
data/lib/wukong/widget/reducers/group.rb +85 -0
data/lib/wukong/widget/reducers/group_concat.rb +70 -0
data/lib/wukong/widget/reducers/moments.rb +72 -0
data/lib/wukong/widget/reducers/sort.rb +130 -0
data/lib/wukong/widget/serializers.rb +287 -0
data/lib/wukong/widget/sink.rb +10 -52
data/lib/wukong/widget/source.rb +7 -113
data/lib/wukong/widget/utils.rb +46 -0
data/lib/wukong/widgets.rb +6 -0
data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
data/spec/examples/dataflow/parsing_spec.rb +12 -11
data/spec/examples/dataflow/simple_spec.rb +32 -6
data/spec/examples/dataflow/telegram_spec.rb +36 -36
data/spec/examples/graph/minimum_spanning_tree_spec.rb +30 -31
data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
data/spec/examples/munging/airline_flights_spec.rb +202 -0
data/spec/examples/text/pig_latin_spec.rb +13 -16
data/spec/examples/workflow/cherry_pie_spec.rb +34 -4
data/spec/hanuman/graph_spec.rb +27 -2
data/spec/hanuman/hanuman_spec.rb +10 -0
data/spec/hanuman/registry_spec.rb +123 -0
data/spec/hanuman/stage_spec.rb +61 -7
data/spec/spec_helper.rb +29 -19
data/spec/support/hanuman_test_helpers.rb +14 -12
data/spec/support/shared_context_for_reducers.rb +37 -0
data/spec/support/shared_examples_for_builders.rb +101 -0
data/spec/support/shared_examples_for_shortcuts.rb +57 -0
data/spec/support/wukong_test_helpers.rb +37 -11
data/spec/wukong/dataflow_spec.rb +77 -55
data/spec/wukong/local_runner_spec.rb +24 -24
data/spec/wukong/model/faker_spec.rb +132 -131
data/spec/wukong/runner_spec.rb +8 -8
data/spec/wukong/widget/filters_spec.rb +61 -0
data/spec/wukong/widget/processors_spec.rb +126 -0
data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
data/spec/wukong/widget/reducers/count_spec.rb +11 -0
data/spec/wukong/widget/reducers/group_spec.rb +20 -0
data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
data/spec/wukong/widget/serializers_spec.rb +92 -0
data/spec/wukong/widget/sink_spec.rb +15 -15
data/spec/wukong/widget/source_spec.rb +65 -41
data/spec/wukong/wukong_spec.rb +10 -0
data/wukong.gemspec +17 -10
metadata +359 -335
data/.document +0 -5
data/VERSION +0 -1
data/bin/hdp-bin +0 -44
data/bin/hdp-bzip +0 -23
data/bin/hdp-cat +0 -3
data/bin/hdp-catd +0 -3
data/bin/hdp-cp +0 -3
data/bin/hdp-du +0 -86
data/bin/hdp-get +0 -3
data/bin/hdp-kill +0 -3
data/bin/hdp-kill-task +0 -3
data/bin/hdp-ls +0 -11
data/bin/hdp-mkdir +0 -2
data/bin/hdp-mkdirp +0 -12
data/bin/hdp-mv +0 -3
data/bin/hdp-parts_to_keys.rb +0 -77
data/bin/hdp-ps +0 -3
data/bin/hdp-put +0 -3
data/bin/hdp-rm +0 -32
data/bin/hdp-sort +0 -40
data/bin/hdp-stream +0 -40
data/bin/hdp-stream-flat +0 -22
data/bin/hdp-stream2 +0 -39
data/bin/hdp-sync +0 -17
data/bin/hdp-wc +0 -67
data/bin/wu-flow +0 -10
data/bin/wu-map +0 -17
data/bin/wu-red +0 -17
data/bin/wukong +0 -17
data/data/CREDITS.md +0 -355
data/data/graph/airfares.tsv +0 -2174
data/data/text/gift_of_the_magi.txt +0 -225
data/data/text/jabberwocky.txt +0 -36
data/data/text/rectification_of_names.txt +0 -33
data/data/twitter/a_atsigns_b.tsv +0 -64
data/data/twitter/a_follows_b.tsv +0 -53
data/data/twitter/tweet.tsv +0 -167
data/data/twitter/twitter_user.tsv +0 -55
data/data/wikipedia/dbpedia-sentences.tsv +0 -1000
data/docpages/INSTALL.textile +0 -92
data/docpages/LICENSE.textile +0 -107
data/docpages/README-elastic_map_reduce.textile +0 -377
data/docpages/README-performance.textile +0 -90
data/docpages/README-wulign.textile +0 -65
data/docpages/UsingWukong-part1-get_ready.textile +0 -17
data/docpages/UsingWukong-part2-ThinkingBigData.textile +0 -75
data/docpages/UsingWukong-part3-parsing.textile +0 -138
data/docpages/_config.yml +0 -39
data/docpages/avro/avro_notes.textile +0 -56
data/docpages/avro/performance.textile +0 -36
data/docpages/avro/tethering.textile +0 -19
data/docpages/bigdata-tips.textile +0 -143
data/docpages/code/api_response_example.txt +0 -20
data/docpages/code/parser_skeleton.rb +0 -38
data/docpages/diagrams/MapReduceDiagram.graffle +0 -0
data/docpages/favicon.ico +0 -0
data/docpages/gem.css +0 -16
data/docpages/hadoop-tips.textile +0 -83
data/docpages/index.textile +0 -92
data/docpages/intro.textile +0 -8
data/docpages/moreinfo.textile +0 -174
data/docpages/news.html +0 -24
data/docpages/pig/PigLatinExpressionsList.txt +0 -122
data/docpages/pig/PigLatinReferenceManual.txt +0 -1640
data/docpages/pig/commandline_params.txt +0 -26
data/docpages/pig/cookbook.html +0 -481
data/docpages/pig/images/hadoop-logo.jpg +0 -0
data/docpages/pig/images/instruction_arrow.png +0 -0
data/docpages/pig/images/pig-logo.gif +0 -0
data/docpages/pig/piglatin_ref1.html +0 -1103
data/docpages/pig/piglatin_ref2.html +0 -14340
data/docpages/pig/setup.html +0 -505
data/docpages/pig/skin/basic.css +0 -166
data/docpages/pig/skin/breadcrumbs.js +0 -237
data/docpages/pig/skin/fontsize.js +0 -166
data/docpages/pig/skin/getBlank.js +0 -40
data/docpages/pig/skin/getMenu.js +0 -45
data/docpages/pig/skin/images/chapter.gif +0 -0
data/docpages/pig/skin/images/chapter_open.gif +0 -0
data/docpages/pig/skin/images/current.gif +0 -0
data/docpages/pig/skin/images/external-link.gif +0 -0
data/docpages/pig/skin/images/header_white_line.gif +0 -0
data/docpages/pig/skin/images/page.gif +0 -0
data/docpages/pig/skin/images/pdfdoc.gif +0 -0
data/docpages/pig/skin/images/rc-b-l-15-1body-2menu-3menu.png +0 -0
data/docpages/pig/skin/images/rc-b-r-15-1body-2menu-3menu.png +0 -0
data/docpages/pig/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png +0 -0
data/docpages/pig/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png +0 -0
data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png +0 -0
data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png +0 -0
data/docpages/pig/skin/images/rc-t-r-15-1body-2menu-3menu.png +0 -0
data/docpages/pig/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png +0 -0
data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png +0 -0
data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png +0 -0
data/docpages/pig/skin/print.css +0 -54
data/docpages/pig/skin/profile.css +0 -181
data/docpages/pig/skin/screen.css +0 -587
data/docpages/pig/tutorial.html +0 -1059
data/docpages/pig/udf.html +0 -1509
data/docpages/tutorial.textile +0 -283
data/docpages/usage.textile +0 -195
data/docpages/wutils.textile +0 -263
data/examples/dataflow/complex.rb +0 -11
data/examples/dataflow/donuts.rb +0 -13
data/examples/tiny_count/jabberwocky_output.tsv +0 -92
data/examples/word_count.rb +0 -48
data/examples/workflow/fiddle.rb +0 -24
data/lib/away/escapement.rb +0 -129
data/lib/away/exe.rb +0 -11
data/lib/away/experimental.rb +0 -5
data/lib/away/from_file.rb +0 -52
data/lib/away/job.rb +0 -56
data/lib/away/job/rake_compat.rb +0 -17
data/lib/away/registry.rb +0 -79
data/lib/away/runner.rb +0 -276
data/lib/away/runner/execute.rb +0 -121
data/lib/away/script.rb +0 -161
data/lib/away/script/hadoop_command.rb +0 -240
data/lib/away/source/file_list_source.rb +0 -15
data/lib/away/source/looper.rb +0 -18
data/lib/away/task.rb +0 -219
data/lib/hanuman/action.rb +0 -21
data/lib/hanuman/chain.rb +0 -4
data/lib/hanuman/graphviz.rb +0 -74
data/lib/hanuman/resource.rb +0 -6
data/lib/hanuman/slot.rb +0 -87
data/lib/hanuman/slottable.rb +0 -220
data/lib/wukong/bad_record.rb +0 -15
data/lib/wukong/event.rb +0 -44
data/lib/wukong/local_runner.rb +0 -55
data/lib/wukong/mapred.rb +0 -3
data/lib/wukong/universe.rb +0 -48
data/lib/wukong/widget/filter.rb +0 -81
data/lib/wukong/widget/gibberish.rb +0 -123
data/lib/wukong/widget/monitor.rb +0 -26
data/lib/wukong/widget/reducer.rb +0 -66
data/lib/wukong/widget/stringifier.rb +0 -50
data/lib/wukong/workflow.rb +0 -22
data/lib/wukong/workflow/command.rb +0 -42
data/old/config/emr-example.yaml +0 -48
data/old/examples/README.txt +0 -17
data/old/examples/contrib/jeans/README.markdown +0 -165
data/old/examples/contrib/jeans/data/normalized_sizes +0 -3
data/old/examples/contrib/jeans/data/orders.tsv +0 -1302
data/old/examples/contrib/jeans/data/sizes +0 -3
data/old/examples/contrib/jeans/normalize.rb +0 -20
data/old/examples/contrib/jeans/sizes.rb +0 -55
data/old/examples/corpus/bnc_word_freq.rb +0 -44
data/old/examples/corpus/bucket_counter.rb +0 -47
data/old/examples/corpus/dbpedia_abstract_to_sentences.rb +0 -86
data/old/examples/corpus/sentence_bigrams.rb +0 -53
data/old/examples/corpus/sentence_coocurrence.rb +0 -66
data/old/examples/corpus/stopwords.rb +0 -138
data/old/examples/corpus/words_to_bigrams.rb +0 -53
data/old/examples/emr/README.textile +0 -110
data/old/examples/emr/dot_wukong_dir/credentials.json +0 -7
data/old/examples/emr/dot_wukong_dir/emr.yaml +0 -69
data/old/examples/emr/dot_wukong_dir/emr_bootstrap.sh +0 -33
data/old/examples/emr/elastic_mapreduce_example.rb +0 -28
data/old/examples/network_graph/adjacency_list.rb +0 -74
data/old/examples/network_graph/breadth_first_search.rb +0 -72
data/old/examples/network_graph/gen_2paths.rb +0 -68
data/old/examples/network_graph/gen_multi_edge.rb +0 -112
data/old/examples/network_graph/gen_symmetric_links.rb +0 -64
data/old/examples/pagerank/README.textile +0 -6
data/old/examples/pagerank/gen_initial_pagerank_graph.pig +0 -57
data/old/examples/pagerank/pagerank.rb +0 -72
data/old/examples/pagerank/pagerank_initialize.rb +0 -42
data/old/examples/pagerank/run_pagerank.sh +0 -21
data/old/examples/sample_records.rb +0 -33
data/old/examples/server_logs/apache_log_parser.rb +0 -15
data/old/examples/server_logs/nook.rb +0 -48
data/old/examples/server_logs/nook/faraday_dummy_adapter.rb +0 -94
data/old/examples/server_logs/user_agent.rb +0 -40
data/old/examples/simple_word_count.rb +0 -82
data/old/examples/size.rb +0 -61
data/old/examples/stats/avg_value_frequency.rb +0 -86
data/old/examples/stats/binning_percentile_estimator.rb +0 -140
data/old/examples/stats/data/avg_value_frequency.tsv +0 -3
data/old/examples/stats/rank_and_bin.rb +0 -173
data/old/examples/stupidly_simple_filter.rb +0 -40
data/old/examples/word_count.rb +0 -75
data/old/graph/graphviz_builder.rb +0 -580
data/old/graph_easy/Attributes.pm +0 -4181
data/old/graph_easy/Graphviz.pm +0 -2232
data/old/wukong.rb +0 -18
data/old/wukong/and_pig.rb +0 -38
data/old/wukong/bad_record.rb +0 -18
data/old/wukong/datatypes.rb +0 -24
data/old/wukong/datatypes/enum.rb +0 -127
data/old/wukong/datatypes/fake_types.rb +0 -17
data/old/wukong/decorator.rb +0 -28
data/old/wukong/encoding/asciize.rb +0 -108
data/old/wukong/extensions.rb +0 -16
data/old/wukong/extensions/array.rb +0 -18
data/old/wukong/extensions/blank.rb +0 -93
data/old/wukong/extensions/class.rb +0 -189
data/old/wukong/extensions/date_time.rb +0 -53
data/old/wukong/extensions/emittable.rb +0 -69
data/old/wukong/extensions/enumerable.rb +0 -79
data/old/wukong/extensions/hash.rb +0 -167
data/old/wukong/extensions/hash_keys.rb +0 -16
data/old/wukong/extensions/hash_like.rb +0 -150
data/old/wukong/extensions/hashlike_class.rb +0 -47
data/old/wukong/extensions/module.rb +0 -2
data/old/wukong/extensions/pathname.rb +0 -27
data/old/wukong/extensions/string.rb +0 -65
data/old/wukong/extensions/struct.rb +0 -17
data/old/wukong/extensions/symbol.rb +0 -11
data/old/wukong/filename_pattern.rb +0 -74
data/old/wukong/helper.rb +0 -7
data/old/wukong/helper/stopwords.rb +0 -195
data/old/wukong/helper/tokenize.rb +0 -35
data/old/wukong/logger.rb +0 -38
data/old/wukong/periodic_monitor.rb +0 -72
data/old/wukong/schema.rb +0 -269
data/old/wukong/script.rb +0 -286
data/old/wukong/script/avro_command.rb +0 -5
data/old/wukong/script/cassandra_loader_script.rb +0 -40
data/old/wukong/script/emr_command.rb +0 -168
data/old/wukong/script/hadoop_command.rb +0 -237
data/old/wukong/script/local_command.rb +0 -41
data/old/wukong/store.rb +0 -10
data/old/wukong/store/base.rb +0 -27
data/old/wukong/store/cassandra.rb +0 -10
data/old/wukong/store/cassandra/streaming.rb +0 -75
data/old/wukong/store/cassandra/struct_loader.rb +0 -21
data/old/wukong/store/cassandra_model.rb +0 -91
data/old/wukong/store/chh_chunked_flat_file_store.rb +0 -37
data/old/wukong/store/chunked_flat_file_store.rb +0 -48
data/old/wukong/store/conditional_store.rb +0 -57
data/old/wukong/store/factory.rb +0 -8
data/old/wukong/store/flat_file_store.rb +0 -89
data/old/wukong/store/key_store.rb +0 -51
data/old/wukong/store/null_store.rb +0 -15
data/old/wukong/store/read_thru_store.rb +0 -22
data/old/wukong/store/tokyo_tdb_key_store.rb +0 -33
data/old/wukong/store/tyrant_rdb_key_store.rb +0 -57
data/old/wukong/store/tyrant_tdb_key_store.rb +0 -20
data/old/wukong/streamer.rb +0 -30
data/old/wukong/streamer/accumulating_reducer.rb +0 -83
data/old/wukong/streamer/base.rb +0 -126
data/old/wukong/streamer/counting_reducer.rb +0 -25
data/old/wukong/streamer/filter.rb +0 -20
data/old/wukong/streamer/instance_streamer.rb +0 -15
data/old/wukong/streamer/json_streamer.rb +0 -21
data/old/wukong/streamer/line_streamer.rb +0 -12
data/old/wukong/streamer/list_reducer.rb +0 -31
data/old/wukong/streamer/rank_and_bin_reducer.rb +0 -145
data/old/wukong/streamer/record_streamer.rb +0 -14
data/old/wukong/streamer/reducer.rb +0 -11
data/old/wukong/streamer/set_reducer.rb +0 -14
data/old/wukong/streamer/struct_streamer.rb +0 -48
data/old/wukong/streamer/summing_reducer.rb +0 -29
data/old/wukong/streamer/uniq_by_last_reducer.rb +0 -51
data/old/wukong/typed_struct.rb +0 -12
data/spec/away/encoding_spec.rb +0 -32
data/spec/away/exe_spec.rb +0 -20
data/spec/away/flow_spec.rb +0 -82
data/spec/away/graph_spec.rb +0 -6
data/spec/away/job_spec.rb +0 -15
data/spec/away/rake_compat_spec.rb +0 -9
data/spec/away/script_spec.rb +0 -81
data/spec/hanuman/graphviz_spec.rb +0 -29
data/spec/hanuman/slot_spec.rb +0 -2
data/spec/support/examples_helper.rb +0 -10
data/spec/support/streamer_test_helpers.rb +0 -6
data/spec/support/wukong_widget_helpers.rb +0 -66
data/spec/wukong/processor_spec.rb +0 -109
data/spec/wukong/widget/filter_spec.rb +0 -99
data/spec/wukong/widget/stringifier_spec.rb +0 -51
data/spec/wukong/workflow/command_spec.rb +0 -5

data/docpages/pig/tutorial.html DELETED

@@ -1,1059 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
-<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
-<meta name="Forrest-skin-name" content="pelt">
-<title>Pig Tutorial</title>
-<link type="text/css" href="skin/basic.css" rel="stylesheet">
-<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
-<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
-<link type="text/css" href="skin/profile.css" rel="stylesheet">
-<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
-<link rel="shortcut icon" href="">
-</head>
-<body onload="init()">
-<script type="text/javascript">ndeSetTextSize();</script>
-<div id="top">
-<!--+
-    |breadtrail
-    +-->
-<div class="breadtrail">
-<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/pig/">Pig</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
-</div>
-<!--+
-    |header
-    +-->
-<div class="header">
-<!--+
-    |start group logo
-    +-->
-<div class="grouplogo">
-<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
-</div>
-<!--+
-    |end group logo
-    +-->
-<!--+
-    |start Project Logo
-    +-->
-<div class="projectlogo">
-<a href="http://hadoop.apache.org/pig/"><img class="logoImage" alt="Pig" src="images/pig-logo.gif" title="A platform for analyzing large datasets."></a>
-</div>
-<!--+
-    |end Project Logo
-    +-->
-<!--+
-    |start Search
-    +-->
-<div class="searchbox">
-<form action="http://www.google.com/search" method="get" class="roundtopsmall">
-<input value="" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
-                    <input name="Search" value="Search" type="submit">
-</form>
-</div>
-<!--+
-    |end search
-    +-->
-<!--+
-    |start Tabs
-    +-->
-<ul id="tabs">
-<li>
-<a class="unselected" href="http://hadoop.apache.org/pig/">Project</a>
-</li>
-<li>
-<a class="unselected" href="http://wiki.apache.org/pig/">Wiki</a>
-</li>
-<li class="current">
-<a class="selected" href="index.html">Pig 0.7.0 Documentation</a>
-</li>
-</ul>
-<!--+
-    |end Tabs
-    +-->
-</div>
-</div>
-<div id="main">
-<div id="publishedStrip">
-<!--+
-    |start Subtabs
-    +-->
-<div id="level2tabs"></div>
-<!--+
-    |end Endtabs
-    +-->
-<script type="text/javascript"><!--
-document.write("Last Published: " + document.lastModified);
-//  --></script>
-</div>
-<!--+
-    |breadtrail
-    +-->
-<div class="breadtrail">
-             &nbsp;
-           </div>
-<!--+
-    |start Menu, mainarea
-    +-->
-<!--+
-    |start Menu
-    +-->
-<div id="menu">
-<div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Pig</div>
-<div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
-<div class="menuitem">
-<a href="index.html">Overview</a>
-</div>
-<div class="menuitem">
-<a href="setup.html">Setup</a>
-</div>
-<div class="menupage">
-<div class="menupagetitle">Tutorial</div>
-</div>
-<div class="menuitem">
-<a href="piglatin_ref1.html">Pig Latin 1</a>
-</div>
-<div class="menuitem">
-<a href="piglatin_ref2.html">Pig Latin 2</a>
-</div>
-<div class="menuitem">
-<a href="cookbook.html">Cookbook</a>
-</div>
-<div class="menuitem">
-<a href="udf.html">UDFs</a>
-</div>
-</div>
-<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Zebra</div>
-<div id="menu_1.2" class="menuitemgroup">
-<div class="menuitem">
-<a href="zebra_overview.html">Zebra Overview </a>
-</div>
-<div class="menuitem">
-<a href="zebra_users.html">Zebra Users </a>
-</div>
-<div class="menuitem">
-<a href="zebra_reference.html">Zebra Reference </a>
-</div>
-<div class="menuitem">
-<a href="zebra_mapreduce.html">Zebra MapReduce </a>
-</div>
-<div class="menuitem">
-<a href="zebra_pig.html">Zebra Pig </a>
-</div>
-<div class="menuitem">
-<a href="zebra_stream.html">Zebra Streaming </a>
-</div>
-</div>
-<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Miscellaneous</div>
-<div id="menu_1.3" class="menuitemgroup">
-<div class="menuitem">
-<a href="api/">API Docs</a>
-</div>
-<div class="menuitem">
-<a href="http://wiki.apache.org/pig/">Wiki</a>
-</div>
-<div class="menuitem">
-<a href="http://wiki.apache.org/pig/FAQ">FAQ</a>
-</div>
-<div class="menuitem">
-<a href="http://hadoop.apache.org/pig/releases.html">Release Notes</a>
-</div>
-</div>
-<div id="credit"></div>
-<div id="roundbottom">
-<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
-<!--+
-  |alternative credits
-  +-->
-<div id="credit2"></div>
-</div>
-<!--+
-    |end Menu
-    +-->
-<!--+
-    |start content
-    +-->
-<div id="content">
-<div title="Portable Document Format" class="pdflink">
-<a class="dida" href="tutorial.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
-        PDF</a>
-</div>
-<h1>Pig Tutorial</h1>
-<div id="minitoc-area">
-<ul class="minitoc">
-<li>
-<a href="#Overview">Overview</a>
-</li>
-<li>
-<a href="#Java+Installation"> Java Installation</a>
-</li>
-<li>
-<a href="#Pig+Installation"> Pig Installation</a>
-</li>
-<li>
-<a href="#Running+the+Pig+Scripts+in+Local+Mode"> Running the Pig Scripts in Local Mode</a>
-</li>
-<li>
-<a href="#Running+the+Pig+Scripts+in+Mapreduce+Mode"> Running the Pig Scripts in Mapreduce Mode</a>
-</li>
-<li>
-<a href="#Pig+Tutorial+File"> Pig Tutorial File</a>
-</li>
-<li>
-<a href="#Pig+Script+1%3A+Query+Phrase+Popularity"> Pig Script 1: Query Phrase Popularity</a>
-</li>
-<li>
-<a href="#Pig+Script+2%3A+Temporal+Query+Phrase+Popularity">Pig Script 2: Temporal Query Phrase Popularity</a>
-</li>
-</ul>
-</div>
-<a name="N1000D"></a><a name="Overview"></a>
-<h2 class="h3">Overview</h2>
-<div class="section">
-<p>The Pig tutorial shows you how to run two Pig scripts in local mode and mapreduce mode.   </p>
-<ul>
-<li>
-<p>
-<strong>Local Mode</strong>: To run the scripts in local mode, no Hadoop or HDFS installation is required. All files are installed and run from your local host and file system. </p>
-</li>
-<li>
-<p>
-<strong>Mapreduce Mode</strong>: To run the scripts in mapreduce mode, you need access to a Hadoop cluster and HDFS installation. </p>
-</li>
-</ul>
-<p>The Pig tutorial file (tutorial/pigtutorial.tar.gz file in the pig distribution) includes the Pig JAR file (pig.jar) and the tutorial files (tutorial.jar, Pigs scripts, log files).
-These files work with Hadoop 0.20.2 and include everything you need to run the Pig scripts.</p>
-<p>To get started, follow these basic steps:  </p>
-<ol>
-<li>
-<p>Install Java </p>
-</li>
-<li>
-<p>Install Pig </p>
-</li>
-<li>
-<p>Run the Pig scripts - in Local or Hadoop mode </p>
-</li>
-</ol>
-</div>
-<a name="N10042"></a><a name="Java+Installation"></a>
-<h2 class="h3"> Java Installation</h2>
-<div class="section">
-<p>Make sure your run-time environment includes the following: </p>
-<ul>
-<li>
-<p>Java 1.6 or higher (preferably from Sun) </p>
-</li>
-<li>
-<p>The JAVA_HOME environment variable is set the root of your Java installation.  </p>
-</li>
-</ul>
-</div>
-<a name="N10059"></a><a name="Pig+Installation"></a>
-<h2 class="h3"> Pig Installation</h2>
-<div class="section">
-<p>To install Pig, do the following: </p>
-<ol>
-<li>
-<p>Download the Pig tutorial file to your local directory. </p>
-</li>
-<li>
-<p>Unzip the Pig tutorial file (the files are stored in a newly created directory, pigtmp). </p>
-<pre class="code">
-$ tar -xzf pigtutorial.tar.gz
-</pre>
-</li>
-<li>
-<p>Move to the pigtmp directory.  </p>
-</li>
-<li>
-<p>Review the contents of the Pig tutorial file. </p>
-</li>
-<li>
-<p>Copy the <strong>pig.jar</strong> file to the appropriate directory on your system. For example: /home/me/pig.  </p>
-</li>
-<li>
-<p>Create an environment variable, <strong>PIGDIR</strong>, and point it to your directory; for example, export PIGDIR=/home/me/pig (bash, sh) or setenv PIGDIR /home/me/pig (tcsh, csh).  </p>
-</li>
-</ol>
-</div>
-<a name="N1008E"></a><a name="Running+the+Pig+Scripts+in+Local+Mode"></a>
-<h2 class="h3"> Running the Pig Scripts in Local Mode</h2>
-<div class="section">
-<p>To run the Pig scripts in local mode, do the following: </p>
-<ol>
-<li>
-<p>Set the maximum memory for Java.</p>
-<pre class="code">
-java -Xmx256m -cp pig.jar org.apache.pig.Main -x local script1-local.pig
-java -Xmx256m -cp pig.jar org.apache.pig.Main -x local script2-local.pig
-</pre>
-</li>
-<li>
-<p>Move to the pigtmp directory. </p>
-</li>
-<li>
-<p>Review Pig Script 1 and Pig Script 2. </p>
-</li>
-<li>
-<p>Execute the following command (using either script1-local.pig or script2-local.pig). </p>
-<pre class="code">
-$ java -cp $PIGDIR/pig.jar org.apache.pig.Main -x local script1-local.pig
-</pre>
-</li>
-<li>
-<p>Review the result files, located in the part-r-00000 directory.</p>
-<p>The output may contain a few Hadoop warnings which can be ignored:</p>
-<pre class="code">
-2010-04-08 12:55:33,642 [main] INFO  org.apache.hadoop.metrics.jvm.JvmMetrics
-- Cannot initialize JVM Metrics with processName=JobTracker, sessionId= - already initialized
-</pre>
-</li>
-</ol>
-</div>
-<a name="N100C3"></a><a name="Running+the+Pig+Scripts+in+Mapreduce+Mode"></a>
-<h2 class="h3"> Running the Pig Scripts in Mapreduce Mode</h2>
-<div class="section">
-<p>To run the Pig scripts in mapreduce mode, do the following: </p>
-<ol>
-<li>
-<p>Move to the pigtmp directory. </p>
-</li>
-<li>
-<p>Review Pig Script 1 and Pig Script 2. </p>
-</li>
-<li>
-<p>Copy the excite.log.bz2 file from the pigtmp directory to the HDFS directory. </p>
-<pre class="code">
-$ hadoop fs &ndash;copyFromLocal excite.log.bz2 .
-</pre>
-</li>
-<li>
-<p>Set the HADOOP_CONF_DIR environment variable to the location of your core-site.xml, hdfs-site.xml and mapred-site.xml files. </p>
-</li>
-<li>
-<p>Execute the following command (using either script1-hadoop.pig or script2-hadoop.pig): </p>
-<pre class="code">
-$ java -cp $PIGDIR/pig.jar:$HADOOP_CONF_DIR  org.apache.pig.Main script1-hadoop.pig
-</pre>
-</li>
-<li>
-<p>Review the result files, located in the script1-hadoop-results or script2-hadoop-results HDFS directory: </p>
-<pre class="code">
-$ hadoop fs -ls script1-hadoop-results
-$ hadoop fs -cat 'script1-hadoop-results/*' | less
-</pre>
-</li>
-</ol>
-</div>
-<a name="N100FA"></a><a name="Pig+Tutorial+File"></a>
-<h2 class="h3"> Pig Tutorial File</h2>
-<div class="section">
-<p>The contents of the Pig tutorial file (pigtutorial.tar.gz) are described here. </p>
-<table class="ForrestTable" cellspacing="1" cellpadding="4">
-<tr>
-<td colspan="1" rowspan="1">
-<p>
-<strong>File</strong>
-</p>
-</td>
-<td colspan="1" rowspan="1">
-<p>
-<strong>Description</strong>
-</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> pig.jar </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig JAR file </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> tutorial.jar </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> User-defined functions (UDFs) and Java classes </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script1-local.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 1, Query Phrase Popularity (local mode) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script1-hadoop.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 1, Query Phrase Popularity (Hadoop cluster) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script2-local.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 2, Temporal Query Phrase Popularity (local mode)</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script2-hadoop.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 2, Temporal Query Phrase Popularity (Hadoop cluster) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> excite-small.log </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Log file, Excite search engine (local mode) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> excite.log.bz2 </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Log file, Excite search engine (Hadoop cluster) </p>
-</td>
-</tr>
-</table>
-<p>The user-defined functions (UDFs) are described here. </p>
-<table class="ForrestTable" cellspacing="1" cellpadding="4">
-<tr>
-<td colspan="1" rowspan="1">
-<p>
-<strong>UDF</strong>
-</p>
-</td>
-<td colspan="1" rowspan="1">
-<p>
-<strong>Description</strong>
-</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> ExtractHour </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Extracts the hour from the record.</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> NGramGenerator </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Composes n-grams from the set of words. </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> NonURLDetector </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Removes the record if the query field is empty or a URL. </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> ScoreGenerator </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Calculates a "popularity" score for the n-gram.</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> ToLower </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Changes the query field to lowercase. </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> TutorialUtil </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Divides the query string into a set of words.</p>
-</td>
-</tr>
-</table>
-</div>
-<a name="N10247"></a><a name="Pig+Script+1%3A+Query+Phrase+Popularity"></a>
-<h2 class="h3"> Pig Script 1: Query Phrase Popularity</h2>
-<div class="section">
-<p>The Query Phrase Popularity script (script1-local.pig or script1-hadoop.pig) processes a search query log file from the Excite search engine and finds search phrases that occur with particular high frequency during certain times of the day. </p>
-<p>The script is shown here: </p>
-<ul>
-<li>
-<p> Register the tutorial JAR file so that the included UDFs can be called in the script. </p>
-</li>
-</ul>
-<pre class="code">
-REGISTER ./tutorial.jar;
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to load the excite log file (excite.log or excite-small.log) into the &ldquo;raw&rdquo; bag as an array of records with the fields <strong>user</strong>, <strong>time</strong>, and <strong>query</strong>.  </p>
-</li>
-</ul>
-<pre class="code">
-raw = LOAD 'excite.log' USING PigStorage('\t') AS (user, time, query);
-</pre>
-<ul>
-<li>
-<p> Call the NonURLDetector UDF to remove records if the query field is empty or a URL.  </p>
-</li>
-</ul>
-<pre class="code">
-clean1 = FILTER raw BY org.apache.pig.tutorial.NonURLDetector(query);
-</pre>
-<ul>
-<li>
-<p> Call the ToLower UDF to change the query field to lowercase.  </p>
-</li>
-</ul>
-<pre class="code">
-clean2 = FOREACH clean1 GENERATE user, time, org.apache.pig.tutorial.ToLower(query) as query;
-</pre>
-<ul>
-<li>
-<p> Because the log file only contains queries for a single day, we are only interested in the hour. The excite query log timestamp format is YYMMDDHHMMSS. Call the ExtractHour UDF to extract the hour (HH) from the time field. </p>
-</li>
-</ul>
-<pre class="code">
-houred = FOREACH clean2 GENERATE user, org.apache.pig.tutorial.ExtractHour(time) as hour, query;
-</pre>
-<ul>
-<li>
-<p> Call the NGramGenerator UDF to compose the n-grams of the query. </p>
-</li>
-</ul>
-<pre class="code">
-ngramed1 = FOREACH houred GENERATE user, hour, flatten(org.apache.pig.tutorial.NGramGenerator(query)) as ngram;
-</pre>
-<ul>
-<li>
-<p> Use the DISTINCT operator to get the unique n-grams for all records.  </p>
-</li>
-</ul>
-<pre class="code">
-ngramed2 = DISTINCT ngramed1;
-</pre>
-<ul>
-<li>
-<p> Use the GROUP operator to group records by n-gram and hour. </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency1 = GROUP ngramed2 BY (ngram, hour);
-</pre>
-<ul>
-<li>
-<p> Use the COUNTfunction to get the count (occurrences) of each n-gram.  </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency2 = FOREACH hour_frequency1 GENERATE flatten($0), COUNT($1) as count;
-</pre>
-<ul>
-<li>
-<p> Use the GROUP operator to group records by n-gram only. Each group now corresponds to a distinct n-gram and has the count for each hour. </p>
-</li>
-</ul>
-<pre class="code">
-uniq_frequency1 = GROUP hour_frequency2 BY group::ngram;
-</pre>
-<ul>
-<li>
-<p> For each group, identify the hour in which this n-gram is used with a particularly high frequency. Call the ScoreGenerator UDF to calculate a "popularity" score for the n-gram. </p>
-</li>
-</ul>
-<pre class="code">
-uniq_frequency2 = FOREACH uniq_frequency1 GENERATE flatten($0), flatten(org.apache.pig.tutorial.ScoreGenerator($1));
-</pre>
-<ul>
-<li>
-<p> Use the FOREACH-GENERATE operator to assign names to the fields.  </p>
-</li>
-</ul>
-<pre class="code">
-uniq_frequency3 = FOREACH uniq_frequency2 GENERATE $1 as hour, $0 as ngram, $2 as score, $3 as count, $4 as mean;
-</pre>
-<ul>
-<li>
-<p> Use the FILTER operator to move all records with a score less than or equal to 2.0. </p>
-</li>
-</ul>
-<pre class="code">
-filtered_uniq_frequency = FILTER uniq_frequency3 BY score &gt; 2.0;
-</pre>
-<ul>
-<li>
-<p> Use the ORDER operator to sort the remaining records by hour and score.  </p>
-</li>
-</ul>
-<pre class="code">
-ordered_uniq_frequency = ORDER filtered_uniq_frequency BY (hour, score);
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to store the results. The output file contains a list of n-grams with the following fields: <strong>hour</strong>, <strong>ngram</strong>, <strong>score</strong>, <strong>count</strong>, <strong>mean</strong>. </p>
-</li>
-</ul>
-<pre class="code">
-STORE ordered_uniq_frequency INTO '/tmp/tutorial-results' USING PigStorage();
-</pre>
-</div>
-<a name="N10320"></a><a name="Pig+Script+2%3A+Temporal+Query+Phrase+Popularity"></a>
-<h2 class="h3">Pig Script 2: Temporal Query Phrase Popularity</h2>
-<div class="section">
-<p>The Temporal Query Phrase Popularity script (script2-local.pig or script2-hadoop.pig) processes a search query log file from the Excite search engine and compares the occurrence of frequency of search phrases across two time periods separated by twelve hours. </p>
-<p>The script is shown here: </p>
-<ul>
-<li>
-<p> Register the tutorial JAR file so that the user-defined functions (UDFs) can be called in the script. </p>
-</li>
-</ul>
-<pre class="code">
-REGISTER ./tutorial.jar;
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to load the excite log file (excite.log or excite-small.log) into the &ldquo;raw&rdquo; bag as an array of records with the fields <strong>user</strong>, <strong>time</strong>, and <strong>query</strong>. </p>
-</li>
-</ul>
-<pre class="code">
-raw = LOAD 'excite.log' USING PigStorage('\t') AS (user, time, query);
-</pre>
-<ul>
-<li>
-<p> Call the NonURLDetector UDF to remove records if the query field is empty or a URL. </p>
-</li>
-</ul>
-<pre class="code">
-clean1 = FILTER raw BY org.apache.pig.tutorial.NonURLDetector(query);
-</pre>
-<ul>
-<li>
-<p> Call the ToLower UDF to change the query field to lowercase. </p>
-</li>
-</ul>
-<pre class="code">
-clean2 = FOREACH clean1 GENERATE user, time, org.apache.pig.tutorial.ToLower(query) as query;
-</pre>
-<ul>
-<li>
-<p> Because the log file only contains queries for a single day, we are only interested in the hour. The excite query log timestamp format is YYMMDDHHMMSS. Call the ExtractHour UDF to extract the hour from the time field. </p>
-</li>
-</ul>
-<pre class="code">
-houred = FOREACH clean2 GENERATE user, org.apache.pig.tutorial.ExtractHour(time) as hour, query;
-</pre>
-<ul>
-<li>
-<p> Call the NGramGenerator UDF to compose the n-grams of the query. </p>
-</li>
-</ul>
-<pre class="code">
-ngramed1 = FOREACH houred GENERATE user, hour, flatten(org.apache.pig.tutorial.NGramGenerator(query)) as ngram;
-</pre>
-<ul>
-<li>
-<p> Use the DISTINCT operator to get the unique n-grams for all records.  </p>
-</li>
-</ul>
-<pre class="code">
-ngramed2 = DISTINCT ngramed1;
-</pre>
-<ul>
-<li>
-<p> Use the GROUP operator to group the records by n-gram and hour.  </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency1 = GROUP ngramed2 BY (ngram, hour);
-</pre>
-<ul>
-<li>
-<p> Use the COUNT function to get the count (occurrences) of each n-gram.  </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency2 = FOREACH hour_frequency1 GENERATE flatten($0), COUNT($1) as count;
-</pre>
-<ul>
-<li>
-<p> Use the FOREACH-GENERATE operator to assign names to the fields. </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency3 = FOREACH hour_frequency2 GENERATE $0 as ngram, $1 as hour, $2 as count;
-</pre>
-<ul>
-<li>
-<p> Use the  FILTERoperator to get the n-grams for hour &lsquo;00&rsquo;  </p>
-</li>
-</ul>
-<pre class="code">
-hour00 = FILTER hour_frequency2 BY hour eq '00';
-</pre>
-<ul>
-<li>
-<p> Uses the FILTER operators to get the n-grams for hour &lsquo;12&rsquo; </p>
-</li>
-</ul>
-<pre class="code">
-hour12 = FILTER hour_frequency3 BY hour eq '12';
-</pre>
-<ul>
-<li>
-<p> Use the JOIN operator to get the n-grams that appear in both hours. </p>
-</li>
-</ul>
-<pre class="code">
-same = JOIN hour00 BY $0, hour12 BY $0;
-</pre>
-<ul>
-<li>
-<p> Use the FOREACH-GENERATE operator to record their frequency. </p>
-</li>
-</ul>
-<pre class="code">
-same1 = FOREACH same GENERATE hour_frequency2::hour00::group::ngram as ngram, $2 as count00, $5 as count12;
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to store the results. The output file contains a list of n-grams with the following fields: <strong>hour</strong>, <strong>count00</strong>, <strong>count12</strong>. </p>
-</li>
-</ul>
-<pre class="code">
-STORE same1 INTO '/tmp/tutorial-join-results' USING PigStorage();
-</pre>
-</div>
-</div>
-<!--+
-    |end content
-    +-->
-<div class="clearboth">&nbsp;</div>
-</div>
-<div id="footer">
-<!--+
-    |start bottomstrip
-    +-->
-<div class="lastmodified">
-<script type="text/javascript"><!--
-document.write("Last Published: " + document.lastModified);
-//  --></script>
-</div>
-<div class="copyright">
-        Copyright &copy;
-         2007-2010 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
-</div>
-<!--+
-    |end bottomstrip
-    +-->
-</div>
-</body>
-</html>