RubyGems - wukong - Versions diffs - 3.0.0.pre → 3.0.0.pre2 - Mend

wukong 3.0.0.pre → 3.0.0.pre2

Files changed (476) hide show

data/.gitignore +46 -33
data/.gitmodules +3 -0
data/.rspec +1 -1
data/.travis.yml +8 -1
data/.yardopts +0 -13
data/Guardfile +4 -6
data/{LICENSE.textile → LICENSE.md} +43 -55
data/README-old.md +422 -0
data/README.md +279 -418
data/Rakefile +21 -5
data/TODO.md +6 -6
data/bin/wu-clean-encoding +31 -0
data/bin/wu-lign +2 -2
data/bin/wu-local +69 -0
data/bin/wu-server +70 -0
data/examples/Gemfile +38 -0
data/examples/README.md +9 -0
data/examples/dataflow/apache_log_line.rb +64 -25
data/examples/dataflow/fibonacci_series.rb +101 -0
data/examples/dataflow/parse_apache_logs.rb +37 -7
data/examples/{dataflow.rb → dataflow/scraper_macro_flow.rb} +0 -0
data/examples/dataflow/simple.rb +4 -4
data/examples/geo.rb +4 -0
data/examples/geo/geo_grids.numbers +0 -0
data/examples/geo/geolocated.rb +331 -0
data/examples/geo/quadtile.rb +69 -0
data/examples/geo/spec/geolocated_spec.rb +247 -0
data/examples/geo/tile_fetcher.rb +77 -0
data/examples/graph/minimum_spanning_tree.rb +61 -61
data/examples/jabberwocky.txt +36 -0
data/examples/models/wikipedia.rb +20 -0
data/examples/munging/Gemfile +8 -0
data/examples/munging/airline_flights/airline.rb +57 -0
data/examples/munging/airline_flights/airline_flights.rake +83 -0
data/{lib/wukong/settings.rb → examples/munging/airline_flights/airplane.rb} +0 -0
data/examples/munging/airline_flights/airport.rb +211 -0
data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
data/examples/munging/airline_flights/flight.rb +156 -0
data/examples/munging/airline_flights/models.rb +4 -0
data/examples/munging/airline_flights/parse.rb +26 -0
data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
data/examples/munging/airline_flights/route.rb +35 -0
data/examples/munging/airline_flights/tasks.rake +83 -0
data/examples/munging/airline_flights/timezone_fixup.rb +62 -0
data/examples/munging/airline_flights/topcities.rb +167 -0
data/examples/munging/airports/40_wbans.txt +40 -0
data/examples/munging/airports/filter_weather_reports.rb +37 -0
data/examples/munging/airports/join.pig +31 -0
data/examples/munging/airports/to_tsv.rb +33 -0
data/examples/munging/airports/usa_wbans.pig +19 -0
data/examples/munging/airports/usa_wbans.txt +2157 -0
data/examples/munging/airports/wbans.pig +19 -0
data/examples/munging/airports/wbans.txt +2310 -0
data/examples/munging/geo/geo_json.rb +54 -0
data/examples/munging/geo/geo_models.rb +69 -0
data/examples/munging/geo/geonames_models.rb +78 -0
data/examples/munging/geo/iso_codes.rb +172 -0
data/examples/munging/geo/reconcile_countries.rb +124 -0
data/examples/munging/geo/tasks.rake +71 -0
data/examples/munging/rake_helper.rb +62 -0
data/examples/munging/weather/.gitignore +1 -0
data/examples/munging/weather/Gemfile +4 -0
data/examples/munging/weather/Rakefile +28 -0
data/examples/munging/weather/extract_ish.rb +13 -0
data/examples/munging/weather/models/weather.rb +119 -0
data/examples/munging/weather/utils/noaa_downloader.rb +46 -0
data/examples/munging/wikipedia/README.md +34 -0
data/examples/munging/wikipedia/Rakefile +193 -0
data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +4 -0
data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
data/examples/munging/wikipedia/dbpedia/extract_links.rb +193 -0
data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
data/examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig +18 -0
data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb +21 -0
data/examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old +27 -0
data/examples/munging/wikipedia/pagelinks/augment_pagelinks.pig +29 -0
data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb +14 -0
data/examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old +25 -0
data/examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig +29 -0
data/examples/munging/wikipedia/pageviews/augment_pageviews.pig +32 -0
data/examples/munging/wikipedia/pageviews/extract_pageviews.rb +85 -0
data/examples/munging/wikipedia/pig_style_guide.md +25 -0
data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig +19 -0
data/examples/munging/wikipedia/subuniverse/sub_articles.pig +23 -0
data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig +24 -0
data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig +22 -0
data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig +22 -0
data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig +26 -0
data/examples/munging/wikipedia/subuniverse/sub_pageviews.pig +29 -0
data/examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig +24 -0
data/examples/munging/wikipedia/utils/get_namespaces.rb +86 -0
data/examples/munging/wikipedia/utils/munging_utils.rb +68 -0
data/examples/munging/wikipedia/utils/namespaces.json +1 -0
data/examples/rake_helper.rb +85 -0
data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
data/examples/server_logs/logline.rb +95 -0
data/examples/server_logs/models.rb +66 -0
data/examples/server_logs/page_counts.pig +48 -0
data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
data/{old/examples/server_logs/breadcrumbs.rb → examples/server_logs/server_logs-03-breadcrumbs-full.rb} +26 -30
data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
data/examples/string_reverser.rb +26 -0
data/examples/text/pig_latin.rb +2 -2
data/examples/text/regional_flavor/README.md +14 -0
data/examples/text/regional_flavor/article_wordbags.pig +39 -0
data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
data/examples/word_count/accumulator.rb +26 -0
data/examples/word_count/tokenizer.rb +13 -0
data/examples/word_count/word_count.rb +6 -0
data/examples/workflow/cherry_pie.dot +97 -0
data/examples/workflow/cherry_pie.png +0 -0
data/examples/workflow/cherry_pie.rb +61 -26
data/lib/hanuman.rb +34 -7
data/lib/hanuman/graph.rb +55 -31
data/lib/hanuman/graphvizzer.rb +199 -178
data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
data/lib/hanuman/link.rb +35 -0
data/lib/hanuman/registry.rb +46 -0
data/lib/hanuman/stage.rb +76 -32
data/lib/wukong.rb +23 -24
data/lib/wukong/boot.rb +87 -0
data/lib/wukong/configuration.rb +8 -0
data/lib/wukong/dataflow.rb +45 -78
data/lib/wukong/driver.rb +99 -0
data/lib/wukong/emitter.rb +22 -0
data/lib/wukong/model/faker.rb +24 -24
data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
data/lib/wukong/processor.rb +60 -114
data/lib/wukong/spec_helpers.rb +81 -0
data/lib/wukong/spec_helpers/integration_driver.rb +144 -0
data/lib/wukong/spec_helpers/integration_driver_matchers.rb +219 -0
data/lib/wukong/spec_helpers/processor_helpers.rb +95 -0
data/lib/wukong/spec_helpers/processor_methods.rb +108 -0
data/lib/wukong/spec_helpers/shared_examples.rb +15 -0
data/lib/wukong/spec_helpers/spec_driver.rb +28 -0
data/lib/wukong/spec_helpers/spec_driver_matchers.rb +195 -0
data/lib/wukong/version.rb +2 -1
data/lib/wukong/widget/filters.rb +311 -0
data/lib/wukong/widget/processors.rb +156 -0
data/lib/wukong/widget/reducers.rb +7 -0
data/lib/wukong/widget/reducers/accumulator.rb +73 -0
data/lib/wukong/widget/reducers/bin.rb +318 -0
data/lib/wukong/widget/reducers/count.rb +61 -0
data/lib/wukong/widget/reducers/group.rb +85 -0
data/lib/wukong/widget/reducers/group_concat.rb +70 -0
data/lib/wukong/widget/reducers/moments.rb +72 -0
data/lib/wukong/widget/reducers/sort.rb +130 -0
data/lib/wukong/widget/serializers.rb +287 -0
data/lib/wukong/widget/sink.rb +10 -52
data/lib/wukong/widget/source.rb +7 -113
data/lib/wukong/widget/utils.rb +46 -0
data/lib/wukong/widgets.rb +6 -0
data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
data/spec/examples/dataflow/parsing_spec.rb +12 -11
data/spec/examples/dataflow/simple_spec.rb +32 -6
data/spec/examples/dataflow/telegram_spec.rb +36 -36
data/spec/examples/graph/minimum_spanning_tree_spec.rb +30 -31
data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
data/spec/examples/munging/airline_flights_spec.rb +202 -0
data/spec/examples/text/pig_latin_spec.rb +13 -16
data/spec/examples/workflow/cherry_pie_spec.rb +34 -4
data/spec/hanuman/graph_spec.rb +27 -2
data/spec/hanuman/hanuman_spec.rb +10 -0
data/spec/hanuman/registry_spec.rb +123 -0
data/spec/hanuman/stage_spec.rb +61 -7
data/spec/spec_helper.rb +29 -19
data/spec/support/hanuman_test_helpers.rb +14 -12
data/spec/support/shared_context_for_reducers.rb +37 -0
data/spec/support/shared_examples_for_builders.rb +101 -0
data/spec/support/shared_examples_for_shortcuts.rb +57 -0
data/spec/support/wukong_test_helpers.rb +37 -11
data/spec/wukong/dataflow_spec.rb +77 -55
data/spec/wukong/local_runner_spec.rb +24 -24
data/spec/wukong/model/faker_spec.rb +132 -131
data/spec/wukong/runner_spec.rb +8 -8
data/spec/wukong/widget/filters_spec.rb +61 -0
data/spec/wukong/widget/processors_spec.rb +126 -0
data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
data/spec/wukong/widget/reducers/count_spec.rb +11 -0
data/spec/wukong/widget/reducers/group_spec.rb +20 -0
data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
data/spec/wukong/widget/serializers_spec.rb +92 -0
data/spec/wukong/widget/sink_spec.rb +15 -15
data/spec/wukong/widget/source_spec.rb +65 -41
data/spec/wukong/wukong_spec.rb +10 -0
data/wukong.gemspec +17 -10
metadata +359 -335
data/.document +0 -5
data/VERSION +0 -1
data/bin/hdp-bin +0 -44
data/bin/hdp-bzip +0 -23
data/bin/hdp-cat +0 -3
data/bin/hdp-catd +0 -3
data/bin/hdp-cp +0 -3
data/bin/hdp-du +0 -86
data/bin/hdp-get +0 -3
data/bin/hdp-kill +0 -3
data/bin/hdp-kill-task +0 -3
data/bin/hdp-ls +0 -11
data/bin/hdp-mkdir +0 -2
data/bin/hdp-mkdirp +0 -12
data/bin/hdp-mv +0 -3
data/bin/hdp-parts_to_keys.rb +0 -77
data/bin/hdp-ps +0 -3
data/bin/hdp-put +0 -3
data/bin/hdp-rm +0 -32
data/bin/hdp-sort +0 -40
data/bin/hdp-stream +0 -40
data/bin/hdp-stream-flat +0 -22
data/bin/hdp-stream2 +0 -39
data/bin/hdp-sync +0 -17
data/bin/hdp-wc +0 -67
data/bin/wu-flow +0 -10
data/bin/wu-map +0 -17
data/bin/wu-red +0 -17
data/bin/wukong +0 -17
data/data/CREDITS.md +0 -355
data/data/graph/airfares.tsv +0 -2174
data/data/text/gift_of_the_magi.txt +0 -225
data/data/text/jabberwocky.txt +0 -36
data/data/text/rectification_of_names.txt +0 -33
data/data/twitter/a_atsigns_b.tsv +0 -64
data/data/twitter/a_follows_b.tsv +0 -53
data/data/twitter/tweet.tsv +0 -167
data/data/twitter/twitter_user.tsv +0 -55
data/data/wikipedia/dbpedia-sentences.tsv +0 -1000
data/docpages/INSTALL.textile +0 -92
data/docpages/LICENSE.textile +0 -107
data/docpages/README-elastic_map_reduce.textile +0 -377
data/docpages/README-performance.textile +0 -90
data/docpages/README-wulign.textile +0 -65
data/docpages/UsingWukong-part1-get_ready.textile +0 -17
data/docpages/UsingWukong-part2-ThinkingBigData.textile +0 -75
data/docpages/UsingWukong-part3-parsing.textile +0 -138
data/docpages/_config.yml +0 -39
data/docpages/avro/avro_notes.textile +0 -56
data/docpages/avro/performance.textile +0 -36
data/docpages/avro/tethering.textile +0 -19
data/docpages/bigdata-tips.textile +0 -143
data/docpages/code/api_response_example.txt +0 -20
data/docpages/code/parser_skeleton.rb +0 -38
data/docpages/diagrams/MapReduceDiagram.graffle +0 -0
data/docpages/favicon.ico +0 -0
data/docpages/gem.css +0 -16
data/docpages/hadoop-tips.textile +0 -83
data/docpages/index.textile +0 -92
data/docpages/intro.textile +0 -8
data/docpages/moreinfo.textile +0 -174
data/docpages/news.html +0 -24
data/docpages/pig/PigLatinExpressionsList.txt +0 -122
data/docpages/pig/PigLatinReferenceManual.txt +0 -1640
data/docpages/pig/commandline_params.txt +0 -26
data/docpages/pig/cookbook.html +0 -481
data/docpages/pig/images/hadoop-logo.jpg +0 -0
data/docpages/pig/images/instruction_arrow.png +0 -0
data/docpages/pig/images/pig-logo.gif +0 -0
data/docpages/pig/piglatin_ref1.html +0 -1103
data/docpages/pig/piglatin_ref2.html +0 -14340
data/docpages/pig/setup.html +0 -505
data/docpages/pig/skin/basic.css +0 -166
data/docpages/pig/skin/breadcrumbs.js +0 -237
data/docpages/pig/skin/fontsize.js +0 -166
data/docpages/pig/skin/getBlank.js +0 -40
data/docpages/pig/skin/getMenu.js +0 -45
data/docpages/pig/skin/images/chapter.gif +0 -0
data/docpages/pig/skin/images/chapter_open.gif +0 -0
data/docpages/pig/skin/images/current.gif +0 -0
data/docpages/pig/skin/images/external-link.gif +0 -0
data/docpages/pig/skin/images/header_white_line.gif +0 -0
data/docpages/pig/skin/images/page.gif +0 -0
data/docpages/pig/skin/images/pdfdoc.gif +0 -0
data/docpages/pig/skin/images/rc-b-l-15-1body-2menu-3menu.png +0 -0
data/docpages/pig/skin/images/rc-b-r-15-1body-2menu-3menu.png +0 -0
data/docpages/pig/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png +0 -0
data/docpages/pig/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png +0 -0
data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png +0 -0
data/docpages/pig/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png +0 -0
data/docpages/pig/skin/images/rc-t-r-15-1body-2menu-3menu.png +0 -0
data/docpages/pig/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png +0 -0
data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png +0 -0
data/docpages/pig/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png +0 -0
data/docpages/pig/skin/print.css +0 -54
data/docpages/pig/skin/profile.css +0 -181
data/docpages/pig/skin/screen.css +0 -587
data/docpages/pig/tutorial.html +0 -1059
data/docpages/pig/udf.html +0 -1509
data/docpages/tutorial.textile +0 -283
data/docpages/usage.textile +0 -195
data/docpages/wutils.textile +0 -263
data/examples/dataflow/complex.rb +0 -11
data/examples/dataflow/donuts.rb +0 -13
data/examples/tiny_count/jabberwocky_output.tsv +0 -92
data/examples/word_count.rb +0 -48
data/examples/workflow/fiddle.rb +0 -24
data/lib/away/escapement.rb +0 -129
data/lib/away/exe.rb +0 -11
data/lib/away/experimental.rb +0 -5
data/lib/away/from_file.rb +0 -52
data/lib/away/job.rb +0 -56
data/lib/away/job/rake_compat.rb +0 -17
data/lib/away/registry.rb +0 -79
data/lib/away/runner.rb +0 -276
data/lib/away/runner/execute.rb +0 -121
data/lib/away/script.rb +0 -161
data/lib/away/script/hadoop_command.rb +0 -240
data/lib/away/source/file_list_source.rb +0 -15
data/lib/away/source/looper.rb +0 -18
data/lib/away/task.rb +0 -219
data/lib/hanuman/action.rb +0 -21
data/lib/hanuman/chain.rb +0 -4
data/lib/hanuman/graphviz.rb +0 -74
data/lib/hanuman/resource.rb +0 -6
data/lib/hanuman/slot.rb +0 -87
data/lib/hanuman/slottable.rb +0 -220
data/lib/wukong/bad_record.rb +0 -15
data/lib/wukong/event.rb +0 -44
data/lib/wukong/local_runner.rb +0 -55
data/lib/wukong/mapred.rb +0 -3
data/lib/wukong/universe.rb +0 -48
data/lib/wukong/widget/filter.rb +0 -81
data/lib/wukong/widget/gibberish.rb +0 -123
data/lib/wukong/widget/monitor.rb +0 -26
data/lib/wukong/widget/reducer.rb +0 -66
data/lib/wukong/widget/stringifier.rb +0 -50
data/lib/wukong/workflow.rb +0 -22
data/lib/wukong/workflow/command.rb +0 -42
data/old/config/emr-example.yaml +0 -48
data/old/examples/README.txt +0 -17
data/old/examples/contrib/jeans/README.markdown +0 -165
data/old/examples/contrib/jeans/data/normalized_sizes +0 -3
data/old/examples/contrib/jeans/data/orders.tsv +0 -1302
data/old/examples/contrib/jeans/data/sizes +0 -3
data/old/examples/contrib/jeans/normalize.rb +0 -20
data/old/examples/contrib/jeans/sizes.rb +0 -55
data/old/examples/corpus/bnc_word_freq.rb +0 -44
data/old/examples/corpus/bucket_counter.rb +0 -47
data/old/examples/corpus/dbpedia_abstract_to_sentences.rb +0 -86
data/old/examples/corpus/sentence_bigrams.rb +0 -53
data/old/examples/corpus/sentence_coocurrence.rb +0 -66
data/old/examples/corpus/stopwords.rb +0 -138
data/old/examples/corpus/words_to_bigrams.rb +0 -53
data/old/examples/emr/README.textile +0 -110
data/old/examples/emr/dot_wukong_dir/credentials.json +0 -7
data/old/examples/emr/dot_wukong_dir/emr.yaml +0 -69
data/old/examples/emr/dot_wukong_dir/emr_bootstrap.sh +0 -33
data/old/examples/emr/elastic_mapreduce_example.rb +0 -28
data/old/examples/network_graph/adjacency_list.rb +0 -74
data/old/examples/network_graph/breadth_first_search.rb +0 -72
data/old/examples/network_graph/gen_2paths.rb +0 -68
data/old/examples/network_graph/gen_multi_edge.rb +0 -112
data/old/examples/network_graph/gen_symmetric_links.rb +0 -64
data/old/examples/pagerank/README.textile +0 -6
data/old/examples/pagerank/gen_initial_pagerank_graph.pig +0 -57
data/old/examples/pagerank/pagerank.rb +0 -72
data/old/examples/pagerank/pagerank_initialize.rb +0 -42
data/old/examples/pagerank/run_pagerank.sh +0 -21
data/old/examples/sample_records.rb +0 -33
data/old/examples/server_logs/apache_log_parser.rb +0 -15
data/old/examples/server_logs/nook.rb +0 -48
data/old/examples/server_logs/nook/faraday_dummy_adapter.rb +0 -94
data/old/examples/server_logs/user_agent.rb +0 -40
data/old/examples/simple_word_count.rb +0 -82
data/old/examples/size.rb +0 -61
data/old/examples/stats/avg_value_frequency.rb +0 -86
data/old/examples/stats/binning_percentile_estimator.rb +0 -140
data/old/examples/stats/data/avg_value_frequency.tsv +0 -3
data/old/examples/stats/rank_and_bin.rb +0 -173
data/old/examples/stupidly_simple_filter.rb +0 -40
data/old/examples/word_count.rb +0 -75
data/old/graph/graphviz_builder.rb +0 -580
data/old/graph_easy/Attributes.pm +0 -4181
data/old/graph_easy/Graphviz.pm +0 -2232
data/old/wukong.rb +0 -18
data/old/wukong/and_pig.rb +0 -38
data/old/wukong/bad_record.rb +0 -18
data/old/wukong/datatypes.rb +0 -24
data/old/wukong/datatypes/enum.rb +0 -127
data/old/wukong/datatypes/fake_types.rb +0 -17
data/old/wukong/decorator.rb +0 -28
data/old/wukong/encoding/asciize.rb +0 -108
data/old/wukong/extensions.rb +0 -16
data/old/wukong/extensions/array.rb +0 -18
data/old/wukong/extensions/blank.rb +0 -93
data/old/wukong/extensions/class.rb +0 -189
data/old/wukong/extensions/date_time.rb +0 -53
data/old/wukong/extensions/emittable.rb +0 -69
data/old/wukong/extensions/enumerable.rb +0 -79
data/old/wukong/extensions/hash.rb +0 -167
data/old/wukong/extensions/hash_keys.rb +0 -16
data/old/wukong/extensions/hash_like.rb +0 -150
data/old/wukong/extensions/hashlike_class.rb +0 -47
data/old/wukong/extensions/module.rb +0 -2
data/old/wukong/extensions/pathname.rb +0 -27
data/old/wukong/extensions/string.rb +0 -65
data/old/wukong/extensions/struct.rb +0 -17
data/old/wukong/extensions/symbol.rb +0 -11
data/old/wukong/filename_pattern.rb +0 -74
data/old/wukong/helper.rb +0 -7
data/old/wukong/helper/stopwords.rb +0 -195
data/old/wukong/helper/tokenize.rb +0 -35
data/old/wukong/logger.rb +0 -38
data/old/wukong/periodic_monitor.rb +0 -72
data/old/wukong/schema.rb +0 -269
data/old/wukong/script.rb +0 -286
data/old/wukong/script/avro_command.rb +0 -5
data/old/wukong/script/cassandra_loader_script.rb +0 -40
data/old/wukong/script/emr_command.rb +0 -168
data/old/wukong/script/hadoop_command.rb +0 -237
data/old/wukong/script/local_command.rb +0 -41
data/old/wukong/store.rb +0 -10
data/old/wukong/store/base.rb +0 -27
data/old/wukong/store/cassandra.rb +0 -10
data/old/wukong/store/cassandra/streaming.rb +0 -75
data/old/wukong/store/cassandra/struct_loader.rb +0 -21
data/old/wukong/store/cassandra_model.rb +0 -91
data/old/wukong/store/chh_chunked_flat_file_store.rb +0 -37
data/old/wukong/store/chunked_flat_file_store.rb +0 -48
data/old/wukong/store/conditional_store.rb +0 -57
data/old/wukong/store/factory.rb +0 -8
data/old/wukong/store/flat_file_store.rb +0 -89
data/old/wukong/store/key_store.rb +0 -51
data/old/wukong/store/null_store.rb +0 -15
data/old/wukong/store/read_thru_store.rb +0 -22
data/old/wukong/store/tokyo_tdb_key_store.rb +0 -33
data/old/wukong/store/tyrant_rdb_key_store.rb +0 -57
data/old/wukong/store/tyrant_tdb_key_store.rb +0 -20
data/old/wukong/streamer.rb +0 -30
data/old/wukong/streamer/accumulating_reducer.rb +0 -83
data/old/wukong/streamer/base.rb +0 -126
data/old/wukong/streamer/counting_reducer.rb +0 -25
data/old/wukong/streamer/filter.rb +0 -20
data/old/wukong/streamer/instance_streamer.rb +0 -15
data/old/wukong/streamer/json_streamer.rb +0 -21
data/old/wukong/streamer/line_streamer.rb +0 -12
data/old/wukong/streamer/list_reducer.rb +0 -31
data/old/wukong/streamer/rank_and_bin_reducer.rb +0 -145
data/old/wukong/streamer/record_streamer.rb +0 -14
data/old/wukong/streamer/reducer.rb +0 -11
data/old/wukong/streamer/set_reducer.rb +0 -14
data/old/wukong/streamer/struct_streamer.rb +0 -48
data/old/wukong/streamer/summing_reducer.rb +0 -29
data/old/wukong/streamer/uniq_by_last_reducer.rb +0 -51
data/old/wukong/typed_struct.rb +0 -12
data/spec/away/encoding_spec.rb +0 -32
data/spec/away/exe_spec.rb +0 -20
data/spec/away/flow_spec.rb +0 -82
data/spec/away/graph_spec.rb +0 -6
data/spec/away/job_spec.rb +0 -15
data/spec/away/rake_compat_spec.rb +0 -9
data/spec/away/script_spec.rb +0 -81
data/spec/hanuman/graphviz_spec.rb +0 -29
data/spec/hanuman/slot_spec.rb +0 -2
data/spec/support/examples_helper.rb +0 -10
data/spec/support/streamer_test_helpers.rb +0 -6
data/spec/support/wukong_widget_helpers.rb +0 -66
data/spec/wukong/processor_spec.rb +0 -109
data/spec/wukong/widget/filter_spec.rb +0 -99
data/spec/wukong/widget/stringifier_spec.rb +0 -51
data/spec/wukong/workflow/command_spec.rb +0 -5

@@ -1,1059 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
-<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta content="Apache Forrest" name="Generator">
-<meta name="Forrest-version" content="0.8">
-<meta name="Forrest-skin-name" content="pelt">
-<title>Pig Tutorial</title>
-<link type="text/css" href="skin/basic.css" rel="stylesheet">
-<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
-<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
-<link type="text/css" href="skin/profile.css" rel="stylesheet">
-<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
-<link rel="shortcut icon" href="">
-</head>
-<body onload="init()">
-<script type="text/javascript">ndeSetTextSize();</script>
-<div id="top">
-<!--+
-    |breadtrail
-    +-->
-<div class="breadtrail">
-<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/pig/">Pig</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
-</div>
-<!--+
-    |header
-    +-->
-<div class="header">
-<!--+
-    |start group logo
-    +-->
-<div class="grouplogo">
-<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
-</div>
-<!--+
-    |end group logo
-    +-->
-<!--+
-    |start Project Logo
-    +-->
-<div class="projectlogo">
-<a href="http://hadoop.apache.org/pig/"><img class="logoImage" alt="Pig" src="images/pig-logo.gif" title="A platform for analyzing large datasets."></a>
-</div>
-<!--+
-    |end Project Logo
-    +-->
-<!--+
-    |start Search
-    +-->
-<div class="searchbox">
-<form action="http://www.google.com/search" method="get" class="roundtopsmall">
-<input value="" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
-                    <input name="Search" value="Search" type="submit">
-</form>
-</div>
-<!--+
-    |end search
-    +-->
-<!--+
-    |start Tabs
-    +-->
-<ul id="tabs">
-<li>
-<a class="unselected" href="http://hadoop.apache.org/pig/">Project</a>
-</li>
-<li>
-<a class="unselected" href="http://wiki.apache.org/pig/">Wiki</a>
-</li>
-<li class="current">
-<a class="selected" href="index.html">Pig 0.7.0 Documentation</a>
-</li>
-</ul>
-<!--+
-    |end Tabs
-    +-->
-</div>
-</div>
-<div id="main">
-<div id="publishedStrip">
-<!--+
-    |start Subtabs
-    +-->
-<div id="level2tabs"></div>
-<!--+
-    |end Endtabs
-    +-->
-<script type="text/javascript"><!--
-document.write("Last Published: " + document.lastModified);
-//  --></script>
-</div>
-<!--+
-    |breadtrail
-    +-->
-<div class="breadtrail">
-             &nbsp;
-           </div>
-<!--+
-    |start Menu, mainarea
-    +-->
-<!--+
-    |start Menu
-    +-->
-<div id="menu">
-<div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Pig</div>
-<div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
-<div class="menuitem">
-<a href="index.html">Overview</a>
-</div>
-<div class="menuitem">
-<a href="setup.html">Setup</a>
-</div>
-<div class="menupage">
-<div class="menupagetitle">Tutorial</div>
-</div>
-<div class="menuitem">
-<a href="piglatin_ref1.html">Pig Latin 1</a>
-</div>
-<div class="menuitem">
-<a href="piglatin_ref2.html">Pig Latin 2</a>
-</div>
-<div class="menuitem">
-<a href="cookbook.html">Cookbook</a>
-</div>
-<div class="menuitem">
-<a href="udf.html">UDFs</a>
-</div>
-</div>
-<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Zebra</div>
-<div id="menu_1.2" class="menuitemgroup">
-<div class="menuitem">
-<a href="zebra_overview.html">Zebra Overview </a>
-</div>
-<div class="menuitem">
-<a href="zebra_users.html">Zebra Users </a>
-</div>
-<div class="menuitem">
-<a href="zebra_reference.html">Zebra Reference </a>
-</div>
-<div class="menuitem">
-<a href="zebra_mapreduce.html">Zebra MapReduce </a>
-</div>
-<div class="menuitem">
-<a href="zebra_pig.html">Zebra Pig </a>
-</div>
-<div class="menuitem">
-<a href="zebra_stream.html">Zebra Streaming </a>
-</div>
-</div>
-<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Miscellaneous</div>
-<div id="menu_1.3" class="menuitemgroup">
-<div class="menuitem">
-<a href="api/">API Docs</a>
-</div>
-<div class="menuitem">
-<a href="http://wiki.apache.org/pig/">Wiki</a>
-</div>
-<div class="menuitem">
-<a href="http://wiki.apache.org/pig/FAQ">FAQ</a>
-</div>
-<div class="menuitem">
-<a href="http://hadoop.apache.org/pig/releases.html">Release Notes</a>
-</div>
-</div>
-<div id="credit"></div>
-<div id="roundbottom">
-<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
-<!--+
-  |alternative credits
-  +-->
-<div id="credit2"></div>
-</div>
-<!--+
-    |end Menu
-    +-->
-<!--+
-    |start content
-    +-->
-<div id="content">
-<div title="Portable Document Format" class="pdflink">
-<a class="dida" href="tutorial.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
-        PDF</a>
-</div>
-<h1>Pig Tutorial</h1>
-<div id="minitoc-area">
-<ul class="minitoc">
-<li>
-<a href="#Overview">Overview</a>
-</li>
-<li>
-<a href="#Java+Installation"> Java Installation</a>
-</li>
-<li>
-<a href="#Pig+Installation"> Pig Installation</a>
-</li>
-<li>
-<a href="#Running+the+Pig+Scripts+in+Local+Mode"> Running the Pig Scripts in Local Mode</a>
-</li>
-<li>
-<a href="#Running+the+Pig+Scripts+in+Mapreduce+Mode"> Running the Pig Scripts in Mapreduce Mode</a>
-</li>
-<li>
-<a href="#Pig+Tutorial+File"> Pig Tutorial File</a>
-</li>
-<li>
-<a href="#Pig+Script+1%3A+Query+Phrase+Popularity"> Pig Script 1: Query Phrase Popularity</a>
-</li>
-<li>
-<a href="#Pig+Script+2%3A+Temporal+Query+Phrase+Popularity">Pig Script 2: Temporal Query Phrase Popularity</a>
-</li>
-</ul>
-</div>
-<a name="N1000D"></a><a name="Overview"></a>
-<h2 class="h3">Overview</h2>
-<div class="section">
-<p>The Pig tutorial shows you how to run two Pig scripts in local mode and mapreduce mode.   </p>
-<ul>
-<li>
-<p>
-<strong>Local Mode</strong>: To run the scripts in local mode, no Hadoop or HDFS installation is required. All files are installed and run from your local host and file system. </p>
-</li>
-<li>
-<p>
-<strong>Mapreduce Mode</strong>: To run the scripts in mapreduce mode, you need access to a Hadoop cluster and HDFS installation. </p>
-</li>
-</ul>
-<p>The Pig tutorial file (tutorial/pigtutorial.tar.gz file in the pig distribution) includes the Pig JAR file (pig.jar) and the tutorial files (tutorial.jar, Pigs scripts, log files).
-These files work with Hadoop 0.20.2 and include everything you need to run the Pig scripts.</p>
-<p>To get started, follow these basic steps:  </p>
-<ol>
-<li>
-<p>Install Java </p>
-</li>
-<li>
-<p>Install Pig </p>
-</li>
-<li>
-<p>Run the Pig scripts - in Local or Hadoop mode </p>
-</li>
-</ol>
-</div>
-<a name="N10042"></a><a name="Java+Installation"></a>
-<h2 class="h3"> Java Installation</h2>
-<div class="section">
-<p>Make sure your run-time environment includes the following: </p>
-<ul>
-<li>
-<p>Java 1.6 or higher (preferably from Sun) </p>
-</li>
-<li>
-<p>The JAVA_HOME environment variable is set the root of your Java installation.  </p>
-</li>
-</ul>
-</div>
-<a name="N10059"></a><a name="Pig+Installation"></a>
-<h2 class="h3"> Pig Installation</h2>
-<div class="section">
-<p>To install Pig, do the following: </p>
-<ol>
-<li>
-<p>Download the Pig tutorial file to your local directory. </p>
-</li>
-<li>
-<p>Unzip the Pig tutorial file (the files are stored in a newly created directory, pigtmp). </p>
-<pre class="code">
-$ tar -xzf pigtutorial.tar.gz
-</pre>
-</li>
-<li>
-<p>Move to the pigtmp directory.  </p>
-</li>
-<li>
-<p>Review the contents of the Pig tutorial file. </p>
-</li>
-<li>
-<p>Copy the <strong>pig.jar</strong> file to the appropriate directory on your system. For example: /home/me/pig.  </p>
-</li>
-<li>
-<p>Create an environment variable, <strong>PIGDIR</strong>, and point it to your directory; for example, export PIGDIR=/home/me/pig (bash, sh) or setenv PIGDIR /home/me/pig (tcsh, csh).  </p>
-</li>
-</ol>
-</div>
-<a name="N1008E"></a><a name="Running+the+Pig+Scripts+in+Local+Mode"></a>
-<h2 class="h3"> Running the Pig Scripts in Local Mode</h2>
-<div class="section">
-<p>To run the Pig scripts in local mode, do the following: </p>
-<ol>
-<li>
-<p>Set the maximum memory for Java.</p>
-<pre class="code">
-java -Xmx256m -cp pig.jar org.apache.pig.Main -x local script1-local.pig
-java -Xmx256m -cp pig.jar org.apache.pig.Main -x local script2-local.pig
-</pre>
-</li>
-<li>
-<p>Move to the pigtmp directory. </p>
-</li>
-<li>
-<p>Review Pig Script 1 and Pig Script 2. </p>
-</li>
-<li>
-<p>Execute the following command (using either script1-local.pig or script2-local.pig). </p>
-<pre class="code">
-$ java -cp $PIGDIR/pig.jar org.apache.pig.Main -x local script1-local.pig
-</pre>
-</li>
-<li>
-<p>Review the result files, located in the part-r-00000 directory.</p>
-<p>The output may contain a few Hadoop warnings which can be ignored:</p>
-<pre class="code">
-2010-04-08 12:55:33,642 [main] INFO  org.apache.hadoop.metrics.jvm.JvmMetrics
-- Cannot initialize JVM Metrics with processName=JobTracker, sessionId= - already initialized
-</pre>
-</li>
-</ol>
-</div>
-<a name="N100C3"></a><a name="Running+the+Pig+Scripts+in+Mapreduce+Mode"></a>
-<h2 class="h3"> Running the Pig Scripts in Mapreduce Mode</h2>
-<div class="section">
-<p>To run the Pig scripts in mapreduce mode, do the following: </p>
-<ol>
-<li>
-<p>Move to the pigtmp directory. </p>
-</li>
-<li>
-<p>Review Pig Script 1 and Pig Script 2. </p>
-</li>
-<li>
-<p>Copy the excite.log.bz2 file from the pigtmp directory to the HDFS directory. </p>
-<pre class="code">
-$ hadoop fs &ndash;copyFromLocal excite.log.bz2 .
-</pre>
-</li>
-<li>
-<p>Set the HADOOP_CONF_DIR environment variable to the location of your core-site.xml, hdfs-site.xml and mapred-site.xml files. </p>
-</li>
-<li>
-<p>Execute the following command (using either script1-hadoop.pig or script2-hadoop.pig): </p>
-<pre class="code">
-$ java -cp $PIGDIR/pig.jar:$HADOOP_CONF_DIR  org.apache.pig.Main script1-hadoop.pig
-</pre>
-</li>
-<li>
-<p>Review the result files, located in the script1-hadoop-results or script2-hadoop-results HDFS directory: </p>
-<pre class="code">
-$ hadoop fs -ls script1-hadoop-results
-$ hadoop fs -cat 'script1-hadoop-results/*' | less
-</pre>
-</li>
-</ol>
-</div>
-<a name="N100FA"></a><a name="Pig+Tutorial+File"></a>
-<h2 class="h3"> Pig Tutorial File</h2>
-<div class="section">
-<p>The contents of the Pig tutorial file (pigtutorial.tar.gz) are described here. </p>
-<table class="ForrestTable" cellspacing="1" cellpadding="4">
-<tr>
-<td colspan="1" rowspan="1">
-<p>
-<strong>File</strong>
-</p>
-</td>
-<td colspan="1" rowspan="1">
-<p>
-<strong>Description</strong>
-</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> pig.jar </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig JAR file </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> tutorial.jar </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> User-defined functions (UDFs) and Java classes </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script1-local.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 1, Query Phrase Popularity (local mode) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script1-hadoop.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 1, Query Phrase Popularity (Hadoop cluster) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script2-local.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 2, Temporal Query Phrase Popularity (local mode)</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> script2-hadoop.pig </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Pig Script 2, Temporal Query Phrase Popularity (Hadoop cluster) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> excite-small.log </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Log file, Excite search engine (local mode) </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> excite.log.bz2 </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Log file, Excite search engine (Hadoop cluster) </p>
-</td>
-</tr>
-</table>
-<p>The user-defined functions (UDFs) are described here. </p>
-<table class="ForrestTable" cellspacing="1" cellpadding="4">
-<tr>
-<td colspan="1" rowspan="1">
-<p>
-<strong>UDF</strong>
-</p>
-</td>
-<td colspan="1" rowspan="1">
-<p>
-<strong>Description</strong>
-</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> ExtractHour </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Extracts the hour from the record.</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> NGramGenerator </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Composes n-grams from the set of words. </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> NonURLDetector </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Removes the record if the query field is empty or a URL. </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> ScoreGenerator </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Calculates a "popularity" score for the n-gram.</p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> ToLower </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Changes the query field to lowercase. </p>
-</td>
-</tr>
-<tr>
-<td colspan="1" rowspan="1">
-<p> TutorialUtil </p>
-</td>
-<td colspan="1" rowspan="1">
-<p> Divides the query string into a set of words.</p>
-</td>
-</tr>
-</table>
-</div>
-<a name="N10247"></a><a name="Pig+Script+1%3A+Query+Phrase+Popularity"></a>
-<h2 class="h3"> Pig Script 1: Query Phrase Popularity</h2>
-<div class="section">
-<p>The Query Phrase Popularity script (script1-local.pig or script1-hadoop.pig) processes a search query log file from the Excite search engine and finds search phrases that occur with particular high frequency during certain times of the day. </p>
-<p>The script is shown here: </p>
-<ul>
-<li>
-<p> Register the tutorial JAR file so that the included UDFs can be called in the script. </p>
-</li>
-</ul>
-<pre class="code">
-REGISTER ./tutorial.jar;
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to load the excite log file (excite.log or excite-small.log) into the &ldquo;raw&rdquo; bag as an array of records with the fields <strong>user</strong>, <strong>time</strong>, and <strong>query</strong>.  </p>
-</li>
-</ul>
-<pre class="code">
-raw = LOAD 'excite.log' USING PigStorage('\t') AS (user, time, query);
-</pre>
-<ul>
-<li>
-<p> Call the NonURLDetector UDF to remove records if the query field is empty or a URL.  </p>
-</li>
-</ul>
-<pre class="code">
-clean1 = FILTER raw BY org.apache.pig.tutorial.NonURLDetector(query);
-</pre>
-<ul>
-<li>
-<p> Call the ToLower UDF to change the query field to lowercase.  </p>
-</li>
-</ul>
-<pre class="code">
-clean2 = FOREACH clean1 GENERATE user, time, org.apache.pig.tutorial.ToLower(query) as query;
-</pre>
-<ul>
-<li>
-<p> Because the log file only contains queries for a single day, we are only interested in the hour. The excite query log timestamp format is YYMMDDHHMMSS. Call the ExtractHour UDF to extract the hour (HH) from the time field. </p>
-</li>
-</ul>
-<pre class="code">
-houred = FOREACH clean2 GENERATE user, org.apache.pig.tutorial.ExtractHour(time) as hour, query;
-</pre>
-<ul>
-<li>
-<p> Call the NGramGenerator UDF to compose the n-grams of the query. </p>
-</li>
-</ul>
-<pre class="code">
-ngramed1 = FOREACH houred GENERATE user, hour, flatten(org.apache.pig.tutorial.NGramGenerator(query)) as ngram;
-</pre>
-<ul>
-<li>
-<p> Use the DISTINCT operator to get the unique n-grams for all records.  </p>
-</li>
-</ul>
-<pre class="code">
-ngramed2 = DISTINCT ngramed1;
-</pre>
-<ul>
-<li>
-<p> Use the GROUP operator to group records by n-gram and hour. </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency1 = GROUP ngramed2 BY (ngram, hour);
-</pre>
-<ul>
-<li>
-<p> Use the COUNTfunction to get the count (occurrences) of each n-gram.  </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency2 = FOREACH hour_frequency1 GENERATE flatten($0), COUNT($1) as count;
-</pre>
-<ul>
-<li>
-<p> Use the GROUP operator to group records by n-gram only. Each group now corresponds to a distinct n-gram and has the count for each hour. </p>
-</li>
-</ul>
-<pre class="code">
-uniq_frequency1 = GROUP hour_frequency2 BY group::ngram;
-</pre>
-<ul>
-<li>
-<p> For each group, identify the hour in which this n-gram is used with a particularly high frequency. Call the ScoreGenerator UDF to calculate a "popularity" score for the n-gram. </p>
-</li>
-</ul>
-<pre class="code">
-uniq_frequency2 = FOREACH uniq_frequency1 GENERATE flatten($0), flatten(org.apache.pig.tutorial.ScoreGenerator($1));
-</pre>
-<ul>
-<li>
-<p> Use the FOREACH-GENERATE operator to assign names to the fields.  </p>
-</li>
-</ul>
-<pre class="code">
-uniq_frequency3 = FOREACH uniq_frequency2 GENERATE $1 as hour, $0 as ngram, $2 as score, $3 as count, $4 as mean;
-</pre>
-<ul>
-<li>
-<p> Use the FILTER operator to move all records with a score less than or equal to 2.0. </p>
-</li>
-</ul>
-<pre class="code">
-filtered_uniq_frequency = FILTER uniq_frequency3 BY score &gt; 2.0;
-</pre>
-<ul>
-<li>
-<p> Use the ORDER operator to sort the remaining records by hour and score.  </p>
-</li>
-</ul>
-<pre class="code">
-ordered_uniq_frequency = ORDER filtered_uniq_frequency BY (hour, score);
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to store the results. The output file contains a list of n-grams with the following fields: <strong>hour</strong>, <strong>ngram</strong>, <strong>score</strong>, <strong>count</strong>, <strong>mean</strong>. </p>
-</li>
-</ul>
-<pre class="code">
-STORE ordered_uniq_frequency INTO '/tmp/tutorial-results' USING PigStorage();
-</pre>
-</div>
-<a name="N10320"></a><a name="Pig+Script+2%3A+Temporal+Query+Phrase+Popularity"></a>
-<h2 class="h3">Pig Script 2: Temporal Query Phrase Popularity</h2>
-<div class="section">
-<p>The Temporal Query Phrase Popularity script (script2-local.pig or script2-hadoop.pig) processes a search query log file from the Excite search engine and compares the occurrence of frequency of search phrases across two time periods separated by twelve hours. </p>
-<p>The script is shown here: </p>
-<ul>
-<li>
-<p> Register the tutorial JAR file so that the user-defined functions (UDFs) can be called in the script. </p>
-</li>
-</ul>
-<pre class="code">
-REGISTER ./tutorial.jar;
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to load the excite log file (excite.log or excite-small.log) into the &ldquo;raw&rdquo; bag as an array of records with the fields <strong>user</strong>, <strong>time</strong>, and <strong>query</strong>. </p>
-</li>
-</ul>
-<pre class="code">
-raw = LOAD 'excite.log' USING PigStorage('\t') AS (user, time, query);
-</pre>
-<ul>
-<li>
-<p> Call the NonURLDetector UDF to remove records if the query field is empty or a URL. </p>
-</li>
-</ul>
-<pre class="code">
-clean1 = FILTER raw BY org.apache.pig.tutorial.NonURLDetector(query);
-</pre>
-<ul>
-<li>
-<p> Call the ToLower UDF to change the query field to lowercase. </p>
-</li>
-</ul>
-<pre class="code">
-clean2 = FOREACH clean1 GENERATE user, time, org.apache.pig.tutorial.ToLower(query) as query;
-</pre>
-<ul>
-<li>
-<p> Because the log file only contains queries for a single day, we are only interested in the hour. The excite query log timestamp format is YYMMDDHHMMSS. Call the ExtractHour UDF to extract the hour from the time field. </p>
-</li>
-</ul>
-<pre class="code">
-houred = FOREACH clean2 GENERATE user, org.apache.pig.tutorial.ExtractHour(time) as hour, query;
-</pre>
-<ul>
-<li>
-<p> Call the NGramGenerator UDF to compose the n-grams of the query. </p>
-</li>
-</ul>
-<pre class="code">
-ngramed1 = FOREACH houred GENERATE user, hour, flatten(org.apache.pig.tutorial.NGramGenerator(query)) as ngram;
-</pre>
-<ul>
-<li>
-<p> Use the DISTINCT operator to get the unique n-grams for all records.  </p>
-</li>
-</ul>
-<pre class="code">
-ngramed2 = DISTINCT ngramed1;
-</pre>
-<ul>
-<li>
-<p> Use the GROUP operator to group the records by n-gram and hour.  </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency1 = GROUP ngramed2 BY (ngram, hour);
-</pre>
-<ul>
-<li>
-<p> Use the COUNT function to get the count (occurrences) of each n-gram.  </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency2 = FOREACH hour_frequency1 GENERATE flatten($0), COUNT($1) as count;
-</pre>
-<ul>
-<li>
-<p> Use the FOREACH-GENERATE operator to assign names to the fields. </p>
-</li>
-</ul>
-<pre class="code">
-hour_frequency3 = FOREACH hour_frequency2 GENERATE $0 as ngram, $1 as hour, $2 as count;
-</pre>
-<ul>
-<li>
-<p> Use the  FILTERoperator to get the n-grams for hour &lsquo;00&rsquo;  </p>
-</li>
-</ul>
-<pre class="code">
-hour00 = FILTER hour_frequency2 BY hour eq '00';
-</pre>
-<ul>
-<li>
-<p> Uses the FILTER operators to get the n-grams for hour &lsquo;12&rsquo; </p>
-</li>
-</ul>
-<pre class="code">
-hour12 = FILTER hour_frequency3 BY hour eq '12';
-</pre>
-<ul>
-<li>
-<p> Use the JOIN operator to get the n-grams that appear in both hours. </p>
-</li>
-</ul>
-<pre class="code">
-same = JOIN hour00 BY $0, hour12 BY $0;
-</pre>
-<ul>
-<li>
-<p> Use the FOREACH-GENERATE operator to record their frequency. </p>
-</li>
-</ul>
-<pre class="code">
-same1 = FOREACH same GENERATE hour_frequency2::hour00::group::ngram as ngram, $2 as count00, $5 as count12;
-</pre>
-<ul>
-<li>
-<p> Use the PigStorage function to store the results. The output file contains a list of n-grams with the following fields: <strong>hour</strong>, <strong>count00</strong>, <strong>count12</strong>. </p>
-</li>
-</ul>
-<pre class="code">
-STORE same1 INTO '/tmp/tutorial-join-results' USING PigStorage();
-</pre>
-</div>
-</div>
-<!--+
-    |end content
-    +-->
-<div class="clearboth">&nbsp;</div>
-</div>
-<div id="footer">
-<!--+
-    |start bottomstrip
-    +-->
-<div class="lastmodified">
-<script type="text/javascript"><!--
-document.write("Last Published: " + document.lastModified);
-//  --></script>
-</div>
-<div class="copyright">
-        Copyright &copy;
-         2007-2010 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
-</div>
-<!--+
-    |end bottomstrip
-    +-->
-</div>
-</body>
-</html>