ul-wukong 4.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (261) hide show
  1. checksums.yaml +15 -0
  2. data/.gitignore +60 -0
  3. data/.gitmodules +6 -0
  4. data/.rspec +2 -0
  5. data/.travis.yml +19 -0
  6. data/.yardopts +6 -0
  7. data/CHANGELOG.md +7 -0
  8. data/Gemfile +17 -0
  9. data/Guardfile +12 -0
  10. data/LICENSE.md +95 -0
  11. data/NOTES-travis.md +31 -0
  12. data/README-old.md +422 -0
  13. data/README.md +1308 -0
  14. data/Rakefile +28 -0
  15. data/TODO.md +99 -0
  16. data/bin/cutc +30 -0
  17. data/bin/cuttab +5 -0
  18. data/bin/greptrue +6 -0
  19. data/bin/md5sort +20 -0
  20. data/bin/setcat +11 -0
  21. data/bin/tabchar +5 -0
  22. data/bin/uniq-ord +59 -0
  23. data/bin/uniqc +3 -0
  24. data/bin/wu +34 -0
  25. data/bin/wu-clean-encoding +31 -0
  26. data/bin/wu-date +13 -0
  27. data/bin/wu-datetime +13 -0
  28. data/bin/wu-hist +3 -0
  29. data/bin/wu-lign +186 -0
  30. data/bin/wu-local +4 -0
  31. data/bin/wu-plus +9 -0
  32. data/bin/wu-source +5 -0
  33. data/bin/wu-sum +31 -0
  34. data/diagrams/wu_local.dot +39 -0
  35. data/diagrams/wu_local.dot.png +0 -0
  36. data/examples/Gemfile +38 -0
  37. data/examples/README.md +9 -0
  38. data/examples/basic/string_reverser.rb +23 -0
  39. data/examples/basic/tiny_count.rb +8 -0
  40. data/examples/basic/word_count/accumulator.rb +26 -0
  41. data/examples/basic/word_count/tokenizer.rb +13 -0
  42. data/examples/basic/word_count/word_count.rb +6 -0
  43. data/examples/dataflow/scraper_macro_flow.rb +28 -0
  44. data/examples/deploy_pack/Gemfile +6 -0
  45. data/examples/deploy_pack/README.md +6 -0
  46. data/examples/deploy_pack/a/b/c/.gitkeep +0 -0
  47. data/examples/deploy_pack/app/processors/string_reverser.rb +5 -0
  48. data/examples/deploy_pack/config/environment.rb +1 -0
  49. data/examples/dsl/dataflow/fibonacci_series.rb +101 -0
  50. data/examples/dsl/dataflow/scraper_macro_flow.rb +28 -0
  51. data/examples/dsl/dataflow/simple.rb +12 -0
  52. data/examples/dsl/dataflow/telegram.rb +45 -0
  53. data/examples/dsl/workflow/cherry_pie.dot +97 -0
  54. data/examples/dsl/workflow/cherry_pie.md +104 -0
  55. data/examples/dsl/workflow/cherry_pie.png +0 -0
  56. data/examples/dsl/workflow/cherry_pie.rb +101 -0
  57. data/examples/empty/.gitkeep +0 -0
  58. data/examples/examples_helper.rb +9 -0
  59. data/examples/geo.rb +4 -0
  60. data/examples/geo/geo_grids.numbers +0 -0
  61. data/examples/geo/geolocated.rb +331 -0
  62. data/examples/geo/quadtile.rb +69 -0
  63. data/examples/geo/spec/geolocated_spec.rb +247 -0
  64. data/examples/geo/tile_fetcher.rb +77 -0
  65. data/examples/graph/implied_geolocation/README.md +63 -0
  66. data/examples/graph/minimum_spanning_tree/airfares_graphviz.rb +73 -0
  67. data/examples/improver/tweet_summary.rb +73 -0
  68. data/examples/loadable.rb +2 -0
  69. data/examples/munging/airline_flights/airline_flights.rake +83 -0
  70. data/examples/munging/airline_flights/airplane.rb +0 -0
  71. data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
  72. data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
  73. data/examples/munging/airline_flights/indexable.rb +75 -0
  74. data/examples/munging/airline_flights/indexable_spec.rb +90 -0
  75. data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
  76. data/examples/munging/airline_flights/tasks.rake +83 -0
  77. data/examples/munging/airline_flights/topcities.rb +167 -0
  78. data/examples/munging/geo/geo_json.rb +54 -0
  79. data/examples/munging/geo/geo_models.rb +69 -0
  80. data/examples/munging/geo/geonames_models.rb +107 -0
  81. data/examples/munging/geo/iso_codes.rb +172 -0
  82. data/examples/munging/geo/reconcile_countries.rb +124 -0
  83. data/examples/munging/geo/tasks.rake +71 -0
  84. data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
  85. data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
  86. data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
  87. data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
  88. data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
  89. data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
  90. data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
  91. data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +5 -0
  92. data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
  93. data/examples/munging/wikipedia/dbpedia/extract_links-cruft.rb +66 -0
  94. data/examples/munging/wikipedia/dbpedia/extract_links.rb +260 -0
  95. data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
  96. data/examples/rake_helper.rb +97 -0
  97. data/examples/ruby_project/Gemfile +6 -0
  98. data/examples/ruby_project/README.md +6 -0
  99. data/examples/ruby_project/a/b/c/.gitkeep +0 -0
  100. data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
  101. data/examples/server_logs/logline.rb +95 -0
  102. data/examples/server_logs/models.rb +66 -0
  103. data/examples/server_logs/page_counts.pig +48 -0
  104. data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
  105. data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
  106. data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
  107. data/examples/server_logs/server_logs-03-breadcrumbs-full.rb +71 -0
  108. data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
  109. data/examples/serverlogs/geo_ip_mapping/munge_geolite.rb +82 -0
  110. data/examples/serverlogs/models/logline.rb +102 -0
  111. data/examples/serverlogs/parser/apache_parser_widget.rb +46 -0
  112. data/examples/serverlogs/visit_paths/common.rb +4 -0
  113. data/examples/serverlogs/visit_paths/page_counts.pig +48 -0
  114. data/examples/serverlogs/visit_paths/serverlogs-01-parse-script.rb +11 -0
  115. data/examples/serverlogs/visit_paths/serverlogs-02-histograms-full.rb +31 -0
  116. data/examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb +12 -0
  117. data/examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb +67 -0
  118. data/examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb +38 -0
  119. data/examples/splitter.rb +94 -0
  120. data/examples/string_reverser.rb +7 -0
  121. data/examples/text/pig_latin/pig_latinizer.rb +35 -0
  122. data/examples/text/pig_latin/pig_latinizer_widget.rb +16 -0
  123. data/examples/text/regional_flavor/README.md +14 -0
  124. data/examples/text/regional_flavor/article_wordbags.pig +39 -0
  125. data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
  126. data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
  127. data/examples/twitter.rb +5 -0
  128. data/lib/hanuman.rb +36 -0
  129. data/lib/hanuman/graph.rb +97 -0
  130. data/lib/hanuman/graphvizzer.rb +206 -0
  131. data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
  132. data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
  133. data/lib/hanuman/link.rb +35 -0
  134. data/lib/hanuman/registry.rb +46 -0
  135. data/lib/hanuman/stage.rb +128 -0
  136. data/lib/hanuman/tree.rb +67 -0
  137. data/lib/wu/geo.rb +4 -0
  138. data/lib/wu/geo/geo_grids.numbers +0 -0
  139. data/lib/wu/geo/geolocated.rb +331 -0
  140. data/lib/wu/geo/quadtile.rb +69 -0
  141. data/lib/wu/graph/union_find.rb +62 -0
  142. data/lib/wu/model/reconcilable.rb +63 -0
  143. data/lib/wu/munging.rb +71 -0
  144. data/lib/wu/social/models/twitter.rb +31 -0
  145. data/lib/wu/wikipedia/models.rb +20 -0
  146. data/lib/wukong.rb +54 -0
  147. data/lib/wukong/dataflow.rb +43 -0
  148. data/lib/wukong/doc_helpers.rb +14 -0
  149. data/lib/wukong/doc_helpers/dataflow_handler.rb +29 -0
  150. data/lib/wukong/doc_helpers/field_handler.rb +91 -0
  151. data/lib/wukong/doc_helpers/processor_handler.rb +29 -0
  152. data/lib/wukong/driver.rb +214 -0
  153. data/lib/wukong/driver/event_machine_driver.rb +15 -0
  154. data/lib/wukong/driver/wiring.rb +68 -0
  155. data/lib/wukong/local.rb +42 -0
  156. data/lib/wukong/local/runner.rb +96 -0
  157. data/lib/wukong/local/stdio_driver.rb +104 -0
  158. data/lib/wukong/logger.rb +102 -0
  159. data/lib/wukong/model/faker.rb +136 -0
  160. data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
  161. data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
  162. data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
  163. data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
  164. data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
  165. data/lib/wukong/plugin.rb +48 -0
  166. data/lib/wukong/processor.rb +110 -0
  167. data/lib/wukong/rake_helper.rb +6 -0
  168. data/lib/wukong/runner.rb +169 -0
  169. data/lib/wukong/runner/boot_sequence.rb +123 -0
  170. data/lib/wukong/runner/code_loader.rb +52 -0
  171. data/lib/wukong/runner/command_runner.rb +44 -0
  172. data/lib/wukong/runner/deploy_pack_loader.rb +75 -0
  173. data/lib/wukong/runner/help_message.rb +42 -0
  174. data/lib/wukong/source.rb +33 -0
  175. data/lib/wukong/source/source_driver.rb +74 -0
  176. data/lib/wukong/source/source_runner.rb +38 -0
  177. data/lib/wukong/spec_helpers.rb +74 -0
  178. data/lib/wukong/spec_helpers/integration_tests.rb +150 -0
  179. data/lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb +207 -0
  180. data/lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb +97 -0
  181. data/lib/wukong/spec_helpers/shared_examples.rb +22 -0
  182. data/lib/wukong/spec_helpers/unit_tests.rb +135 -0
  183. data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb +132 -0
  184. data/lib/wukong/spec_helpers/unit_tests/unit_test_matchers.rb +169 -0
  185. data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +60 -0
  186. data/lib/wukong/version.rb +3 -0
  187. data/lib/wukong/widget/echo.rb +55 -0
  188. data/lib/wukong/widget/extract.rb +122 -0
  189. data/lib/wukong/widget/filters.rb +452 -0
  190. data/lib/wukong/widget/logger.rb +56 -0
  191. data/lib/wukong/widget/operators.rb +82 -0
  192. data/lib/wukong/widget/reducers.rb +10 -0
  193. data/lib/wukong/widget/reducers/accumulator.rb +73 -0
  194. data/lib/wukong/widget/reducers/bin.rb +368 -0
  195. data/lib/wukong/widget/reducers/count.rb +73 -0
  196. data/lib/wukong/widget/reducers/group.rb +128 -0
  197. data/lib/wukong/widget/reducers/group_concat.rb +98 -0
  198. data/lib/wukong/widget/reducers/improver.rb +71 -0
  199. data/lib/wukong/widget/reducers/join_xml.rb +37 -0
  200. data/lib/wukong/widget/reducers/moments.rb +72 -0
  201. data/lib/wukong/widget/reducers/sort.rb +180 -0
  202. data/lib/wukong/widget/reducers/uniq.rb +91 -0
  203. data/lib/wukong/widget/serializers.rb +317 -0
  204. data/lib/wukong/widget/utils.rb +46 -0
  205. data/lib/wukong/widgets.rb +7 -0
  206. data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
  207. data/spec/examples/dataflow/parse_apache_logs_spec.rb +8 -0
  208. data/spec/examples/dataflow/parsing_spec.rb +14 -0
  209. data/spec/examples/dataflow/simple_spec.rb +34 -0
  210. data/spec/examples/dataflow/telegram_spec.rb +43 -0
  211. data/spec/examples/graph/minimum_spanning_tree_spec.rb +34 -0
  212. data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
  213. data/spec/examples/munging/airline_flights_spec.rb +202 -0
  214. data/spec/examples/text/pig_latin_spec.rb +18 -0
  215. data/spec/examples/workflow/cherry_pie_spec.rb +36 -0
  216. data/spec/hanuman/graph_spec.rb +119 -0
  217. data/spec/hanuman/hanuman_spec.rb +10 -0
  218. data/spec/hanuman/registry_spec.rb +123 -0
  219. data/spec/hanuman/stage_spec.rb +81 -0
  220. data/spec/hanuman/tree_spec.rb +119 -0
  221. data/spec/spec.opts +1 -0
  222. data/spec/spec_helper.rb +43 -0
  223. data/spec/support/example_test_helpers.rb +95 -0
  224. data/spec/support/hanuman_test_helpers.rb +92 -0
  225. data/spec/support/integration_helper.rb +38 -0
  226. data/spec/support/model_test_helpers.rb +115 -0
  227. data/spec/support/shared_context_for_graphs.rb +57 -0
  228. data/spec/support/shared_context_for_reducers.rb +37 -0
  229. data/spec/support/shared_examples_for_builders.rb +94 -0
  230. data/spec/support/shared_examples_for_shortcuts.rb +57 -0
  231. data/spec/wu/model/reconcilable_spec.rb +152 -0
  232. data/spec/wukong/dataflow_spec.rb +87 -0
  233. data/spec/wukong/driver_spec.rb +154 -0
  234. data/spec/wukong/local/runner_spec.rb +29 -0
  235. data/spec/wukong/local/stdio_driver_spec.rb +73 -0
  236. data/spec/wukong/local_spec.rb +6 -0
  237. data/spec/wukong/logger_spec.rb +49 -0
  238. data/spec/wukong/model/faker_spec.rb +132 -0
  239. data/spec/wukong/processor_spec.rb +21 -0
  240. data/spec/wukong/runner_spec.rb +132 -0
  241. data/spec/wukong/source_spec.rb +6 -0
  242. data/spec/wukong/widget/extract_spec.rb +101 -0
  243. data/spec/wukong/widget/filters_spec.rb +79 -0
  244. data/spec/wukong/widget/logger_spec.rb +23 -0
  245. data/spec/wukong/widget/operators_spec.rb +25 -0
  246. data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
  247. data/spec/wukong/widget/reducers/count_spec.rb +11 -0
  248. data/spec/wukong/widget/reducers/group_spec.rb +21 -0
  249. data/spec/wukong/widget/reducers/join_xml_spec.rb +25 -0
  250. data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
  251. data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
  252. data/spec/wukong/widget/reducers/uniq_spec.rb +14 -0
  253. data/spec/wukong/widget/serializers_spec.rb +114 -0
  254. data/spec/wukong/widget/sink_spec.rb +19 -0
  255. data/spec/wukong/widget/source_spec.rb +65 -0
  256. data/spec/wukong/wu-local_spec.rb +109 -0
  257. data/spec/wukong/wu-source_spec.rb +32 -0
  258. data/spec/wukong/wu_spec.rb +14 -0
  259. data/spec/wukong/wukong_spec.rb +10 -0
  260. data/wukong.gemspec +35 -0
  261. metadata +465 -0
@@ -0,0 +1,71 @@
1
+ require_relative('../../rake_helper')
2
+
3
+ Pathname.register_paths(
4
+ geo_data: [:data, 'geo'],
5
+ geo_work: [:work, 'geo'],
6
+ geo_code: File.dirname(__FILE__),
7
+ #
8
+ iso_3166: [:geo_data, 'iso_codes', "iso_3166.tsv" ],
9
+ geonames_countries: [:geo_data, 'geonames', "geonames_countries.json" ],
10
+ #
11
+ countries_json: [:geo_work, "countries.json" ],
12
+ country_name_lookup: [:geo_work, "country_name_lookup.tsv" ],
13
+ )
14
+
15
+ chain :geo do
16
+ code_files = FileList[Pathname.of(:geo_code, '*.rb').to_s]
17
+ chain(:countries) do
18
+
19
+ task(:load) do
20
+ require_relative('./geo_models')
21
+ require_relative('./geo_json')
22
+ require_relative('./geonames_models')
23
+ require_relative('./iso_codes')
24
+ require_relative('./reconcile_countries')
25
+ CountryReconciler.load_reconciled_countries
26
+ end
27
+
28
+ # desc 'load the ISO 3166 countries'
29
+ # task(:countries_iso_3166, after: [code_files, :force]) do |dest|
30
+ # require_relative('./iso_codes')
31
+ # p Wukong::Data::CountryCode.for_any_name('Bolivia')
32
+ # end
33
+
34
+ # step(:geonames_countries, doc: 'load the Geonames countries',
35
+ # invoke: 'geo:countries:load',
36
+ # # , after: [code_files, :force]
37
+ # ) do |dest|
38
+ # Wukong::Data::GeonamesGeoJson.load(:geonames_countries)
39
+ # end
40
+
41
+ desc 'Add the iso_codes data to the geonames countries'
42
+ create_file(:countries_json, invoke: 'geo:countries:load', after: [code_files, :force]) do |dest|
43
+ Geo::Country.values.each do |country|
44
+ dest << country.to_json << "\n"
45
+ end
46
+ end
47
+
48
+ desc 'Add the iso_codes data to the geonames countries'
49
+ create_file(:country_name_lookup, invoke: 'geo:countries:load', after: [code_files, :force]) do |dest|
50
+ Geo::Country.values.each do |ct|
51
+ ct.names.each do |alt_name|
52
+ dest << [ct.country_id, ct.country_al3id, ct.country_numid,
53
+ ct.tld_id, ct.geonames_id,
54
+ ct.name,
55
+ Geo::Place.slugify_name(alt_name), alt_name
56
+ ].join("\t") << "\n"
57
+ end
58
+ end
59
+ end
60
+
61
+ # task(:country_name_lookup => :load) do
62
+ # Geo::CountryNameLookup.load
63
+ # end
64
+
65
+ end
66
+ end
67
+
68
+ task :default => [
69
+ # 'geo:countries',
70
+ 'geo:countries:country_name_lookup'
71
+ ]
@@ -0,0 +1,79 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Extracts wikipedia articles from bzipped xml, outputs them in TSV. Article
4
+ # text is XML encoded, but all newlines and tabs (in fact, all control
5
+ # characters) are converted to XML entities, making it safe to truck around as
6
+ # TSV.
7
+ #
8
+ # Sample Pig LOAD Statement:
9
+ #
10
+ # all_articles = LOAD '$articles' AS
11
+ # (id:long, namespace:int, title:chararray, revision_timestamp:long, redirect:chararray, text:chararray);
12
+ #
13
+
14
+ # ## Usage
15
+ #
16
+ # Flattens the wikipedia 'enwiki-latest-pages-articles.xml.gz' into a
17
+ # one-line-per-record heap.
18
+ #
19
+ # examples/munging/wikipedia/articles/extract_articles-templated.rb --rm --run \
20
+ # /data/origin/dumps.wikimedia.org/enwiki/20120601/enwiki-20120601-pages-articles.xml
21
+ # /data/results/wikipedia/full/articles.json.tsv
22
+ #
23
+
24
+ require 'wukong'
25
+ require 'wukong/streamer/encoding_cleaner'
26
+ require 'crack/xml'
27
+ require 'multi_json'
28
+ require 'oj'
29
+ require_relative '../utils/munging_utils.rb'
30
+
31
+ module ArticlesExtractor
32
+ class Mapper < Wukong::Streamer::LineStreamer
33
+ include Wukong::Streamer::EncodingCleaner
34
+ include MungingUtils
35
+
36
+ def lines
37
+ @lines ||= []
38
+ end
39
+
40
+ def recordize line
41
+ lines << line
42
+ if line =~ /<\/page>/
43
+ result = Crack::XML::parse(lines.join("\n"))
44
+ @lines = []
45
+ return [result]
46
+ else
47
+ return nil
48
+ end
49
+ end
50
+
51
+ def process record
52
+ if record.has_key? 'mediawiki'
53
+ record = record['mediawiki']
54
+ end
55
+
56
+ redirect = record['page']['redirect'] ? record['page']['redirect']['title'] : ''
57
+ timestamp = Time.iso8601(record['page']['revision']['timestamp']).to_flat
58
+ raw_text = record['page']['revision']['text']
59
+
60
+ # some few parts per million articles have an empty body -- workaround
61
+ raw_text = '' if not raw_text.is_a?(String)
62
+
63
+ result = [
64
+ record['page']['id'],
65
+ record['page']['ns'],
66
+ scrub_control_chars(record['page']['title']),
67
+ record['page']['revision']['id'],
68
+ timestamp,
69
+ scrub_control_chars(redirect),
70
+ safe_json_encode(raw_text)
71
+ ]
72
+ yield result
73
+ end
74
+ end
75
+ end
76
+
77
+ # Force it to run in a single map task, to avoid writing a custom input format.
78
+ # The job runs in 2 hours, once; much less than the time it'd take me to do so.
79
+ Wukong::Script.new(ArticlesExtractor::Mapper, nil, min_split_size: 1152921504606846976).run
@@ -0,0 +1,136 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Extracts wikipedia articles from bzipped xml, outputs them in TSV. Article
4
+ # text is XML encoded, but all newlines and tabs (in fact, all control
5
+ # characters) are converted to XML entities, making it safe to truck around as
6
+ # TSV.
7
+
8
+ # ## Schema
9
+ #
10
+ # Sample Pig LOAD Statement:
11
+ #
12
+ # all_articles = LOAD '$articles' AS
13
+ # (id:long, namespace:int, title:chararray, revision_timestamp:long, redirect:chararray, text:chararray);
14
+ #
15
+
16
+ # ## Usage
17
+ #
18
+ # Flattens the wikipedia 'enwiki-latest-pages-articles.xml.gz' into a
19
+ # one-line-per-record heap.
20
+ #
21
+ # examples/munging/wikipedia/articles/extract_articles-templated.rb --rm --run \
22
+ # /data/origin/dumps.wikimedia.org/enwiki/20120601/enwiki-20120601-pages-articles.xml
23
+ # /data/results/wikipedia/full/articles.tsv
24
+ #
25
+
26
+ require 'wukong'
27
+ require 'wukong/streamer/encoding_cleaner'
28
+ require 'crack/xml'
29
+ require 'multi_json'
30
+ require_relative '../utils/munging_utils.rb'
31
+
32
+ # <page>
33
+ # <title>Anarchism</title>
34
+ # <ns>0</ns>
35
+ # <id>12</id>
36
+ # <revision>
37
+ # <id>370845941</id>
38
+ # <timestamp>2010-06-29T20:14:56Z</timestamp>
39
+ # <contributor>
40
+ # <username>Centographer</username>
41
+ # <id>12640258</id>
42
+ # </contributor>
43
+ # <minor />
44
+ # <comment>clarifying not ordinary anarcho-socialism</comment>
45
+ # <text xml:space="preserve">
46
+ # ...snip ...
47
+ # </text>
48
+ # <sha1>...</sha1>
49
+ # </revision>
50
+ # </page>
51
+ #
52
+ module ArticlesExtractor
53
+ class Mapper < Wukong::Streamer::LineStreamer
54
+ include Wukong::Streamer::EncodingCleaner
55
+ include MungingUtils
56
+
57
+ def initialize(*)
58
+ super
59
+ @lines = []
60
+ @state = :out_of_article
61
+ @num_lines = 0
62
+ end
63
+
64
+ # Bolt together all lines between a <page> and a </page> marker.
65
+ def recordize line
66
+ @num_lines += 1
67
+ return if @state == :out_of_article && (ARTICLE_BEG_RE !~ line)
68
+ @state = :in_article
69
+ #
70
+ @lines << line
71
+ if ARTICLE_END_RE =~ line
72
+ result = @lines.join("\n")
73
+ @lines = []
74
+ @state = :out_of_article
75
+ return [result]
76
+ else
77
+ return nil
78
+ end
79
+ end
80
+
81
+ def process article
82
+ info = ARTICLE_RE.match(article)
83
+ if not info then warn "Bad match line #{@lines}: #{article.to_s[0..2000]}" ; return ; end
84
+
85
+ timestamp = [info[:rts_yr], info[:rts_mo], info[:rts_day], info[:rts_hr], info[:rts_min], info[:rts_sec], 'Z'].join
86
+ text = Crack::XML::parse("<text>#{info[:text]}</text>")['text'] || ''
87
+ redirect = info[:redirect] || ''
88
+
89
+ record = [
90
+ info[:id],
91
+ info[:ns],
92
+ scrub_control_chars(info[:title]),
93
+ info[:revision_id],
94
+ timestamp,
95
+ scrub_control_chars(redirect),
96
+ safe_json_encode(text)
97
+ ]
98
+ yield record
99
+ end
100
+ end
101
+
102
+ ARTICLE_BEG_RE = %r{\A\s*<page>\z}
103
+ ARTICLE_END_RE = %r{\A\s*</page>\z}
104
+ ARTICLE_RE = %r{\A
105
+ \s*<page>
106
+ \s* <title>(?<title>[^<]*)</title>
107
+ \s* <ns>(?<ns>\d+)</ns>
108
+ \s* <id>(?<id>\d+)</id>
109
+ \s* (?:<redirect\stitle=\"(?<redirect>[^\"]+)\"\s/>)?
110
+ \s* (?:<restrictions>(?<restrictions>[^<]+)</restrictions>)?
111
+ \s* <revision>
112
+ \s* <id>(?<revision_id>\d+)</id>
113
+ \s* <timestamp>(?<rts_yr>\d\d\d\d)-(?<rts_mo>\d\d)-(?<rts_day>\d\d)T(?<rts_hr>\d\d):(?<rts_min>\d\d):(?<rts_sec>\d\d)Z</timestamp>
114
+ \s* (?:
115
+ <contributor>\s*<username>[^<]+</username>\s*<id>\d+</id>\s*</contributor> |
116
+ <contributor>\s*<ip>[\d\.]+</ip>\s*</contributor> |
117
+ <contributor\sdeleted="deleted"\s/>
118
+ )
119
+ \s* (?:<minor\s/>)?
120
+ \s* (?:<comment>[^<]*</comment>|<comment\sdeleted="deleted"\s/>)?
121
+ \s* (?:
122
+ <text\sxml:space="preserve">
123
+ (?<text>.*)
124
+ </text>
125
+ |
126
+ <text\sxml:space="preserve"\s/>
127
+ )
128
+ \s* (?:<sha1>(?<sha1>[a-z0-9]+)</sha1> | <sha1\s/>)
129
+ \s* </revision>
130
+ \s*</page>\s*\z}xmo
131
+
132
+ end
133
+
134
+ # Force it to run in a single map task, to avoid writing a custom input format.
135
+ # The job runs in 2 hours, once; much less than the time it'd take me to do so.
136
+ Wukong::Script.new(ArticlesExtractor::Mapper, nil, min_split_size: 1152921504606846976).run
@@ -0,0 +1,54 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Generate plain-text versions of articles from the tsv-converted raw article data
4
+ # (output from extract_articles)
5
+ #
6
+ # This strips out template tags, wiki links, and so forth
7
+ #
8
+ # Everything that's left is either actual text, or nicely detached punctuation.
9
+
10
+ # ## Usage
11
+ #
12
+ # Uses the output of extract_articles-templated.rb:
13
+ #
14
+ # examples/munging/wikipedia/articles/textualize_articles.rb --rm --run \
15
+ # /data/results/wikipedia/full/articles.json.tsv \
16
+ # /data/results/wikipedia/full/article_texts.json.tsv
17
+ #
18
+
19
+ require 'wukong'
20
+ require 'multi_json'
21
+ require 'oj'
22
+ require 'strscan'
23
+ require 'find'
24
+ require 'sanitize'
25
+ #
26
+ require_relative '../utils/munging_utils.rb'
27
+ require_relative './wp2txt_article'
28
+ require_relative './wp2txt_utils'
29
+
30
+ module TextualizeArticles
31
+
32
+ class Mapper < Wukong::Streamer::RecordStreamer
33
+ include MungingUtils
34
+
35
+ @@errors = 0
36
+ MAX_ERRORS = 1_000
37
+
38
+ def process(id, namespace, title, revision_id, timestamp, redirect, raw_text)
39
+
40
+ text = MultiJson.decode(raw_text)
41
+ article = Wp2txt::Article.new(text, title)
42
+ jsonized_text = MultiJson.encode(article.polish)
43
+
44
+ yield [id, namespace, title, revision_id, timestamp, redirect, jsonized_text]
45
+
46
+ rescue StandardError => err
47
+ Wukong.bad_record("Bad Record", err, raw_text)
48
+ raise "Too many errors" if (@@errors += 1) > MAX_ERRORS
49
+ end
50
+
51
+ end
52
+ end
53
+
54
+ Wukong::Script.new(TextualizeArticles::Mapper, nil).run
@@ -0,0 +1,43 @@
1
+
2
+ # cat /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/enwiki-20120601-pages-articles.xml | ruby -ne '$_.chomp!; case $_ when %r{\A(\s*)<redirect title="[^"]+" />\z} then puts %Q{#{$1}<redirect title=\"...\" />} when %r{\A(\s*<[^>]+>)[^<]*(</\w+>)\z} then puts "#{$1}...#{$2}" ; when %r{\A(\s*<[^>]+>)} then puts $1 ; when %r{\A[^<]*(</[\w-]+>)\z} then puts $1 else false end ' > /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/xml-tags-seen.txt &
3
+
4
+ # cat /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/xml-tags-seen.txt | sort -S1G --temp=/mnt{,2,3,4}/tmp | uniq -c | sort -n | tee xml-tags-census.txt
5
+
6
+ # cat /mnt/data/origin/dumps.wikimedia.org/enwiki/20120601/xml-tags-seen.txt | sort -S1800M --temp=/mnt{2,3,4}/tmp | uniq -c | sort -n | tee xml-tags-census.txt &
7
+
8
+
9
+ # 609844 <page>
10
+ # 609844 <title>...</title>
11
+ # 609844 <ns>...</ns>
12
+ # 609844 <id>...</id>
13
+ # 714 <restrictions>...</restrictions>
14
+ #
15
+ # 251254 <redirect title="..." />
16
+ # 609844 <revision>
17
+ # 609844 <id>...</id>
18
+ # 609844 <timestamp>...</timestamp>
19
+ #
20
+ # 609843 <contributor>
21
+ # 548236 <username>...</username>
22
+ # 61607 <ip>...</ip>
23
+ # 548236 <id>...</id>
24
+ # 609843 </contributor>
25
+ #
26
+ # 288319 <minor />
27
+ # 529222 <comment>...</comment>
28
+ # 108 <comment>
29
+ # 108 </comment>
30
+ #
31
+ # 224818 <text xml:space="preserve">...</text>
32
+ # 385019 <text xml:space="preserve">
33
+ # 7 <text xml:space="preserve" />
34
+ #
35
+ # 346490 <sha1>...</sha1>
36
+ # 263354 <sha1 />
37
+ #
38
+ # 609843 </revision>
39
+ # 609843 </page>
40
+ #
41
+ # 384998 </text>
42
+ # 20 </text>
43
+ # 643676 </page>
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Yoichiro Hasebe
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,259 @@
1
+ #!/usr/bin/env ruby
2
+ # -*- coding: utf-8 -*-
3
+
4
+ # Taken from Yoichiro Yohasebe's [`wp2txt` project](https://github.com/yohasebe/wp2txt)
5
+ # with liberal modifications for our purposes.
6
+ #
7
+ # This software is distributed under the MIT License. Please see the `./wp2txt-LICENSE.txt` file.
8
+
9
+ require 'strscan'
10
+ require_relative 'wp2txt_utils'
11
+
12
+ module Wp2txt
13
+
14
+ # possible element type, which could be later chosen to print or not to print
15
+ # :mw_heading
16
+ # :mw_htable
17
+ # :mw_quote
18
+ # :mw_unordered
19
+ # :mw_ordered
20
+ # :mw_definition
21
+ # :mw_pre
22
+ # :mw_paragraph
23
+ # :mw_comment
24
+ # :mw_math
25
+ # :mw_source
26
+ # :mw_inputbox
27
+ # :mw_template
28
+ # :mw_link
29
+ # :mw_summary
30
+ # :mw_blank
31
+ # :mw_redirect
32
+
33
+ # an article contains elements, each of which is [TYPE, string]
34
+ class Article
35
+
36
+ include Wp2txt
37
+ attr_accessor :elements, :title
38
+
39
+ # class varialbes to save resource for generating regexps
40
+ # those with a trailing number 1 represent opening tag/markup
41
+ # those with a trailing number 2 represent closing tag/markup
42
+ # those without a trailing number contain both opening/closing tags/markups
43
+
44
+ @@in_template_regex = Regexp.new('^\s*\{\{[^\}]+\}\}\s*$')
45
+ @@in_link_regex = Regexp.new('^\s*\[.*\]\s*$')
46
+
47
+ @@in_inputbox_regex = Regexp.new('<inputbox>.*?<\/inputbox>')
48
+ @@in_inputbox_regex1 = Regexp.new('<inputbox>')
49
+ @@in_inputbox_regex2 = Regexp.new('<\/inputbox>')
50
+
51
+ @@in_source_regex = Regexp.new('<source.*?>.*?<\/source>')
52
+ @@in_source_regex1 = Regexp.new('<source.*?>')
53
+ @@in_source_regex2 = Regexp.new('<\/source>')
54
+
55
+ @@in_math_regex = Regexp.new('<math.*?>.*?<\/math>')
56
+ @@in_math_regex1 = Regexp.new('<math.*?>')
57
+ @@in_math_regex2 = Regexp.new('<\/math>')
58
+
59
+ @@in_heading_regex = Regexp.new('^=+.*?=+$')
60
+
61
+ @@in_html_table_regex = Regexp.new('<table.*?><\/table>')
62
+ @@in_html_table_regex1 = Regexp.new('<table\b')
63
+ @@in_html_table_regex2 = Regexp.new('<\/\s*table>')
64
+
65
+ @@in_table_regex1 = Regexp.new('^\s*\{\|')
66
+ @@in_table_regex2 = Regexp.new('^\|\}.*?$')
67
+
68
+ @@in_unordered_regex = Regexp.new('^\*')
69
+ @@in_ordered_regex = Regexp.new('^\#')
70
+ @@in_pre_regex = Regexp.new('^ ')
71
+ @@in_definition_regex = Regexp.new('^[\;\:]')
72
+
73
+ @@blank_line_regex = Regexp.new('^\s*$')
74
+
75
+ @@redirect_regex = Regexp.new('#(?:REDIRECT|転送)\s+\[\[(.+)\]\]', Regexp::IGNORECASE)
76
+
77
+ def initialize(text, title = "", strip_tmarker = false)
78
+ @title = title.strip
79
+ @strip_tmarker = strip_tmarker
80
+ parse text
81
+ end
82
+
83
+ def create_element(tp, text)
84
+ [tp, text]
85
+ end
86
+
87
+ def parse(source)
88
+ self.class.remove_comments(source)
89
+ @elements = []
90
+ mode = nil
91
+ open_stack = []
92
+ close_stack = []
93
+ source.each_line do |line|
94
+
95
+ case mode
96
+ when :mw_table
97
+ if @@in_table_regex2 =~ line
98
+ mode = nil
99
+ end
100
+ @elements.last.last << line
101
+ next
102
+ when :mw_inputbox
103
+ if @@in_inputbox_regex2 =~ line
104
+ mode = nil
105
+ end
106
+ @elements.last.last << line
107
+ next
108
+ when :mw_source
109
+ if @@in_source_regex2 =~ line
110
+ mode = nil
111
+ end
112
+ @elements.last.last << line
113
+ next
114
+ when :mw_math
115
+ if @@in_math_regex2 =~ line
116
+ mode = nil
117
+ end
118
+ @elements.last.last << line
119
+ next
120
+ when :mw_htable
121
+ if @@in_html_table_regex2 =~ line
122
+ mode = nil
123
+ end
124
+ @elements.last.last << line
125
+ next
126
+ end
127
+
128
+ case line
129
+ when @@blank_line_regex
130
+ @elements << create_element(:mw_blank, "\n")
131
+ when @@redirect_regex
132
+ @elements << create_element(:mw_redirect, line)
133
+ when @@in_template_regex
134
+ @elements << create_element(:mw_template, line)
135
+ when @@in_heading_regex
136
+ @elements << create_element(:mw_heading, "\n" + line + "\n")
137
+ when @@in_inputbox_regex
138
+ @elements << create_element(:mw_inputbox, line)
139
+ when @@in_inputbox_regex1
140
+ mode = :mw_inputbox
141
+ @elements << create_element(:mw_inputbox, line)
142
+ when @@in_source_regex
143
+ @elements << create_element(:mw_source, line)
144
+ when @@in_source_regex1
145
+ mode = :mw_source
146
+ @elements << create_element(:mw_source, line)
147
+ when @@in_math_regex
148
+ @elements << create_element(:mw_math, line)
149
+ when @@in_math_regex1
150
+ mode = :mw_math
151
+ @elements << create_element(:mw_math, line)
152
+ when @@in_html_table_regex
153
+ @elements << create_element(:mw_htable, line)
154
+ when @@in_html_table_regex1
155
+ mode = :mw_htable
156
+ @elements << create_element(:mw_htable, line)
157
+ when @@in_table_regex1
158
+ mode = :mw_table
159
+ @elements << create_element(:mw_table, line)
160
+ when @@in_unordered_regex
161
+ line = line.sub(/\A[\*\#\;\:\ ]+/, "") if @strip_tmarker
162
+ @elements << create_element(:mw_unordered, line)
163
+ when @@in_ordered_regex
164
+ line = line.sub(/\A[\*\#\;\:\ ]+/, "") if @strip_tmarker
165
+ @elements << create_element(:mw_ordered, line)
166
+ when @@in_pre_regex
167
+ line = line.sub(/\A\^\ /, "") if @strip_tmarker
168
+ @elements << create_element(:mw_pre, line)
169
+ when @@in_definition_regex
170
+ line = line.sub(/\A[\;\:\ ]+/, "") if @strip_tmarker
171
+ @elements << create_element(:mw_definition, line)
172
+ when @@in_link_regex
173
+ @elements << create_element(:mw_link, line)
174
+ else
175
+ @elements << create_element(:mw_paragraph, line)
176
+ end
177
+ end
178
+ @elements
179
+ end
180
+
181
+ def self.remove_comments(text)
182
+ # remove all comment texts
183
+ # and insert as many number of new line chars included in
184
+ # each comment instead
185
+ text.gsub!(/\<\!\-\-(.*?)\-\-\>/m) do |content|
186
+ num_of_newlines = content.count("\n")
187
+ (num_of_newlines == 0) ? "" : ("\n" * num_of_newlines)
188
+ end
189
+ end
190
+
191
+ EXCLUDE_SECTIONS = {
192
+ mw_title: false,
193
+ mw_heading: false,
194
+ mw_paragraph: false,
195
+ mw_link: false,
196
+ mw_redirect: false,
197
+ #
198
+ mw_pre: false,
199
+ mw_quote: false,
200
+ mw_unordered: false,
201
+ mw_ordered: false,
202
+ mw_definition: false,
203
+ #
204
+ mw_table: true,
205
+ mw_htable: true,
206
+ mw_blank: true,
207
+ mw_math: true,
208
+ mw_source: true,
209
+ mw_template: true,
210
+ }
211
+
212
+ def polish
213
+ contents = []
214
+ elements.each do |el_type, element|
215
+ contents << "+#{el_type.to_s.upcase}+\t" if $DEBUG_MODE
216
+ next if EXCLUDE_SECTIONS[el_type]
217
+ #
218
+ case el_type
219
+ when :mw_heading then contents << format_wiki(element)
220
+ when :mw_paragraph then contents << format_wiki(element)
221
+ when :mw_link then contents << format_wiki(element)
222
+ when :mw_redirect then contents << format_wiki(element) << "\n\n"
223
+ when :mw_pre then contents << element
224
+ when :mw_quote then contents << format_wiki(element)
225
+ when :mw_unordered then contents << format_wiki(element)
226
+ when :mw_ordered then contents << format_wiki(element)
227
+ when :mw_definition then contents << format_wiki(element)
228
+ when :mw_table, :mw_htable then contents << format_wiki(element)
229
+ when :mw_math, :mw_source then contents << format_wiki(element)
230
+ when :mw_blank then contents << format_wiki(element)
231
+ else
232
+ warn "Unknown section #{el_type}, content '#{element.to_s.gsub(/[\r\n]+/m,'')[0..200]}'"
233
+ contents << format_wiki(element)
234
+ end
235
+ end
236
+ text = contents.join
237
+
238
+ # Extract text from <b>..</b> and so forth; remove contents of <ref>...</ref> completely
239
+ text = clean_html(text)
240
+ # translate some recognizable special characters
241
+ text = special_chr(text)
242
+ # re-hang the no-wiki segments
243
+ unescape_nowiki(text)
244
+ # strip out templates. Several parts per million of these will fail for
245
+ # bad structure; I assume that means some parts per thousand will be
246
+ # mis-estimated. C'est la UGC.
247
+ text = remove_templates(text) if EXCLUDE_SECTIONS[:mw_template]
248
+
249
+ return '' if /\A\s*\z/m =~ text
250
+ #
251
+ result = EXCLUDE_SECTIONS[:mw_title] ? "" : "# #{format_wiki(title)}\n\n"
252
+ result << text
253
+ result.gsub!(/\n\n\n+/m){"\n\n"}
254
+ result << "\n"
255
+
256
+ result
257
+ end
258
+ end
259
+ end