wukong 3.0.0.pre → 3.0.0.pre2

data/examples/munging/wikipedia/pageviews/augment_pageviews.pig

@@ -0,0 +1,32 @@
+/*
+ * Augments raw pageview data with page ID.
+ * Pageview stats are *theoretically* uniquely keyed by namespace 
+ * and title, so that is what is used to join pageviews with page_metadata.
+ *
+ * In practice, the original pageview stats only give the URL visited, and
+ * reliably extracting namespace and title from the URL is difficult. Additionally, 
+ * page names change, redirects happen, and many other small things can go 
+ * wrong with the join. All pageview data is kept in the final table, but 
+ * the page id will be blank in rows where the join failed.
+ *
+ * Output format:
+ * page_id:int, namespace:int, title:chararray, num_visitors:long, 
+ * date:int, time:int, epoch_time:long, day_of_week:int
+ */
+
+%default PAGE_METADATA           '/data/results/wikipedia/full/page_metadata' -- page metadata for all Wikipedia pages
+%default EXTRACTED_PAGEVIEWS     '/data/scratch/wikipedia/full/pageviews'     -- raw extracted pageview stats (see extract_pageviews.rb)
+%default AUGMENTED_PAGEVIEWS_OUT '/data/results/wikipedia/full/pageviews'     -- where output will be stored
+
+page_metadata = LOAD '$PAGE_METADATA' AS 
+ (id:int, namespace:int, title:chararray, 
+  restrictions:chararray, counter:long, is_redirect:int, is_new:int, 
+  random:float, touched:int, page_latest:int, len:int);
+pageviews = LOAD '$EXTRACTED_PAGEVIEWS' AS (namespace:int, title:chararray, 
+  num_visitors:long, date:int, time:int, epoch_time:long, day_of_week:int);
+
+first_join = JOIN page_metadata BY (namespace, title) RIGHT OUTER, pageviews BY (namespace, title);
+final = FOREACH first_join GENERATE
+  page_metadata::id, pageviews::namespace, pageviews::title, pageviews::num_visitors,
+  pageviews::date, pageviews::time, pageviews::epoch_time, pageviews::day_of_week;
+STORE final INTO '$AUGMENTED_PAGEVIEWS_OUT';

data/examples/munging/wikipedia/pageviews/extract_pageviews.rb

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+# encoding:UTF-8
+
+# Pig output format:
+# namespace:int, title:chararray, num_visitors:long, date:int, time:int, epoch_time:long, day_of_week:int
+
+$:.unshift '/home/dlaw/dev/wukong_og/lib'
+$:.unshift '/home/dlaw/dev/gorillib/lib'
+
+require 'uri'
+require 'pathname'
+require 'json'
+require 'wukong'
+require 'wukong/streamer'
+require 'wukong/streamer/encoding_cleaner'
+load '/home/dlaw/dev/wukong/examples/munging/wikipedia/utils/munging_utils.rb'
+
+ENV['map_input_file'] ||= 'pagecounts-20071222-100000.gz'
+
+class String
+  def is_enwiki?
+    return (not (self =~ /^en /).nil?)
+  end
+
+  def is_after_enwiki?
+    return (not (self =~ /^(e[o-z][a-z]*|[f-z][a-z]+) /).nil?)
+  end
+end
+
+module PageviewsExtractor
+  class Mapper < Wukong::Streamer::LineStreamer
+    include Wukong::Streamer::EncodingCleaner
+    include MungingUtils
+
+    ns_json_file = File.open("/home/dlaw/dev/wukong/examples/munging/wikipedia/utils/namespaces.json",'r:UTF-8')
+    NAMESPACES = JSON.parse(ns_json_file.read)
+
+    # the filename strings are formatted as
+    # pagecounts-YYYYMMDD-HH0000.gz
+    def time_from_filename(filename)
+      parts = filename.split('-')
+      year = parts[1][0..3].to_i
+      month = parts[1][4..5].to_i
+      day = parts[1][6..7].to_i
+      hour = parts[2][0..1].to_i
+      return Time.new(year,month,day,hour)
+    end
+
+    def process line
+      # we only want enwiki lines
+      return if @done
+      if line.is_after_enwiki?
+        @done = true
+        return
+      end
+      return if not line.is_enwiki?
+      # we have an enwiki line - process it!
+      fields = line.split(' ')[1..-1]
+      out_fields = []
+      # add the namespace
+      namespace = nil
+      if fields[0].include? ':'
+        namespace = NAMESPACES[fields[0].split(':')[0]]
+        out_fields << (namespace || '0')
+      else
+        out_fields << '0'
+      end
+      # add the title
+      if namespace.nil?
+        out_fields << URI.unescape(fields[0])
+      else
+        out_fields << URI.unescape(fields[0][(fields[0].index(':')||-1)+1..-1])
+      end
+      # add number of visitors in the hour
+      out_fields << fields[2]
+      # grab date info from filename
+      file = Pathname.new(ENV['map_input_file']).basename
+      time = time_from_filename(file.to_s)
+      out_fields += time_columns_from_time(time)
+      yield out_fields
+    end
+  end
+end
+
+Wukong::Script.new(PageviewsExtractor::Mapper, Wukong::Streamer::LineStreamer).run

data/examples/munging/wikipedia/pig_style_guide.md

@@ -0,0 +1,25 @@
+# Pig Style Guide
+
+- Everything except names should be in all caps. E.g.
+
+		first_join = JOIN pages BY (namespace,title) 
+			RIGHT OUTER, pageviews BY (namespace, title);
+
+- Group and align columns in the script in ways that make sense. Don't be afraid of newlines. E.g.
+
+		second_pass   = FOREACH second_pass_j GENERATE 
+ 		  first_pass::from_id, pages::id,
+		  first_pass::from_namespace, first_pass::from_title, 
+		  first_pass::into_namespace, first_pass::into_title;
+		  
+- Columns that form an important sub-set of the table's data should be easily accessible as a unit. 
+
+	E.g. The edge list above has the from and into ids in the first and second columns, making it easy to just get an edge list of ids without the additional metadata.
+
+- When at all possible, you should include sample LOAD statements in the comments for your script. This makes it easy to use the output of your script
+
+- Parameterize as much as possible. All paths should be parameterized. 
+
+- Parameters should be in all caps, e.g. $NODE.
+
+- Parameters should have defaults if at all possible. When you define the default, also include a comment describing the parameter.

data/examples/munging/wikipedia/redirects/redirects_page_metadata.pig

@@ -0,0 +1,19 @@
+/*
+ * Filters the page metadata table, leaving only pages that 
+ * are redirects.
+ *
+ * Output Format (same as page_metadata):
+ * (id:int, namespace:int, title:chararray, restrictions:chararray, 
+ * counter:long, is_redirect:int, is_new:int, random:float,  touched:int, 
+ * page_latest:int, len:int)
+ */
+
+%default PAGE_METADATA '/data/results/wikipedia/full/page_metadata' -- page metdata for all pages in Wikipedia
+%default REDIRECTS_OUT '/data/results/wikipedia/full/redirect_page_metadata' -- place to store page metdata for redirects
+
+page_metadata = LOAD '$PAGE_METADATA' AS (id:int, namespace:int, title:chararray, 
+  restrictions:chararray, counter:long, is_redirect:int, is_new:int, random:float, 
+  touched:int, page_latest:int, len:int);
+
+redirects = FILTER page_metadata BY (is_redirect == 1);
+STORE redirects INTO '$REDIRECTS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_articles.pig

@@ -0,0 +1,23 @@
+/*
+ * This script filters the articles table, leaving only the articles
+ * in the specified subuniverse.
+ *
+ * Output format:
+ * page_id:int, title:chararray, namespace:int, rev_date:int, rev_time:int, 
+ * rev_epoch_time:long, rev_dow:int, article_text:chararray
+ */
+
+%default ARTICLES         '/data/results/wikipedia/full/articles' -- all articles in the wikipedia corpus
+%default SUB_NODES        '/data/results/wikipedia/mini/nodes'    -- all nodes in the subuniverse
+%default SUB_ARTICLES_OUT '/data/results/wikipedia/mini/articles' -- where output will be stored
+
+articles = LOAD '$ARTICLES' AS (page_id:int, title:chararray, namespace:int, 
+  rev_date:int, rev_time:int, rev_epoch_time:long, rev_dow:int, article_text:chararray);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+sub_articles_unfiltered = JOIN articles BY id, sub_nodes BY node_id;
+sub_articles = FOREACH sub_articles_unfiltered GENERATE
+  articles::page_id AS page_id, articles::title AS title, articles::namespace AS namespace,
+  articles::rev_date AS rev_date, articles::rev_time AS rev_time,
+  articles::rev_epoch_time AS rev_epoch_time, articles::rev_dow AS rev_dow,
+  articles::article_text AS article_text;
+STORE sub_articles INTO '$SUB_ARTICLES_OUT';

data/examples/munging/wikipedia/subuniverse/sub_page_metadata.pig

@@ -0,0 +1,24 @@
+/*
+ * This script filters the page metadata table, leaving only the pages 
+ * in the specified subuniverse.
+ *
+ * Output format (same as page_metadata):
+ * id:int, namespace:int, title:chararray, restrictions:chararray, counter:long, 
+ * is_redirect:int, is_new:int, random:float, touched:int, page_latest:int, len:int
+ */
+
+%default PAGE_METADATA         '/data/results/wikipedia/full/page_metadata' -- metadata for all pages in the wikipedia corpus
+%default SUB_NODES             '/data/results/wikipedia/mini/nodes'         -- all nodes in the subuniverse
+%default SUB_PAGE_METADATA_OUT '/data/results/wikipedia/mini/page_metadata' -- where output will be stored
+
+page_metadata = LOAD '$PAGE_METADATA' AS (id:int, namespace:int, title:chararray, 
+  restrictions:chararray, counter:long, is_redirect:int, is_new:int, random:float, 
+                                          touched:int, page_latest:int, len:int);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+sub_page_metadata_unfiltered = JOIN page_metadata BY id, sub_nodes BY node_id;
+sub_page_metadata = FOREACH sub_page_metadata_unfiltered GENERATE
+  page_metadata::id, page_metadata::namespace, page_metadata::title, 
+  page_metadata::restrictions, page_metadata::counter, page_metadata::is_redirect, 
+  page_metadata::is_new, page_metadata::random, page_metadata::touched, 
+  page_metadata::page_latest, page_metadata::len;
+STORE sub_page_metadata INTO '$SUB_PAGE_METADATA_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig

@@ -0,0 +1,22 @@
+/*
+ * This script filters the pagelinks table, leaving only the pagelinks
+ * that start within supplied subuniverse.
+ * 
+ * Output format (same as augmented_pagelinks):
+ * from_id:int, into_id:int, from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray
+ */
+
+%default PAGELINKS         '/data/results/wikipedia/full/pagelinks' -- all edges in the pagelink graph (must be *directed*)
+%default SUB_NODES         '/data/results/wikipedia/mini/nodes'     -- all nodes in the subuniverse
+%default SUB_PAGELINKS_OUT '/data/results/wikipedia/mini/pagelinks' -- where output will be stored
+
+all_pagelinks = LOAD '$PAGELINKS' AS (from_id:int, into_id:int, 
+  from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+
+sub_pagelinks_from = JOIN all_pagelinks BY from_id, sub_nodes BY node_id;
+sub_pagelinks = FOREACH sub_pagelinks_from GENERATE
+  all_pagelinks::from_id, all_pagelinks::into_id, 
+  all_pagelinks::from_namespace, all_pagelinks::from_title,  
+  all_pagelinks::into_namespace, all_pagelinks::into_title;
+STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig

@@ -0,0 +1,22 @@
+/*
+ * This script filters the pagelinks table, leaving only the pagelinks
+ * that terminate within supplied subuniverse.
+ * 
+ * Output format (same as augment_pagelinks):
+ * node_a:int, node_b:int, a_into_b:int, b_into_a:int, is_symmetric:int
+ */
+
+%default PAGELINKS         '/data/results/wikipedia/full/pagelinks' -- all edges in the pagelink graph (must be *directed*)
+%default SUB_NODES         '/data/results/wikipedia/mini/nodes'     -- all nodes in the subuniverse
+%default SUB_PAGELINKS_OUT '/data/results/wikipedia/mini/pagelinks' -- where output will be stored
+
+all_pagelinks = LOAD '$PAGELINKS' AS (from_id:int, into_id:int, 
+  from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+
+sub_pagelinks_into = JOIN all_pagelinks BY into_id, sub_nodes BY node_id;
+sub_pagelinks = FOREACH sub_pagelinks_into GENERATE
+  all_pagelinks::from_id, all_pagelinks::into_id, 
+  all_pagelinks::from_namespace, all_pagelinks::from_title,  
+  all_pagelinks::into_namespace, all_pagelinks::into_title;
+STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig

@@ -0,0 +1,26 @@
+/*
+ * This script filters the pagelinks table, leaving only the pagelinks
+ * that start and end within supplied subuniverse.
+ * 
+ * Output format (same as augment_pagelinks):
+ * from_id:int, into_id:int, from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray
+ */
+
+%default PAGELINKS         '/data/results/wikipedia/full/undirected_pagelinks' -- all edges in the pagelink graph
+%default SUB_NODES         '/data/results/wikipedia/mini/nodes'                -- all nodes in the subuniverse
+%default SUB_PAGELINKS_OUT '/data/results/wikipedia/mini/pagelinks'            -- where output will be stored
+
+all_pagelinks = LOAD '$PAGELINKS' AS (from_id:int, into_id:int, 
+  from_namespace:int, from_title:chararray,  into_namespace:int, into_title:chararray);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+
+sub_pagelinks_in = JOIN all_pagelinks BY from_id, sub_nodes BY node_id;
+sub_pagelinks_unfiltered = JOIN sub_pagelinks_in BY into_id, sub_nodes BY node_id;
+sub_pagelinks = FOREACH sub_pagelinks_unfiltered GENERATE
+  sub_pagelinks_in::all_pagelinks::from_id,
+  sub_pagelinks_in::all_pagelinks::into_id,
+  sub_pagelinks_in::all_pagelinks::from_namespace,
+  sub_pagelinks_in::all_pagelinks::from_title,
+  sub_pagelinks_in::all_pagelinks::into_namespace,
+  sub_pagelinks_in::all_pagelinks::into_title;
+STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_pageviews.pig

@@ -0,0 +1,29 @@
+/*
+ * This script filters the pageviews table, leaving only the pageviews
+ * in the specified subuniverse.
+ *
+ * Parameters:
+ * pageviews - all pageviews in the wikipedia corpus
+ * sub_nodes - the list of nodes in your subuniverse
+ * sub_pageviews_out - the directory where output will be stored
+ * 
+ * Output format (same as pageviews_augment.pig):
+ * id:int, namespace:int, 
+ * page_id:int, title:chararray, namespace:int, rev_date:int, rev_time:int, 
+ * rev_epoch_time:long, rev_dow:int, article_text:chararray
+ */
+
+%default PAGEVIEWS         '/data/results/wikipedia/full/pageviews' -- all pageview stats for the English Wikipedia
+%default SUB_NODES         '/data/results/wikipedia/mini/nodes'     -- all nodes in the subuniverse
+%default SUB_PAGEVIEWS_OUT '/data/results/wikipedia/mini/pageviews' -- where output will be stored
+
+pageviews = LOAD '$PAGEVIEWS' AS (page_id:int, title:chararray, namespace:int, 
+  rev_date:int, rev_time:int, rev_epoch_time:long, rev_dow:int, article_text:chararray);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+sub_pageviews_unfiltered = JOIN pageviews BY id, sub_nodes BY node_id;
+sub_pageviews = FOREACH sub_pageviews_unfiltered GENERATE
+  articles::page_id AS page_id, articles::title AS title, articles::namespace AS namespace,
+  articles::rev_date AS rev_date, articles::rev_time AS rev_time,
+  articles::rev_epoch_time AS rev_epoch_time, articles::rev_dow AS rev_dow,
+  articles::article_text AS article_text;
+STORE sub_pageviews INTO '$SUB_PAGEVIEWS_OUT';

data/examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig

@@ -0,0 +1,24 @@
+/*
+ * This script filters the pagelinks table, leaving only the pagelinks
+ * that start and end within supplied subuniverse.
+ * 
+ * Output format (same as undirected_pagelinks):
+ * node_a:int, node_b:int, a_into_b:int, b_into_a:int, is_symmetric:int
+ */
+
+%default UNDIRECTED_PAGELINKS  '/data/results/wikipedia/full/undirected_pagelinks' -- all edges in the pagelink graph
+%default SUB_NODES             '/data/results/wikipedia/mini/nodes'                -- all nodes in the subuniverse
+%default SUB_PAGELINKS_OUT     '/data/results/wikipedia/mini/pagelinks'            -- where output will be stored
+
+all_pagelinks = LOAD '$UNDIRECTED_PAGELINKS' AS (node_a:int, node_b:int, a_into_b:int, b_into_a:int, is_symmetric:int);
+sub_nodes = LOAD '$SUB_NODES' AS (node_id:int);
+
+sub_pagelinks_in = JOIN all_pagelinks BY node_a, sub_nodes BY node_id;
+sub_pagelinks_unfiltered = JOIN sub_pagelinks_in BY node_b, sub_nodes BY node_id;
+sub_pagelinks = FOREACH sub_pagelinks_unfiltered GENERATE
+  sub_pagelinks_in::all_pagelinks::node_a AS node_a, 
+  sub_pagelinks_in::all_pagelinks::node_b AS node_b,
+  sub_pagelinks_in::all_pagelinks::a_into_b AS a_into_b, 
+  sub_pagelinks_in::all_pagelinks::b_into_a AS b_into_a,
+  sub_pagelinks_in::all_pagelinks::is_symmetric AS is_symmetric;
+STORE sub_pagelinks INTO '$SUB_PAGELINKS_OUT';

data/examples/munging/wikipedia/utils/get_namespaces.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# encoding:UTF-8
+
+# A script that fetches the namespace -> id mapping for
+# all wikipedia languages. The output is stored (by default)
+# in a json file that represents a hash from namespace name => id
+
+require 'ruby-progressbar'
+require 'open-uri'
+require 'set'
+require 'configliere'
+require 'json'
+
+Settings.use :commandline
+
+NS_FILE = 'namespaces'
+
+Settings.define :out_dir, flag: 'o', description: "Directory to drop the namespace file into.", default: File.expand_path(File.dirname(__FILE__))
+Settings.define :verbose, flag: 'v', description: "Get chatty", type: :boolean, default: false
+Settings.define :silent, description: "Say nothing", type: :boolean, default: false
+Settings.define :head_length, flag: 'h', description: "The number of lines to read into the wiki xml for the namespace definitions.", type: Integer, default: 100
+Settings.define :std_out, flag: 's', description: "Print output to standard out.", type: :boolean, default: false
+Settings.define :to_tsv, flag: 't', description: 'Format the output as a TSV instead of JSON', type: :boolean, default:false
+
+Settings.resolve!
+
+Settings.out_dir = File.expand_path(Settings.out_dir)
+
+namespaces = {}
+namespaces_by_wiki = {}
+
+wikis_page = open('http://dumps.wikimedia.org/backup-index.html')
+wikis = Set.new
+
+# grap the list of wikis
+wikis_page.each_line do |line|
+  next unless line =~ />[a-z]*wiki</
+  wikis << line.gsub(/.*>([a-z]*)wiki<.*/,'\1')[0..-2]
+end
+
+if Settings.verbose
+  $stderr.puts "Retrieved the names of #{wikis.size} wikis"
+  $stderr.puts "Grabbing namespace data"
+elsif (not Settings.silent)
+  progressbar = ProgressBar.create(:title => "Retrieving Namespaces...", :total => wikis.size, :format => '%t |%B| %c/%C %e  ')
+end
+
+wikis.each_with_index do |prefix,index|
+  progressbar.increment unless (Settings.silent or Settings.verbose)
+  namespaces_by_wiki[prefix] = {}
+  $stderr.puts "Getting namespaces for #{prefix}.wikipedia.org" if Settings.verbose
+  raw = `curl -s 'http://dumps.wikimedia.org/#{prefix}wiki/latest/#{prefix}wiki-latest-pages-logging.xml.gz' | gzcat | head -n #{Settings.head_length}`
+  #TODO: Make this actually work
+  if $?.exitstatus != 0
+    out = "Could not access page dump for #{prefix}wiki." +
+          " This dump is probably being updated now." + 
+          " Namespaces for this wiki will not be included in the final output"
+    $stderr.puts out
+    next
+  end
+  raw.each_line do |line|
+    next unless line =~ /.*<\/?namespace[^>]*>/
+    match = /<\/?namespace key="(?<key>-?\d+)"[^>]*>(?<ns>[^<]*)<\/namespace>/.match(line)
+    next if match.nil?
+    namespaces[match[:ns]] = match[:key].to_i
+    namespaces_by_wiki[prefix][match[:ns]] = match[:key].to_i
+    $stderr.puts "    #{match[:ns]} -> #{match[:key]}" if Settings.verbose 
+  end
+  $stderr.puts "Finished getting namespaces for #{prefix}.wikipedia.org. #{wikis.size - index} wikis to go" if Settings.verbose
+end
+
+if Settings.to_tsv
+  output = ""
+  namespaces.each_pair do |k,v|
+    output += "#{k}\t#{v}\n"
+  end
+else
+  output = namespaces.to_json
+end
+
+if Settings.std_out
+  pp output
+else
+  filename = "#{Settings.out_dir}/#{NS_FILE}.#{Settings.to_tsv ? "tsv" : "json"}"
+  File.open(filename, 'w') { |f| f.write(output)}
+end

data/examples/munging/wikipedia/utils/munging_utils.rb

@@ -0,0 +1,68 @@
+# encoding:UTF-8
+
+require 'multi_json'
+
+module MungingUtils
+  extend self # you can call MungingUtils.foo, or include it and call on self.
+
+  # all non-keyboard characters (that is, characters outside the 0x20 to 0x127 range)
+  NON_PLAIN_ASCII_RE = /[^\x20-\x7e]/m
+  # characters below 0x20
+  CONTROL_CHARS_RE   = /[\x00-\x19]/m
+
+  def time_columns_from_time(time)
+    columns = []
+    columns << "%04d%02d%02d" % [time.year, time.month, time.day]
+    columns << "%02d%02d%02d" % [time.hour, time.min, time.sec]
+    columns << time.to_i
+    columns << time.wday
+    return columns
+  end
+
+  def warn_record(desc, record=nil)
+    record_info = MultiJson.encode(record)[0..1000] rescue "(unencodeable record) #{record.inspect[0..100]}"
+    Log.warn [desc, record_info].join("\t")
+    nil
+  end
+
+  # Modifies the text in place, replacing all newlines, tabs, and other control
+  # characters with a space (those < ascii 0x20, but not including 0xff). This
+  # uses a whitelist
+  #
+  # Only use this if funny characters aren't suppose to be in there in the first
+  # place; there are safe, easy ways to properly encode, eg `MultiJson.encode()`
+  #
+  def scrub_control_chars(text)
+    text.gsub!(CONTROL_CHARS_RE, ' ')
+    text
+  end
+
+  # Modifies the text in place, replacing all non-keyboard characters (newline,
+  # tab, anything not between ascii 0x20 and 0x7e) with their XML entity encoding
+  def safe_xml_encode(text)
+    text.gsub!(NON_PLAIN_ASCII_RE){|ch|  "\\u%04x" % ch.ord } unless jsonized.ascii_only?
+    text
+  end
+
+
+  # Returns a JSON encoded string, with all non-ASCII characters escaped
+  def safe_json_encode(string)
+    jsonized = MultiJson.encode(string)
+    jsonized.gsub!(NON_PLAIN_ASCII_RE){|ch| "\\u%04x" % ch.ord } unless jsonized.ascii_only?
+    jsonized
+  end
+
+
+end
+
+Time.class_eval do
+  def to_flat
+    utc.strftime("%Y%m%d%H%M%SZ")
+  end
+end
+
+MatchData.class_eval do
+  def as_hash
+    Hash[ names.map{|name| [name.to_sym, self[name]] } ]
+  end
+end