wukong 3.0.0.pre → 3.0.0.pre2

data/old/examples/contrib/jeans/data/sizes

@@ -1,3 +0,0 @@
-australia	253	499	671	663	710	687	774	654	627	422	376	132	25
-spain	37	102	257	177	118	90	144	183	210	222	162	93	17
-sweden	32	167	306	334	314	287	330	366	415	343	266	130	51

data/old/examples/contrib/jeans/normalize.rb

@@ -1,20 +0,0 @@
-#!/usr/bin/env ruby
-# run like so:
-# $> ruby normalize.rb --run=local data/sizes.tsv data/normalized_sizes.tsv
-require 'rubygems'
-require 'wukong'
-require 'active_support/core_ext/enumerable' # for array#sum
-
-module Normalize
-  class Mapper < Wukong::Streamer::RecordStreamer
-    def process(country, *sizes)
-      sizes.map!(&:to_i)
-      sum = sizes.sum.to_f
-      normalized = sizes.map{|x| 100 * x/sum }
-      s = normalized.join(",")
-      yield [country, s]
-    end
-  end
-end
-
-Wukong::Script.new(Normalize::Mapper, nil).run

data/old/examples/contrib/jeans/sizes.rb

@@ -1,55 +0,0 @@
-#!/usr/bin/env ruby
-# run like so:
-# $> ruby sizes.rb --run=local data/orders.tsv data/sizes
-require 'rubygems'
-require 'wukong'
-
-module JeanSizes
-  class Mapper < Wukong::Streamer::RecordStreamer
-    def process(code,model,time,country,reg,col, n1,c1, venue,n3,n4, *sizes)
-      yield [country, *sizes]
-    end
-  end
-
-  #
-  # This uses a ListReducer. It's nice and simple, but requires first
-  # accumulating each key's records in memory.
-  #
-  class JeansListReducer < Wukong::Streamer::ListReducer
-    def finalize
-      return if values.empty?
-      sums = []; 13.times{ sums << 0 }
-      values.each do |country, *sizes|
-        sizes.map!(&:to_i)
-        sums = sums.zip(sizes).map{|sum, val| sum + val }
-      end
-      yield [key, *sums]
-    end
-  end
-
-
-  #
-  # This uses an AccumulatingReducer directly.
-  # It has the advantage of a minimal footprint.
-  #
-  class JeansAccumulatingReducer < Wukong::Streamer::AccumulatingReducer
-    attr_accessor :sums
-
-    # start the sum with 0 for each size
-    def start! *_
-      self.sums = []; 13.times{ self.sums << 0 }
-    end
-    # accumulate each size count into the sizes_sum
-    def accumulate country, *sizes
-      sizes.map!(&:to_i)
-      self.sums = self.sums.zip(sizes).map{|sum, val| sum + val }
-    end
-    # emit [country, size_0_sum, size_1_sum, ...]
-    def finalize
-      yield [key, sums].flatten
-    end
-  end
-
-end
-
-Wukong::Script.new(JeanSizes::Mapper, JeanSizes::JeansListReducer).run

data/old/examples/corpus/bnc_word_freq.rb

@@ -1,44 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)
-require 'rubygems'
-require 'wukong/script'
-
-Settings.define :ripd_root, :default => '/data/chimpmark/ripd'
-BNC_SOURCE_FILE='ucrel.lancs.ac.uk/bncfreq/lists/1_1_all_fullalpha.txt'
-
-# File 1_1_all_fullalpha.txt -- 794771 lines
-#
-# cat /data/chimpmark/ripd/ucrel.lancs.ac.uk/bncfreq/lists/1_1_all_fullalpha.txt | ./bnc_word_freq.rb --map | sort -nk3 > /data/chimpmark/rawd/bnc_word_freq/bnc_word_freq.tsv
-
-class BncParser < Wukong::Streamer::RecordStreamer
-  def before_stream
-    @head_word, @part_of_speech, @head_word_stats = ["","",[]]
-    $stdin.readline
-    $stdin.readline
-  end
-
-  def process _, word, pos, variant, freq_ppm, range, dispersion
-    word_stats = [freq_ppm, range, dispersion]
-
-    unless word == "@"                # lemma for a different head word
-      @head_word       = word
-      @part_of_speech  = pos
-      @head_word_stats = word_stats
-    end
-
-    weirdness = (@head_word =~ /[^a-zA-Z]/)
-
-    if    variant == '%'  # head word with lemmas
-      word_stats = ['','','']
-    elsif variant == ':'  # head word with no lemmas
-      variant = word
-    else
-      weirdness = weirdness || (variant =~ /[^a-zA-Z]/)
-    end
-    yield [@head_word, @part_of_speech, @head_word_stats, variant, word_stats, (weirdness ? 1 : 0)].flatten.join("\t")
-  end
-end
-
-Wukong.run(
-  BncParser, nil
-  )

data/old/examples/corpus/bucket_counter.rb

@@ -1,47 +0,0 @@
-
-class BucketCounter
-  BUCKET_SIZE = 2**24
-  attr_reader :total
-
-  def initialize
-    @hsh = Hash.new{|h,k| h[k] = 0 }
-    @total = 0
-  end
-
-  # def [] val
-  #   @hsh[val]
-  # end
-  # def << val
-  #   @hsh[val] += 1; @total += 1 ; self
-  # end
-
-  def [] val
-    @hsh[val.hash % BUCKET_SIZE]
-  end
-  def << val
-    @hsh[val.hash % BUCKET_SIZE] += 1; @total += 1 ; self
-  end
-
-  def insert *words
-    words.flatten.each{|word| self << word }
-  end
-  def clear
-    @hsh.clear
-    @total = 0
-  end
-
-  def stats
-    { :total => total,
-      :size  => size,
-    }
-  end
-  def size() @hsh.size end
-
-  def full?
-    size.to_f / BUCKET_SIZE > 0.5
-  end
-
-  def each *args, &block
-    @hsh.each(*args, &block)
-  end
-end

data/old/examples/corpus/dbpedia_abstract_to_sentences.rb

@@ -1,86 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-
-#
-# Use the stanford NLP parse to split a piece of text into sentences
-#
-# @example
-#   SentenceParser.split("Beware the Jabberwock, my son! The jaws that bite, the claws that catch! Beware the Jubjub bird, and shun The frumious Bandersnatch!")
-#   # => [["Beware", "the", "Jabberwock", ",", "my", "son", "!"], ["The", "jaws", "that", "bite", ",", "the", "claws", "that", "catch", "!"], ["Beware", "the", "Jubjub", "bird", ",", "and", "shun", "The", "frumious", "Bandersnatch", "!"]]
-#
-class SentenceParser
-  def self.processor
-    return @processor if @processor
-    require 'rubygems'
-    require 'stanfordparser'
-    @processor = StanfordParser::DocumentPreprocessor.new
-  end
-
-  def self.split line
-    processor.getSentencesFromString(line).map{|s| s.map{|w| w.to_s } }
-  end
-end
-
-#
-# takes one document per line
-# splits into sentences
-#
-class WordNGrams < Wukong::Streamer::LineStreamer
-  def recordize line
-    line.strip!
-    line.gsub!(%r{^<http://dbpedia.org/resource/([^>]+)> <[^>]+> \"}, '') ; title = $1
-    line.gsub!(%r{\"@en \.},'')
-    [title, SentenceParser.split(line)]
-  end
-
-  def process title, sentences
-    sentences.each_with_index do |words, idx|
-      yield [title, idx, words].flatten
-    end
-  end
-end
-
-Wukong.run WordNGrams, nil, :partition_fields => 1, :sort_fields => 2
-
-# ---------------------------------------------------------------------------
-#
-# Run Time:
-#
-#   Job Name: dbpedia_abstract_to_sentences.rb---/data/rawd/encyc/dbpedia/dbpedia_dumps/short_abstracts_en.nt---/data/rawd/encyc/dbpedia/dbpedia_parsed/short_abstract_sentences
-#   Status: Succeeded
-#   Started at: Fri Jan 28 03:14:45 UTC 2011
-#   Finished in: 41mins, 50sec
-#   3 machines: master m1.xlarge, 2 c1.xlarge workers; was having some over-memory issues on the c1.xls
-#
-#                                     Counter      Reduce       Total
-#   SLOTS_MILLIS_MAPS                       0              10 126 566
-#   Launched map tasks                      0                      15
-#   Data-local map tasks                    0                      15
-#   SLOTS_MILLIS_REDUCES                    0                   1 217
-#   HDFS_BYTES_READ             1 327 116 133           1 327 116 133
-#   HDFS_BYTES_WRITTEN          1 229 841 020           1 229 841 020
-#   Map input records               3 261 096               3 261 096
-#   Spilled Records                         0                       0
-#   Map input bytes             1 326 524 800           1 326 524 800
-#   SPLIT_RAW_BYTES                     1 500                   1 500
-#   Map output records              9 026 343               9 026 343
-#
-#   Job Name: dbpedia_abstract_to_sentences.rb---/data/rawd/encyc/dbpedia/dbpedia_dumps/long_abstracts_en.nt---/data/rawd/encyc/dbpedia/dbpedia_parsed/long_abstract_sentences
-#   Status: Succeeded
-#   Started at: Fri Jan 28 03:23:08 UTC 2011
-#   Finished in: 41mins, 11sec
-#   3 machines: master m1.xlarge, 2 c1.xlarge workers; was having some over-memory issues on the c1.xls
-#
-#                                     Counter      Reduce       Total
-#   SLOTS_MILLIS_MAPS                       0              19 872 357
-#   Launched map tasks                      0                      29
-#   Data-local map tasks                    0                      29
-#   SLOTS_MILLIS_REDUCES                    0                   5 504
-#   HDFS_BYTES_READ             2 175 900 769           2 175 900 769
-#   HDFS_BYTES_WRITTEN          2 280 332 736           2 280 332 736
-#   Map input records               3 261 096               3 261 096
-#   Spilled Records                         0                       0
-#   Map input bytes             2 174 849 644           2 174 849 644
-#   SPLIT_RAW_BYTES                     2 533                    2533
-#   Map output records             15 425 467              15 425 467

data/old/examples/corpus/sentence_bigrams.rb

@@ -1,53 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)
-require 'rubygems'
-require 'wukong/script'
-require 'bucket_counter'
-
-#
-# Coocurrence counts
-#
-
-#
-# Input is a list of document-idx-sentences, each field is tab-separated
-#   title   idx   word_a    word_b    word_c ...
-#
-# This emits each co-courring pair exactly once; in the case of a three-word
-# sentence the output would be
-#
-#   word_a  word_b
-#   word_a  word_c
-#   word_b  word_c
-#
-class SentenceBigrams < Wukong::Streamer::RecordStreamer
-  def process title, idx, *words
-    words[0..-2].zip(words[1..-1]).each do |word_a, word_b|
-      yield [word_a, word_b]
-    end
-  end
-end
-
-#
-# Combine multiple bucket counts into a single on
-#
-class CombineBuckets < Wukong::Streamer::AccumulatingReducer
-  def get_key *fields
-    fields[0..1]
-  end
-  def start! *args
-    @total = 0
-  end
-  def accumulate *fields
-    @total += 1
-  end
-  def finalize
-    yield [@total, key].flatten
-  end
-end
-
-Wukong.run(
-  SentenceBigrams,
-  CombineBuckets,
-  :io_sort_record_percent => 0.3,
-  :io_sort_mb => 300
-  )

data/old/examples/corpus/sentence_coocurrence.rb

@@ -1,66 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)
-require 'rubygems'
-require 'wukong/script'
-require 'bucket_counter'
-
-#
-# Coocurrence counts
-#
-
-#
-# Input is a list of document-idx-sentences, each field is tab-separated
-#   title   idx   word_a    word_b    word_c ...
-#
-# This emits each co-courring pair exactly once; in the case of a three-word
-# sentence the output would be
-#
-#   word_a  word_b
-#   word_a  word_c
-#   word_b  word_c
-#
-class SentenceCoocurrence < Wukong::Streamer::RecordStreamer
-  def initialize *args
-    super *args
-    @bucket = BucketCounter.new
-  end
-
-  def process title, idx, *words
-    @bucket << words[0..-2].zip(words[1..-1])
-    dump_bucket if @bucket.full?
-  end
-
-  def dump_bucket
-    @bucket.each do |pair_key, count|
-      emit [pair_key, count]
-    end
-    $stderr.puts "bucket stats: #{@bucket.stats.inspect}"
-    @bucket.clear
-  end
-
-  def after_stream
-    dump_bucket
-  end
-end
-
-#
-# Combine multiple bucket counts into a single on
-#
-class CombineBuckets < Wukong::Streamer::AccumulatingReducer
-  def start! *args
-    @total = 0
-  end
-  def accumulate word, count
-    @total += count.to_i
-  end
-  def finalize
-    yield [@total, key] if @total > 20
-  end
-end
-
-Wukong.run(
-  SentenceCoocurrence,
-  CombineBuckets,
-  :io_sort_record_percent => 0.3,
-  :io_sort_mb => 300
-  )

data/old/examples/corpus/stopwords.rb

@@ -1,138 +0,0 @@
-STOPWORDS_3 = %w[
-    the
-    of
-    and
-    a
-    in
-    to
-    it
-    is
-    was
-    I
-    for
-    that
-    you
-    he
-    be
-    with
-    on
-    by
-    at
-    have
-    are
-    not
-    this
-    but
-    had
-    they
-    his
-    from
-    she
-    which
-    or
-    we
-    an
-    were
-    as
-    do
-    been
-    their
-    has
-    would
-    there
-    what
-    will
-    all
-    if
-    can
-    her
-    said
-    who
-    one
-    so
-    up
-    them
-    when
-    some
-    could
-    him
-    into
-    its
-    then
-    two
-    out
-    time
-    my
-    about
-    did
-    your
-    now
-    me
-    other
-    only
-    just
-    more
-    these
-    also
-    any
-    see
-    very
-    may
-    well
-    should
-    than
-    how
-    get
-    way
-    our
-    made
-    got
-    after
-    many
-    those
-    go
-    being
-    because
-    down
-    such
-    through
-    over
-    must
-    still
-    even
-    take
-    too
-    here
-    come
-    own
-    last
-    does
-    oh
-    say
-    no
-    where
-    us
-    same
-    might
-    yes
-    however
-    put
-    world
-    another
-    want
-    most
-    again
-    never
-    under
-    much
-    why
-    each
-    while
-    off
-    went
-    used
-    without
-    give
-    within
-  ]
-RE_STOPWORDS_15 = '(?:'+STOPWORDS_3[0..15].join("|")+')'