wukong 3.0.0.pre → 3.0.0.pre2

data/old/examples/sample_records.rb

@@ -1,33 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-
-Settings.define :sampling_fraction, :type => Float, :required => true, :description => "floating-point number between 0 and 1 giving the fraction of lines to emit: at sampling_fraction=1 all records are emitted, at 0 none are."
-
-#
-# Probabilistically emit some fraction of record/lines
-#
-# Set the sampling fraction at the command line using the
-#   --sampling_fraction=
-# option: for example, to take a random 1/1000th of the lines in huge_files,
-#  ./examples/sample_records.rb --sampling_fraction=0.001 --run huge_files sampled_files
-#
-class Mapper < Wukong::Streamer::LineStreamer
-  include Wukong::Streamer::Filter
-
-  #
-  # randomly decide to emit +sampling_fraction+ fraction of lines
-  #
-  def emit? line
-    rand < Settings.sampling_fraction
-  end
-end
-
-#
-# Executes the script
-#
-Wukong.run( Mapper,
-  nil,
-  :reduce_tasks => 0,
-  :reuse_jvms   => true
-  )

data/old/examples/server_logs/apache_log_parser.rb

@@ -1,15 +0,0 @@
-#!/usr/bin/env ruby -E ASCII-8BIT
-require 'rubygems'
-require 'wukong/script'
-$: << File.dirname(__FILE__)
-require 'logline'
-
-class ApacheLogParser < Wukong::Streamer::LineStreamer
-
-  # create a Logline object from each record and serialize it flat to disk
-  def process line
-    yield Logline.parse(line)
-  end
-end
-
-Wukong.run( ApacheLogParser, nil, :sort_fields => 7 ) if $0 == __FILE__

data/old/examples/server_logs/nook.rb

@@ -1,48 +0,0 @@
-#!/usr/bin/env ruby -E BINARY
-require 'rubygems'
-require 'faraday'
-require 'wukong/script'
-require 'json'
-$: << File.dirname(__FILE__)
-require 'apache_log_parser'
-require 'nook/faraday_dummy_adapter'
-
-Settings.define :target_host,   :default => 'localhost', :description => "The host name or IP address to target"
-Settings.define :target_scheme, :default => 'http',      :description => "Request scheme (http, https)"
-
-#
-# A Nook consumes its input stream and, for each input, generates an HTTP
-# request against a remote host. Please use it for good and never for evil.
-#
-# You can use it from your command line:
-#   zcat /var/www/app/current/log/*access*.log.gz | ./nook.rb --map --host=http://my_own_host.com
-#
-#
-class NookMapper < ApacheLogParser
-  # create a Logline object from each record and serialize it flat to disk
-  def process line
-    super(line) do |logline|
-      start = Time.now
-      resp = fetcher.get(logline.path, :user_agent => logline.ua, :referer => logline.referer)
-      yield [Time.now.to_flat, (Time.now - start).to_f, resp.status, resp.body.size, logline.path, resp.body]
-    end
-  end
-
-  def track record
-    monitor.periodically do |m|
-      m.progress
-    end
-  end
-
-  # a mock fetcher with a uniformly distributed variable delay
-  def fetcher
-    @fetcher ||= Faraday::Connection.new(:url => 'http://localhost:80/') do |f|
-      f.use Faraday::Adapter::Dummy do |dummy|
-        dummy.delay = Proc.new{|env| 0.05  } # 0.2 * rand()
-        # dummy.body = Proc.new{|env| env[:url] }
-      end
-    end
-  end
-end
-
-Wukong.run( NookMapper, nil, :sort_fields => 7 )

data/old/examples/server_logs/nook/faraday_dummy_adapter.rb

@@ -1,94 +0,0 @@
-
-module Faraday
-  class Adapter
-
-    # test = Faraday::Connection.new do |f|
-    #   f.use Faraday::Adapter::Dummy do |dummy|
-    #     dummy.status 404
-    #     dummy.delay  1
-    #   end
-    # end
-    #
-    # # this will delay 0.2s, returning 404 with
-    # resp = text.get("/your/mom", :dummy_delay => 0.2)
-    # resp.body # => {"method":"get","url":"/your/mom","request_headers":{"Dummy-Delay":"0.2","dummy_delay":0.2},"request":{"proxy":null},"ssl":{}}
-    #
-    # More example:
-    #
-    # test = Faraday::Connection.new do |f|
-    #   f.use Faraday::Adapter::Dummy, :status => 503
-    # end
-    #
-    # test = Faraday::Connection.new do |f|
-    #   f.use Faraday::Adapter::Dummy do |dummy|
-    #     dummy.delay = Proc.new{|env| 0.1 + 0.8 * rand() }
-    #   end
-    # end
-    #
-    class Dummy < Middleware
-      include Addressable
-      attr_reader :config
-      def self.loaded?() false end
-
-      # gets value from environment if set, configured instance variable otherwise
-      def value_for env, key
-        val = env[:request_headers]["Dummy-#{header_hash_key(key)}"] || config[key]
-        if val.respond_to?(:call)
-          val = val.call(env)
-        end
-        val
-      end
-
-      # With an optional delay, constructs a [status, headers, response] based on the first of:
-      # * request header field (Dummy-Status, Dummy-Headers, Dummy-Resonse)
-      # * adapter's configuration:
-      # * Unless one of the above is set, body will return a json string taken from the request hash
-      #
-      def call(env)
-        status  = value_for(env, :status)
-        headers = value_for(env, :headers)
-        headers = JSON.load(headers) if headers.is_a? String
-        body    = value_for(env, :body) ||
-          env.dup.tap{|hsh| [:response, :parallel_manager, :body].each{|k| hsh.delete k} }.to_json
-        delay   = value_for(env, :delay).to_f
-        sleep delay if delay > 0
-        headers[:dummy_delay] = delay
-        env.update(
-          :status           => status,
-          :response_headers => headers,
-          :body             => body)
-        @app.call(env)
-      end
-
-      class Configurator < Struct.new(:status, :headers, :delay, :body)
-        def status(val=nil)  self.status  = val if val ; super() end
-        def headers(val=nil) self.headers = val if val ; super() end
-        def body(val=nil)    self.body    = val if val ; super() end
-        def delay(val=nil)   self.delay   = val if val ; super() end
-        def self.from_hash hsh
-          new().tap{|config| hsh.each{|k,v| config.send("#{k}=", v) } }
-        end
-      end
-
-      def initialize(app, defaults={}, &block)
-        super(app)
-        @config = Configurator.from_hash(defaults.reverse_merge(:status => 200, :delay => 0, :headers => {}))
-        configure(&block) if block
-      end
-
-      def configure
-        yield config
-      end
-
-      # same as in Faraday::Utils -- turns :dummy_response_status into 'Dummy-Response-Status'
-      def header_hash_key(str)
-        str.to_s.split('_').each{|w| w.capitalize! }.join('-')
-      end
-
-      def create_multipart(env, params, boundary = nil)
-        stream = super
-        stream.read
-      end
-    end
-  end
-end

data/old/examples/server_logs/user_agent.rb

@@ -1,40 +0,0 @@
-
-
-# For later, if we want to parse user agents:
-#   http://code.google.com/p/browserscope/source/browse/trunk/models/user_agent.py
-#   http://www.useragentstring.com/pages/All/
-#   http://github.com/jaxn/parse-user-agent
-#   http://code.google.com/p/browserscope/wiki/UserAgentParsing
-#   http://code.google.com/p/ua-parser/source/browse/
-#   http://github.com/shenoudab/active_device/tree/master/lib/active_device/
-
-
-#
-# * Mozilla based
-# * Mozilla version
-# * X11 based
-# * Security
-# * OS
-# * CPU family
-# * Language Tag
-# * Renderer (i.e. Webkit, Trident, Presto)
-# * Renderer Version
-# * I don't see a utility for the "KHTML" and "like Gecko" bits, but whatever.
-# * Based on
-# * Browser Build (not really sure about this either)
-
-# * Browser Family (i.e. Firefox, IE, Chrome, etc..)
-# * Project Name (optional, i.e. Namoroka, Shiretoko)
-# * Major Version
-# * Minor Version
-# * Version Third Bit
-# * Version Fourth Bit
-# * Open Question: How should we handle the "alpha/beta" bit, like apre1? I'm inclined to say we put it in its own datapoint and let people group together how ever they want, but not leave it attached to any of the version bits.
-
-# Bot
-# Brand
-# Browser
-# Engine
-# Handset
-# Model
-# OS

data/old/examples/simple_word_count.rb

@@ -1,82 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-
-module WordCount
-  class Mapper < Wukong::Streamer::LineStreamer
-    #
-    # Emit each word in each line.
-    #
-    def process line
-      tokenize(line).each{|word| yield [word, 1] }
-    end
-
-    #
-    # Split a string into its constituent words.
-    #
-    # This is pretty simpleminded:
-    # * downcase the word
-    # * Split at any non-alphanumeric boundary, including '_'
-    # * However, preserve the special cases of 's, 'd or 't at the end of a
-    #   word.
-    #
-    #   tokenize("Ability is a poor man's wealth #johnwoodenquote")
-    #   # => ["ability", "is", "a", "poor", "man's", "wealth", "johnwoodenquote"]
-    #
-    def tokenize str
-      return [] if str.blank?
-      str = str.downcase;
-      # kill off all punctuation except [stuff]'s or [stuff]'t
-      # this includes hyphens (words are split)
-      str = str.
-        gsub(/[^a-zA-Z0-9\']+/, ' ').
-        gsub(/(\w)\'([std])\b/, '\1!\2').gsub(/\'/, ' ').gsub(/!/, "'")
-      # Busticate at whitespace
-      words = str.split(/\s+/)
-      words.reject!{|w| w.blank? }
-      words
-    end
-  end
-
-  #
-  # A bit kinder to your memory manager: accumulate the sum record-by-record:
-  #
-  class Reducer2 < Wukong::Streamer::AccumulatingReducer
-    
-    def start!(*args)
-      @key_count =  0
-    end
-    
-    def accumulate(*args)
-      @key_count += 1
-    end
-    
-    def finalize
-      yield [ key, @key_count ]
-    end
-  end
-
-  #
-  # You can stack up all the values in a list then sum them at once.
-  #
-  # This isn't good style, as it means the whole list is held in memory
-  #
-  class Reducer1 < Wukong::Streamer::ListReducer
-    def finalize
-      yield [ key, values.map(&:last).map(&:to_i).inject(0){|x,tot| x+tot } ]
-    end
-  end
-
-  #
-  # ... easiest of all, though: this is common enough that it's already included
-  #
-  require 'wukong/streamer/count_keys'
-  class Reducer3 < Wukong::Streamer::CountKeys
-  end
-end
-
-# Execute the script
-Wukong.run(
-  WordCount::Mapper,
-  WordCount::Reducer2
-  )

data/old/examples/size.rb

@@ -1,61 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-
-module Size
-  #
-  # Feed the entire dataset through wc and sum the results
-  #
-  class Script < Wukong::Script
-    #
-    # Don't implement a wukong script to do something if there's a unix command
-    # that does it faster: just override map_command or reduce_command in your
-    # subclass of Wukong::Script to return the complete command line
-    #
-    def map_command
-      '/usr/bin/wc'
-    end
-
-    # Make all records go to one reducer
-    def default_options
-      super.merge :reduce_tasks => 1
-    end
-  end
-
-  #
-  # Sums the numeric value of each column in its input
-  #
-  class Reducer < Wukong::Streamer::Base
-    attr_accessor :sums
-
-    #
-    # The unix +wc+ command uses whitespace, not tabs, so we'll recordize
-    # accordingly.
-    #
-    def recordize line
-      line.strip.split(/\s+/)
-    end
-
-    #
-    # add each corresponding column in the input
-    #
-    def process *vals
-      self.sums = vals.zip( sums || [] ).map{|val,sum| val.to_i + sum.to_i }
-    end
-
-    #
-    # run through the whole reduction input and then output the total
-    #
-    def stream *args
-      super *args
-      emit sums
-    end
-  end
-end
-
-# Execute the script
-Size::Script.new(
-  nil,
-  Size::Reducer,
-  :reduce_tasks => 1
-  ).run

data/old/examples/stats/avg_value_frequency.rb

@@ -1,86 +0,0 @@
-#!/usr/bin/env ruby
-# run like so:
-# $> ruby average_value_frequecy.rb --run=local data/stats.tsv data/avf_out.tsv
-require 'rubygems'
-require 'wukong'
-
-#
-# Calculate the average value frequency (AVF) for each data row. AVF for a data
-# point with m attributes is defined as:
-#
-#     avf = (1/m)* sum (frequencies of attributes 1..m)
-#
-# so with the data
-#
-#      1       15      30      25
-#      2       10      10      20
-#      3       50      30      30
-#
-# for the first row, avf = (1/3)*(1+2+1) ~= 1.33. An outlier is identified by
-# a low AVF.
-#
-module AverageValueFrequency
-  # Names for each column's attribute, in order
-  ATTR_NAMES = %w[length width height]
-
-  class HistogramMapper < Wukong::Streamer::RecordStreamer
-    # unroll each row from
-    #     [id,   val1,   val2, ....]
-    # into
-    #     [attr1,   val1]
-    #     [attr2,   val2]
-    #     ...
-    def process id, *values
-      ATTR_NAMES.zip(values).each do |attr, val|
-        yield [attr, val]
-      end
-    end
-  end
-
-  #
-  # Build a histogram of values
-  #
-  class HistogramReducer < Wukong::Streamer::CountingReducer
-    # use the attr and val as the key
-    def get_key attr, val=nil, *_
-      [attr, val]
-    end
-  end
-
-  class AvfRecordMapper < Wukong::Streamer::RecordStreamer
-    # average the frequency of each value
-    def process id, *values
-      sum = 0.0
-      ATTR_NAMES.zip(values).each do |attr, val|
-        sum += histogram[ [attr, val] ].to_i
-      end
-      avf = sum / ATTR_NAMES.length.to_f
-      yield [id, avf, *values]
-    end
-
-    # Load the histogram from a tab-separated file with
-    #   attr    val   freq
-    def histogram
-      return @histogram if @histogram
-      @histogram = { }
-      File.open(options[:histogram_file]).each do |line|
-        attr, val, freq = line.chomp.split("\t")
-        @histogram[ [attr, val] ] = freq
-      end
-      @histogram
-    end
-  end
-end
-
-Settings.use :commandline, :define
-Settings.define :histogram,      :description => "Run the first pass to calculate a histogram"
-Settings.define :avf,            :description => "Run the second pass, to run back over the records with the histogram and find the AVF for each row."
-Settings.define :histogram_file, :description => "File to load the histogram from (supply name of the  output file from first pass)"
-Settings.resolve!
-if Settings[:histogram]
-  Wukong::Script.new(AverageValueFrequency::HistogramMapper, AverageValueFrequency::HistogramReducer).run
-elsif Settings[:avf]
-  Wukong::Script.new(AverageValueFrequency::AvfRecordMapper, nil).run
-else
-  raise "Please specify either --histogram (for first round) or --avf (second round)"
-end