imw 0.1.0 → 0.1.1

data/lib/imw/dataset/scrub/slug.rb

@@ -1,101 +0,0 @@
-require 'rubygems'
-require 'addressable/uri'
-require 'uuidtools'
-require 'scrub'
-require 'scrub_simple_url'
-
-module IMW
-
-  #
-  #
-  # +handle+ -- reasonable effort at a uniq-ish, but human-comprehensible string
-  # Handle should only contain the characters A-Za-z0-9_-./
-  #
-  #
-  class Slug
-    # A humane representation of the handle ('that-one-time-at_foo')
-    attr_reader :handle
-    # The purportedly unique string ('')
-    attr_accessor :uniqish
-
-    def initialize handle
-      self.handle = handle
-      self.uniqish  = handle
-    end
-
-    #
-    # Unless overridden, use the uniqish to
-    # make a name-based UUID within the infochimps.org
-    # namespace
-    #
-    def uuid
-      UUID.sha1_create(UUID_URL_NAMESPACE, full_handle)
-    end
-
-    # Handle with only \w characters -- safe for everything there be
-    def url_sane
-      return '' if !handle
-      handle.gsub(/[^\w\/\:]+/, '-').gsub(/_/, '__').gsub(%r{[/:]+}, '_')
-    end
-
-    def handle= t
-      @handle = self.class.sanitize_handle(t)
-    end
-
-    # Strip all but handle-safe characters
-    def self.sanitize_handle t, turd='-'
-      t = t.gsub(%r{[^\w\-\./]+}, turd)
-    end
-  end
-
-  #
-  # Uses a URL (that's locator, not URI) as a
-  # presumed-uniq identifier.
-  #
-  # +uniqish+ returns the full normalized URL
-  #
-  # +handle+ is formed from the dot-reversed host, the scheme (if not http) and a
-  # sanitized version of the path. (The query string, fragment, etc are stripped
-  # from the handle)
-  #
-  #
-  class URLSlug < Slug
-    attr_accessor :url
-    def initialize url_str
-      self.url     = Addressable::URI.heuristic_parse(url_str).normalize
-      raise "Bad URL #{url}" unless url.host
-      self.uniqish = url.to_s
-      self.handle   = munge_url
-    end
-
-    def uuid
-      UUID.sha1_create(UUID_URL_NAMESPACE, full_handle)
-    end
-  end
-end
-
-
-
-module Sluggable
-  protected
-  def create_slug
-    "Slugging #{self.attributes}"
-    if (self.class.slug_on == :url) || (self.name.blank?)
-      slug = IMW::URLSlug.new(self.url)
-      self.name = slug.handle
-    else
-      slug = IMW::Slug.new(self.name)
-    end
-    self.handle ||= slug.handle
-  end
-  public
-
-  def self.included base
-    base.before :save, :create_slug
-    base.class_eval do
-      def self.slug_on s=nil
-        @slug_on ||= s
-      end
-    end
-  end
-end

data/lib/imw/dataset/stats/counter.rb

@@ -1,23 +0,0 @@
-module IMW
-  class RecordCounter < Hash
-    def record val
-      self[val] ||= 0
-      self[val]  += 1
-    end
-
-    def if_seen val, &block
-      if self[val]
-        yield
-      end
-      record val
-    end
-
-    def unless_seen val, &block
-      unless self[val]
-        yield
-      end
-      record val
-    end
-
-  end
-end

data/lib/imw/dataset/stats.rb

@@ -1,73 +0,0 @@
-#
-# h2. lib/imw/dataset/stats.rb -- statistics for datasets
-#
-# == About
-#
-# Implements methods to calculate very basic statistical properties of
-# a dataset.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
-
-
-module IMW
-  class Dataset
-    #
-    # simple histogram
-    #
-    # Runs down one column/attribute of a dataset
-    # returning counts for that column
-    #
-    def hist slicer
-      counts = { }
-      els = slice(slicer)
-      els.each do |el|
-        counts[el] ||= 0
-        counts[el]  += 1
-      end
-      self.class.new(counts.map{ |el,ct| [ct,el] })
-    end
-
-    def slice slicer
-      case
-      when slicer.respond_to?(:call) then self.map{ |row| slicer.call(row) }
-      else
-        self.map{ |row| row[slicer] }
-      end
-    end
-
-    #
-    # Report
-    #
-    def report slicer, opts={}
-      opts.reverse_merge! :n_top => 20, :hist_args => [], :fmt => "%7d\t%s", :do_hist => true
-      counts = hist(slicer)
-      report_hist  data, counts, slicer, opts if opts[:do_hist]
-      report_sizes data, counts, slicer, opts
-    end
-
-    def report_sizes data, counts, slicer, opts={}
-      fmt  = opts[:fmt]
-      puts fmt % [counts.length,              "unique elements"]
-      puts fmt % [data.length,                "total elements"]
-      puts fmt % [counts.find_all(&:nil?).length, "nil elements"]
-      uniqvals = counts.map{|ct,el| el}.reject(&:nil?)
-      puts " min:\t#{uniqvals.min}"
-      puts " max:\t#{uniqvals.max}"
-    end
-
-    # Most popular
-    def report_hist data, counts, slicer, opts={}
-      top = counts.sort_by{|ct,el| ct}[-opts[:n_top]..-1]
-      puts "Top #{opts[:n_top]} elements for slice through #{slicer}:"
-      puts " -freq-\t-element-"
-      puts top.map{ |ct,el| opts[:fmt] % [ct,el] }
-      puts "-------\t-------"
-    end
-
-  end
-end