imw 0.1.0 → 0.1.1

data/lib/imw/dataset/scraped_uri.rb

@@ -1,305 +0,0 @@
-# -*- coding: utf-8 -*-
-module Linkish
-  def self.included base
-    base.class_eval do
-      include DataMapper::Resource
-      include Infochimps::Resource
-      property      :id,              Integer,        :serial      => true
-      property      :full_url,        String,         :length      => 255,    :nullable => false,                     :unique_index => true
-      has_handle
-      alias_method  :handle_generator, :full_url
-      has_time_and_user_stamps
-      #
-      property      :name,            String,         :length      => 255,    :nullable => false, :default => ''
-      #
-      property      :file_path,       String,    :length => 1024
-      property      :file_time,       DateTime
-      property      :file_size,       Integer
-      property      :file_sha1,       String,    :length => 40
-      property      :tried_fetch,     DataMapper::Resource::Boolean
-      property      :fetched,         DataMapper::Resource::Boolean
-      #
-      before :create, :make_uuid_and_handle
-      before :create, :update_from_file!
-    end
-    base.extend ClassMethods
-  end
-
-  # ===========================================================================
-  #
-  # Delegate methods to uri
-  #
-  def uri
-    @uri ||= Addressable::URI.parse(self.full_url)
-  end
-  # Dispatch anything else to the aggregated uri object
-  def method_missing method, *args
-    if self.uri.respond_to?(method)
-      self.uri.send(method, *args)
-    else
-      super method, *args
-    end
-  end
-
-  def to_s
-    "<a href='#{self.uri.to_s}'>#{self.name}</a>" # <-- !! not escaped !!
-  end
-
-  # ===========================================================================
-  #
-  # ID, naming, etc
-  #
-  def normalize_url!
-    u = Addressable::URI.parse(self.full_url).normalize
-    self.full_url = u.to_s
-  end
-
-  # ===========================================================================
-  #
-  # Properly belongs in FileStore module
-  #
-  #
-  # Refresh cached properties from our copy of the asset.
-  #
-  def update_from_file!
-    self.make_uuid_and_handle # make sure this happened
-    # Set the file path
-    self.file_path = self.to_file_path if self.file_path.blank?
-    # FIXME -- kludge to ripd_root
-    if ! File.exist?(actual_path)
-      self.fetched   = false
-    else
-      self.fetched   = self.tried_fetch = true
-      self.file_size = File.size( actual_path)
-      self.file_time = File.mtime(actual_path)
-    end
-    self.fetched
-  end
-  def actual_path
-    path_to(:ripd_root, self.file_path)
-  end
-
-  # ===========================================================================
-  #
-  # Properly belongs in own module
-  #
-
-  IMW_WGET_OPTIONS = {
-    :root       => :ripd_root,
-    :wait       => 2,
-    :noretry    => true,
-    :log_level  => Logger::DEBUG,
-    :clobber    => false,
-  }
-  #
-  # Fetch from the web
-  #
-  def wget options={}
-    options.reverse_merge! IMW_WGET_OPTIONS
-    cd path_to(options[:root]) do
-      if (not options[:clobber]) && File.file?(file_path) then
-        IMW.log.add options[:log_level], "Skipping #{file_path}"; return
-      end
-      # Do the fetch
-      mkdir_p File.dirname(actual_path)
-      # defaults are --connect-timeout=infinity --read-timeout=900 --tries=20 acc. to man page
-      cmd = %Q{wget -nv "#{full_url}" -O"#{actual_path}" --connect-timeout=5 --read-timeout=10 --tries=1 &}
-      IMW.log.add(options[:log_level], cmd)
-      IMW.log.add(options[:log_level], `#{cmd}`)
-      self.tried_fetch = true
-      sleep options[:wait] # please hammer don't hurt em
-      update_from_file!
-      self.save
-      return self.fetched
-    end
-  end
-
-  #
-  #
-  #
-  def contents options={}
-    wget options
-    if fetched
-      File.open actual_path
-    end
-  end
-
-  # ===========================================================================
-  #
-  # Properly belongs in FileStore
-  #
-
-  protected
-  #
-  # The standard file path for this url's ripped cache
-  #
-  # * leading directory from reverse.dotted.host_scheme:port:user@password
-  # * normalized path/file?query#fragment
-  # * uuid formed from the
-  #
-  def to_file_path
-    file_path_str = ""
-    file_path_str << to_file_path_root_part
-    file_path_str << to_file_path_path_part
-    file_path_str << to_file_path_file_part
-    file_path_str = self.class.path_str_encode(file_path_str)
-    self.class.validate_roundtrip(file_path_str)
-    file_path_str
-  end
-  def file_timestamp
-    file_time.strftime("%Y%m%d-%H%M%S")
-  end
-  def to_file_path_with_timestamp
-    to_file_path + file_timestamp
-  end
-  #
-  # revhost_scheme:port:user@password -- omitting _scheme if it's http, and
-  # omitting :port:user@password if all three are blank.
-  #
-  def to_file_path_root_part
-    root_part_str = ""
-    tld_host_frag = self.class.tier_path_segment(revhost, /^([^\.]+)\.([^\.]{1,2})/)
-    root_part_str << revhost
-    root_part_str << "_#{uri.scheme}"                           unless uri.scheme == 'http'
-    root_part_str << ":#{uri.port}:#{uri.user}@#{uri.password}" unless uri.simple?
-    root_part_str
-  end
-  def to_file_path_path_part
-    uri.path.to_s
-  end
-  def to_file_path_file_part
-    file_path_str = ""
-    file_path_str << "?#{uri.query}"        unless uri.query.nil?
-    file_path_str << "##{uri.fragment}"     unless uri.fragment.nil?
-    file_path_str << "-#{self.uuid}"
-  end
-  public
-
-
-  module ClassMethods
-    #
-    # find_or_creates from url
-    #
-    # url is heuristic_parse'd and normalized by Addressable before lookup:
-    #   "Converts an input to a URI. The input does not have to be a valid URI —
-    #   the method will use heuristics to guess what URI was intended. This is not
-    #   standards compliant, merely user-friendly.
-    #
-    def find_or_create_from_url url_str
-      link = self.find_or_new_from_url url_str
-      link.save
-      link
-    end
-    def find_or_new_from_url url_str # :nodoc:
-      url_str = Addressable::URI.heuristic_parse(url_str).normalize.to_s
-      link = self.first( :full_url => url_str ) || self.new( :full_url => url_str )
-      link.make_uuid_and_handle
-      link.update_from_file!
-      link
-    end
-    def find_or_create_from_file_path ripd_file
-      url_str = Link.url_from_file_path(ripd_file)
-      link = self.first( :full_url => url_str.to_s ) || self.new( :full_url => url_str.to_s )
-      link.file_path = ripd_file
-      link.make_uuid_and_handle
-      link.update_from_file!
-      link.save
-      link
-    end
-    #
-    # Decode url from its file_path
-    #
-    def url_from_file_path fp
-      fp = path_str_decode(fp)
-      m = (%r{\A
-            (#{Addressable::URI::HOST_TLD})  # tld tier
-           /(..?)                            # revhost tier
-           /([^/\:_]+)                       # revhost
-        (?:_([^/\:]+))?                      # _scheme
-        (?::(\d*):([^/]*)@([^@/]*?))?        # :port:user@password
-           /(?:(.*?)/)?                      # /dirs/
-            ([^/]*)                          #  file
-           -([a-f0-9]{32})                   # -uuid
-                                \z}x.match(fp))
-      raise "Can't extract url from file path #{fp}" if !m
-      fp_host, fp_scheme, fp_port, fp_user, fp_pass, fp_path, fp_file, fp_uuid = m.captures
-      fp_host     = fp_host.split('.').reverse.join('.')
-      fp_scheme ||= 'http'
-      fp_pass     = ":#{fp_pass}"             unless fp_pass.blank?
-      fp_userpass = "#{fp_user}#{fp_user}@"   unless fp_user.blank?
-      fp_port     = ":#{fp_port}"             unless fp_port.blank?
-      fp_path     = File.join(*[fp_path, fp_file].compact)
-      "#{fp_scheme}://#{fp_userpass}#{fp_host}#{fp_port}/#{fp_path}"
-    end
-    #
-    # to control files-per-directory madness, take a path segment like "foobar" in
-    #   blah.com/top/foobar/directory
-    # and transform into
-    #   blah.com/top/fo/foobar/directory
-    #
-    # Ex.
-    #   self.class.tier_path_segment('a_username')
-    #   # => 'a_/a_username'
-    #   self.class.tier_path_segment('1')
-    #   # => '1/1'
-    #   self.class.tier_path_segment('com.twitter', /^([^\.]+)\.([^\.]{1,2})/)
-    #   # => 'com/tw/com.twitter'
-    #
-    def self.tier_path_segment(path_seg, re=/(..?)/)
-      frag_seg = re.match(path_seg).captures
-      raise "Can't tier path_seg #{path_seg} using #{re}" if frag_seg.blank?
-      File.join(* [frag_seg, path_seg].flatten )
-    end
-    #
-    #
-    # It's really bad if you can't roundtrip --
-    # since saving is the rare case (only done once!) we insist on checking.
-    #
-    def self.validate_roundtrip file_path_str
-      # uu = self.class.url_from_file_path(file_path_str)
-      # puts "*"*75, uri.to_hash.inspect, ['path str', file_path_str, 'uri', uri.to_s, 'rt', uu.to_s].inspect
-      return_trip_url = Addressable::URI.parse(self.class.url_from_file_path(file_path_str))
-      raise "crapsticks: uri doesn't roundtrip #{file_path_str} to #{uri.to_s}: #{return_trip_url}" if return_trip_url != uri
-    end
-    #
-    # Uses a similar scheme as the 'Quoted Printable' encoding, but more strict
-    # and without linebreaking or anything. The intent is to reversibly and
-    # recognizably store URLs to disk with names that (apart from path) do not
-    # need to be further escaped in filesystem, URL, database or HTML.
-    #
-    # The only characters in a path_encoded string are alpha-numeric /_-.=
-    #
-    # Rules:
-    # * Any character that is not alphanumeric, and is not /_-.  is encoded as an
-    #   equals sign = followed by its upper-case hex encoding.
-    #
-    # * Furthermore, in any sequence of repeated '.' characters, all after the
-    #   first are hex encoded; same with '/'.
-    #
-    # Ex.
-    #   path_encode("www.measuringworth.com/datasets/consumer/result.php?use[]=VCB&use[]=CU&use[]=SZ&year_source=1900&year_result=2007"
-    #   # => www.measuringworth.com/datasets/consumer/result.php=3Fuse=5B=5D=3DVCB=26use=5B=5D=3DCU=26use=5B=5D=3DSZ=26year_source=3D1900=26year_result=3D2007
-    #
-    # Code inspired by "Glenn Parker's response to ruby quiz #23"http://www.rubyquiz.com/quiz23.html
-    #
-    def path_str_encode(str)
-      str.gsub(%r{\.(\.+)}){|chars| '.'+path_encode_chars(chars) }
-      str.gsub(%r{\/(\/+)}){|chars| '/'+path_encode_chars(chars) }
-      str.gsub(%r{[^A-Za-z0-9/_\-\.]+}){|chars| path_encode_chars(chars) }
-    end
-    #
-    # See the notes in path_encode
-    #
-    def path_str_decode(str)
-      str.gsub(/\+([\dA-F]{2})/){ $1.hex.chr }
-    end
-    protected
-    def path_encode_chars(chars) # :nodoc:
-      # send each character to an equals sign followed by its uppercase hex encoding
-      encoded = "";
-      chars.each_byte{|c| encoded << "+%02X" % c }
-      encoded
-    end
-    public
-  end
-end

data/lib/imw/dataset/scrub/old_working_scrubber.rb

@@ -1,87 +0,0 @@
-
-  def self.url_from_file_path fp
-    # FIXME -- doesn't work with extension preservation
-    unless m = (%r{\A([^/_]+)(_[^/]+)?/(?:(.*?)/)?([^/]*)-([a-f0-9]{30,32})\z}.match(fp)) then
-      # m1 = %r{\A([^/_]+)(_[^/]+)?/(?:(.*?))-([a-f0-9]{28,})}i.match(fp);
-      raise "Bad match to #{fp}"
-    end
-    fp_host, fp_scheme, fp_path, fp_file, fp_uuid, fp_ext = m.captures
-    fp_host     = fp_host.split('.').reverse.join('.')
-    fp_scheme ||= 'http'
-    fp_path     = File.join(*[fp_path, fp_file].compact) # FIXME -- no ext
-    url = Addressable::URI.new(fp_scheme, nil, nil, fp_host, nil, fp_path, nil, nil)
-    unless m = (%r{\A([^/_]+)(_[^/]+)?/(?:(.*?)/)?([^/]*)-([a-f0-9]{32})\z}.match(fp)) then
-      # warn "Bad luck!!! #{url.path} hash is #{fp_uuid} vs #{UUID.sha1_create(UUID_INFOCHIMPS_LINKS_NAMESPACE, url.to_s).hexdigest}"
-    end
-    url
-  end
-
-  #
-  # returns [dirname, basename, ext] for the file_path
-  # ext is determined by basename_ext_splitter
-  #
-  def path_split
-    path_split_str path
-  end
-
-  # lowercase; only a-z, num, . -
-  def scrubbed_revhost
-    return unless revhost
-    revhost.downcase.gsub(/[^a-z0-9\.\-]+/i, '')  # note: no _
-  end
-
-  cattr_accessor  :basename_ext_splitter
-  BASENAME_EXT_SPLIT_SMART = /(.+?)\.(tar\.gz|tar\.bz2|[^\.]+)/
-  BASENAME_EXT_NO_SPLIT    = /(.+?)()/
-  self.basename_ext_splitter = BASENAME_EXT_NO_SPLIT
-
-  #
-  # Like File.split but heuristically handles things like .tar.bz2:
-  #
-  #   foo.        => ['foo.', '']
-  #   foo.tar.gz  => ['foo.', '']
-  #   foo.tar.bz2 => ['foo.', '']
-  #   foo.yaml    => ['foo', '']
-  #
-  def path_split_str str
-    if str =~ %r{/.+\z}
-      dirname, basename = %r{\A(.*)/([^/]+)\z}.match(str).captures
-    else
-      dirname, basename = ['', str]
-    end
-    # Get basename, extension (as given by capture groups in basename_ext_splitter)
-    if basename_ext_splitter && (m = /\A#{basename_ext_splitter}\z/i.match(basename))
-      basename, ext = m.captures
-    else
-      basename, ext = [basename, '']
-    end
-    [dirname, basename, ext]
-  end
-
-  # remove all blank components, join the rest with separator
-  def join_non_blank separator, *strs
-    strs.reject(&:blank?).join(separator)
-  end
-
-  # only a-z A-Z, num, .-_/
-  def scrubbed_path
-    path_part = path
-    # colons into /
-    path_part = path_part.gsub(%r{\:+}, '/')
-    # Kill weird chars
-    path_part = path_part.gsub(%r{[^a-zA-Z0-9\.\-_/]+}, '_')
-    # Compact (killing foo/../bar, etc)
-    path_part = path_part.gsub(%r{/[^a-zA-Z0-9]+/}, '/').gsub(%r{/\.\.+/}, '.')
-    # Kill leading & trailing non-alnum
-    path_part = path_part.gsub(%r{^[^a-zA-Z0-9]+}, '').gsub(%r{[^a-zA-Z0-9]+$}, '')
-  end
-
-  #
-  # name for this URL regarded as a file (instance)
-  #
-  def to_file_path
-    dirname, basename, ext = path_split_str(scrubbed_path)
-    basename = join_non_blank '-', basename, uuid
-    basename = join_non_blank '.', basename, ext
-    join_non_blank '/', root_path, dirname, basename
-  end

data/lib/imw/dataset/scrub/scrub.rb

@@ -1,147 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'rubygems'
-require 'active_support'
-require 'uuidtools'
-
-module Scrub
-  class Generic
-    # A regular expression character group
-    # (a bunch of characters ready to drop into /[#{validator}]*/)
-    # whitelisting allowed characters
-    #
-    # Must be overridden in child class
-    class_inheritable_accessor :validator
-
-    # Sentence fragment for error message on failed validation.
-    class_inheritable_accessor :complaint
-    self.complaint = "has characters I can't understand"
-
-    # Proc or string or anything that can be 2nd arg to gsub
-    # to sanitize
-    class_inheritable_accessor :replacer
-    self.replacer  = '-'
-
-    # A regular expression to sanitize objects
-    # if unset or nil, the validator char group
-    class_inheritable_accessor :sanitizer
-
-    # unless overridden or set expressly, just use the
-    # validator
-    def sanitizer
-      @sanitizer || self.validator
-    end
-
-    def sanitize str
-      str = str.to_s
-      str.gsub(%r{([^#{validator.to_s}]+)}u, replacer)
-    end
-
-    def valid? str
-      %r{\A([#{validator.to_s}]*)\z}u.match(str)
-    end
-  end
-
-  #
-  # A permissive, ASCII-only name string - no control chars, newlines, backslash
-  # or <> angle brackets
-  #
-  class Title < Scrub::Generic
-    self.complaint = "should only contain basic keyboard characters (and should not use \\ &lt; or &gt;)."
-    self.validator = %r{a-zA-Z0-9_ ~\!@#\$%\^&\*\(\)\-\+=;\:'"`\[\]\{\}\|,\?\.\/}u
-  end
-
-  #
-  # A permissive, ASCII-only name string - no control chars, newlines, backslash
-  # or <> angle brackets
-  #
-  class UnicodeTitle < Scrub::Title
-    self.complaint  = "should only contain keyboard characters (and should not use \\ &lt; or &gt;)."
-    self.validator  = %r{[:alpha:][:digit:]#{Scrub::Title.validator}}u
-  end
-
-  #
-  # Visible characters and spaces (i.e. anything except control characters, etc.)
-  #
-  class FreeText < Scrub::Generic
-    self.complaint  = "should not contain control characters or that kind of junk."
-    self.validator  = %r{[:print:]\n\t}u
-  end
-
-  module BeginsWithAlpha
-    mattr_accessor :slug
-    self.slug = 'x'
-    # prepend #{slug}#{replacer} to the string if it starts with non-alpha.
-    # so, for instance '23jumpstreet' => 'x_23jumpstreet'
-    def sanitize_with_begins_with_alpha str
-      str = sanitize_without_begins_with_alpha str
-      str = 'x' + replacer + str if (str !~ /^[a-z]/i)  # call at end of chain!
-      str
-    end
-    def valid_with_begins_with_alpha? str
-      (str =~ /^[a-z]/i) && valid_without_begins_with_alpha?(str)
-    end
-    def self.included base
-      base.alias_method_chain :sanitize, :begins_with_alpha # unless defined?(base.sanitize_without_begins_with_alpha)
-      base.alias_method_chain :valid?,   :begins_with_alpha # unless defined?(base.valid_without_begins_with_alpha?)
-    end
-  end
-
-  #
-  # insist that a string be lowercased.
-  #
-  module Lowercased
-    def sanitize_with_lowercased str
-      str = sanitize_without_lowercased str
-      str.downcase # call at end of chain!
-    end
-    def valid_with_lowercase? str
-      (str !~ /[[:upper:]]/u) && valid_without_lowercase?(str)
-    end
-    def self.included base
-      base.alias_method_chain :sanitize, :lowercased # unless defined?(base.sanitize_without_lowercased)
-      base.alias_method_chain :valid?,   :lowercase  # unless defined?(base.valid_without_lowercase?)
-    end
-  end
-
-  #
-  # start with a letter, and contain only A-Za-z0-9_
-  #
-  class Identifier < Scrub::Generic
-    self.complaint  = "should be an identifier: it should start with a letter, and contain only a-z, 0-9 and '_'."
-    self.validator  = %r{a-z0-9_}u
-    self.replacer   = '_'
-    include Scrub::BeginsWithAlpha
-    include Scrub::Lowercased
-  end
-
-  #
-  # start with a letter, and contain only A-Za-z0-9_
-  #
-  class Handle < Scrub::Generic
-    self.complaint  = "should be an identifier: it should start with a letter, and contain only a-z, 0-9 and '_'."
-    self.validator  = %r{a-z0-9_}u
-    self.replacer   = '_'
-    include Scrub::BeginsWithAlpha
-    include Scrub::Lowercased
-  end
-
-  # HANDLE_RE  = %r{\A[a-z][]*\z}i # ascii, not :alpha: etc.
-  # HANDLE_MSG = "should start with a letter, and contain only characters like a-z0-9_-."
-  #
-  # # "Domain names are restricted to the ASCII letters a through z
-  # # (case-insensitive), the digits 0 through 9, and the hyphen, with some other
-  # # restrictions in terms of name length and position of hyphens."
-  # # (http://en.wikipedia.org/wiki/Domain_name#Overview)
-  # # http://tools.ietf.org/html/rfc1034
-  # DOMAIN_RE    = %r{\A[a-z][a-z0-9\-][a-z0-9]\z}i # case insensitive
-  # DOMAIN_MSG   = "should look like a domain name."
-  # DOMAIN_MORE  = "only letters, digits or hyphens (-), start with a letter and end with a letter or number."
-  MSG_EMAIL_BAD      = "should look like an email address (you@somethingsomething.com) and include only letters, numbers and .&nbsp;+&nbsp;-&nbsp;&#37; please."
-  RE_EMAIL_NAME      = '[\w\.%\+\-]+'                          # what you actually see in practice
-  RE_EMAIL_N_RFC2822 = '0-9A-Z!#\$%\&\'\*\+_/=\?^\-`\{|\}~\.' # technically allowed by RFC-2822
-  RE_DOMAIN_HEAD     = '(?:[A-Z0-9\-]+\.)+'
-  RE_DOMAIN_TLD      = '(?:[A-Z]{2}|com|org|net|edu|gov|mil|biz|info|mobi|name|aero|jobs|museum)'
-  RE_EMAIL_OK        = /\A#{RE_EMAIL_NAME}@#{RE_DOMAIN_HEAD}#{RE_DOMAIN_TLD}\z/i
-  RE_EMAIL_RFC2822   = /\A#{RE_EMAIL_N_RFC2822}@#{RE_DOMAIN_HEAD}#{RE_DOMAIN_TLD}\z/i
-
-end

data/lib/imw/dataset/scrub/scrub_simple_url.rb

@@ -1,38 +0,0 @@
-
-
-module IMW
-  module URIScrubber
-
-    def scrubbed
-      to_dirpath
-    end
-  end
-end
-
-module Scrub
-  #
-  # start with a letter, and contain only A-Za-z0-9_
-  #
-  class SimplifiedURL < Scrub::Generic
-    self.complaint  = "should follow our zany simplified URL rules: com.host.dot-reversed:schemeifnothttp/path/seg_men-ts/stuff.ext-SHA1ifweird"
-    self.validator  = %r{#{Addressable::URI::SAFE_CHARS}#{Addressable::URI::RESERVED_CHARS}}u
-    self.replacer   = ''
-    include Scrub::Lowercased
-    attr_accessor :uri
-
-    def valid? str
-      str.to_s.downcase == sanitize(str)
-    end
-
-    def sanitize str
-      # if this fails just normalize once, or don't set $KCODE: http://bit.ly/1664vp
-      uri = Addressable::URI.heuristic_parse(str.to_s).normalize
-      # print [uri.host, uri.host_valid?, uri.path, uri.path_valid?].inspect
-      if uri.host_valid?
-        uri.scrubbed
-      else
-        uri.uuid_path
-      end
-    end
-  end
-end

data/lib/imw/dataset/scrub/scrub_test.rb

@@ -1,60 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-require 'scrub'
-require 'scrub_simple_url'
-
-test_strings = [
-  nil, '', '12', '123', 'simple', 'UPPER', 'CamelCased', 'iden_tifier_23_',
-  'twentyfouralphacharslong', 'twentyfiveatozonlyletters', 'hello.-_there@funnychar.com',
-  "tab\t", "newline\n",
-  "Iñtërnâtiônàlizætiøn",
-  'semicolon;', 'quote"', 'tick\'', 'backtick`', 'percent%', 'plus+', 'space ',
-  'leftanglebracket<', 'ampersand&',
-  "control char-bel\x07",
-  "http://foo.bar.com/",
-  "HTTP://FOO.BAR.com",
-  ".com/zazz",
-  "scheme://user_name@user_acct:passwd@host-name.museum:9047/path;pathquery/p!a-th~2/path?query=param&amp;query=pa%20ram#fragment",
-  "http://web.site.com/path/path/file.ext",
-  "ftp://ftp.site.com/path/path/file.ext",
-  "/absolute/pathname/file.ext",
-  "http://foo.bar.com/.hidden_file_with.ext",
-  "http://foo.bar.com/.hidden_file",
-  "dir/--/non_alpha_path_segment.ext",
-  "http://foo.bar.com/dir/../two_dots_in_path",
-
-]
-
-
-scrubbers = {
-  # :unicode_title   => Scrub::UnicodeTitle.new,
-  # :title           => Scrub::Title.new,
-  # :identifier      => Scrub::Identifier.new,
-  # :free_text       => Scrub::FreeText.new,
-  :handle        => Scrub::Handle.new,
-  :simplified_url  => Scrub::SimplifiedURL.new,
-  # :domain        => Scrub::Domain.new,
-  # :email         => Scrub::Email.new,
-}
-
-scrubbers.each do |scrubber_name, scrubber|
-  puts scrubber_name
-  results = test_strings.map do |test_string|
-    [!!scrubber.valid?(test_string), scrubber.sanitize(test_string).inspect, test_string.inspect ]
-  end
-  results.sort_by{|val,san,orig| val ? 1 : -1 }.each do |val,san,orig|
-    puts "  %-5s %-30s %-30s" % [val,san,orig]
-  end
-end
-
-
-
-# 'foo@bar.com', 'foo@newskool-tld.museum', 'foo@twoletter-tld.de', 'foo@nonexistant-tld.qq',
-#         'r@a.wk', '1234567890-234567890-234567890-234567890-234567890-234567890-234567890-234567890-234567890@gmail.com',
-#         'hello.-_there@funnychar.com', 'uucp%addr@gmail.com', 'hello+routing-str@gmail.com',
-#         'domain@can.haz.many.sub.doma.in',],
-#       :invalid => [nil, '', '!!@nobadchars.com', 'foo@no-rep-dots..com', 'foo@badtld.xxx', 'foo@toolongtld.abcdefg',
-#         'Iñtërnâtiônàlizætiøn@hasnt.happened.to.email', 'need.domain.and.tld@de', "tab\t", "newline\n",
-#         'r@.wk', '1234567890-234567890-234567890-234567890-234567890-234567890-234567890-234567890-234567890@gmail2.com',
-#         # these are technically allowed but not seen in practice:
-#         'uucp!addr@gmail.com', 'semicolon;@gmail.com', 'quote"@gmail.com', 'tick\'@gmail.com', 'backtick`@gmail.com', 'space @gmail.com', 'bracket<@gmail.com', 'bracket>@gmail.com'