loose_tight_dictionary-ruby19 0.0.8

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Seamus Abshere
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,56 @@
+= loose_tight_dictionary
+
+Match things based on string similarity (using the Pair Distance algorithm) and regular expressions.
+
+= Quickstart
+
+    >> right_records = [ 'seamus', 'andy', 'ben' ]
+    => [...]
+    >> left_record = 'Shamus Heaney'
+    => [...]
+    >> d = LooseTightDictionary.new right_records
+    => [...]
+    >> puts d.left_to_right left_record
+    => 'seamus'
+
+Try running the included example file:
+
+    $ ruby examples/first_name_matching.rb 
+    Left side (input)
+    ====================
+    Mr. Seamus
+    Sr. Andy
+    Master BenT
+
+    Right side (output)
+    ====================
+    seamus
+    andy
+    ben
+
+    Results
+    ====================
+    Left record (input)           Right record (output)         Prefix used (if any)          Score                         
+    Mr. Seamus                    seamus                        NULL                          0.666666666666667             
+    Sr. Andy                      andy                          NULL                          0.5                           
+    Master BenT                   ben                           NULL                          0.2                           
+
+= Improving dictionaries
+
+Similarity matching will only get you so far.
+
+    TODO: regex usage
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Seamus Abshere. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "loose_tight_dictionary"
+    gem.summary = %Q{Allows iterative development of dictionaries for big data sets.}
+    gem.description = %Q{Create dictionaries that link rows between two tables (left and right) using loose matching (string similarity) by default and tight matching (regexp) by request.}
+    gem.email = "seamus@abshere.net"
+    gem.homepage = "http://github.com/seamusabshere/loose_tight_dictionary"
+    gem.authors = ["Seamus Abshere"]
+    gem.add_development_dependency "shoulda"
+    gem.add_development_dependency "remote_table", ">=0.2.19"
+    gem.add_dependency 'activesupport', '>=2.3.4'
+    gem.add_dependency 'fastercsv', '>=1.5.3'
+    gem.add_dependency 'andand', '>=1.3.1'
+    gem.add_dependency 'amatch', '>=0.2.5'
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "loose_tight_dictionary #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.8

data/examples/first_name_matching.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+# require 'loose_tight_dictionary'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'loose_tight_dictionary.rb'))
+right_side = [ 'seamus', 'andy', 'ben' ]
+left_side = [ 'Mr. Seamus', 'Sr. Andy', 'Master BenT' ]
+
+puts "Left side (input)"
+puts "=" * 20
+puts left_side
+puts
+
+puts "Right side (output)"
+puts "=" * 20
+puts right_side
+puts
+
+puts "Results"
+puts "=" * 20
+d = LooseTightDictionary.new right_side, :tee => STDOUT, :tee_format => :fixed_width
+d.check left_side
+
+puts d.left_to_right 'Shamus Heaney'

data/examples/icao-bts.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'remote_table'
+require 'ruby-debug'
+require 'logger'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'loose_tight_dictionary.rb'))
+
+$logger = Logger.new STDERR
+$logger.level = Logger::DEBUG
+$logger.datetime_format = "%H:%M:%S"
+# $tee = File.open('tee.csv', 'w')
+$tee = STDOUT
+
+# $ltd_left = /(super|bonanza)/i
+# $ltd_right = /bonanza d-35/i
+# $ltd_dd_left = /bonanza/i
+# $ltd_dd_right = /musk/i
+# $ltd_dd_left_not = /allison/i
+# $ltd_dd_print = true
+# $ltd_ddd_left = /bonanza/i
+# $ltd_ddd_right = /musk/i
+# $ltd_ddd_left_not = /allison/i
+# $ltd_ddd_print = true
+
+@right = RemoteTable.new :url => 'http://www.bts.gov/programs/airline_information/accounting_and_reporting_directives/csv/number_260.csv',
+                        :select => lambda { |record| record['Aircraft Type'].to_i.between?(1, 998) and record['Manufacturer'].present? }
+
+@tightenings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false
+
+@identities = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false
+
+@blockings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false
+
+@positives = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=1&output=csv', :headers => false
+
+@negatives = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=2&output=csv', :headers => false
+
+%w{ tightenings identities blockings }.each do |name|
+  $logger.info name
+  $logger.info "\n" + instance_variable_get("@#{name}").to_a.map { |record| record[0] }.join("\n")
+  $logger.info "\n"
+end
+
+('A'..'Z').each do |letter|
+# %w{ E }.each do |letter|
+  @left = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+                          :encoding => 'US-ASCII',
+                          :row_xpath => '//table/tr[2]/td/table/tr',
+                          :column_xpath => 'td'
+
+  d = LooseTightDictionary.new @right, :tightenings => @tightenings, :identities => @identities, :blockings => @blockings, :logger => $logger, :tee => $tee
+  d.left_reader = lambda { |record| record['Manufacturer'] + ' ' + record['Model'] }
+  d.right_reader = lambda { |record| record['Manufacturer'] + ' ' + record['Long Name'] }
+  d.positives = @positives
+  d.negatives = @negatives
+  d.check @left
+end

data/lib/loose_tight_dictionary.rb

@@ -0,0 +1,346 @@
+require 'active_support'
+require 'active_support/version'
+%w{
+  active_support/core_ext/string
+}.each do |active_support_3_requirement|
+  require active_support_3_requirement
+end if ActiveSupport::VERSION::MAJOR == 3
+require 'amatch'
+require 'andand'
+if RUBY_VERSION >= "1.9"
+  require 'csv'
+else
+  require 'fastercsv'
+end
+
+class LooseTightDictionary
+  class MissedChecks < RuntimeError; end
+  class Mismatch < RuntimeError; end
+  class FalsePositive < RuntimeError; end
+  
+  class T
+    attr_reader :str, :tightened_str
+    def initialize(str, tightened_str)
+      @str = str
+      @tightened_str = tightened_str
+    end
+    
+    def tightened?
+      str != tightened_str
+    end
+    
+    def prefix_and_score(other)
+      prefix = [ tightened_str.length, other.tightened_str.length ].min if tightened? and other.tightened?
+      score = if prefix
+        tightened_str.first(prefix).pair_distance_similar other.tightened_str.first(prefix)
+      else
+        tightened_str.pair_distance_similar other.tightened_str
+      end
+      [ prefix, score ]
+    end
+  end
+
+  include Amatch
+
+  attr_reader :right_records
+  attr_reader :case_sensitive
+
+  attr_accessor :logger
+  attr_accessor :tee
+  attr_accessor :tee_format  
+  attr_accessor :positives
+  attr_accessor :negatives
+  attr_accessor :left_reader
+  attr_accessor :right_reader
+  attr_accessor :blocking_only
+
+  def initialize(right_records, options = {})
+    @right_records = right_records
+    @_raw_tightenings = options[:tightenings] || Array.new
+    @_raw_identities = options[:identities] || Array.new
+    @_raw_blockings = options[:blockings] || Array.new
+    @left_reader = options[:left_reader]
+    @right_reader = options[:right_reader]
+    @positives = options[:positives]
+    @negatives = options[:negatives]
+    @logger = options[:logger]
+    @tee = options[:tee]
+    @tee_format = options[:tee_format] || :fixed_width
+    @case_sensitive = options[:case_sensitive] || false
+    @blocking_only = options[:blocking_only] || false
+  end
+  
+  # def tightenings
+  # def identities
+  # def blockings
+  %w{ tightenings identities blockings }.each do |name|
+    module_eval %{
+      def #{name}
+        @#{name} ||= @_raw_#{name}.map do |i|
+          next if i[0].blank?
+          literal_regexp i[0]
+        end
+      end
+    }
+  end
+  
+  def blocking_only?
+    !!blocking_only
+  end
+
+  def inline_check(left_record, right_record)
+    return unless positives.present? or negatives.present?
+    
+    left = read_left left_record
+    right = read_right right_record
+    
+    if positive_record = positives.andand.detect { |record| record[0] == left }
+      correct_right = positive_record[1]
+      if correct_right.blank? and right.present?
+        logger.andand.debug "  Mismatch! (should match SOMETHING)"
+        raise Mismatch
+      elsif right != correct_right
+        logger.andand.debug "  Mismatch! (should be #{correct_right})"
+        raise Mismatch
+      end
+    end
+    
+    if negative_record = negatives.andand.detect { |record| record[0] == left }
+      incorrect_right = negative_record[1]
+      if incorrect_right.blank? and right.present?
+        logger.andand.debug "  False positive! (should NOT match ANYTHING)"
+        raise FalsePositive
+      elsif right == incorrect_right
+        logger.andand.debug "  False positive! (should NOT be #{incorrect_right})"
+        raise FalsePositive
+      end
+    end
+  end
+
+  def check(left_records)
+    header = [ 'Left record (input)', 'Right record (output)', 'Prefix used (if any)', 'Score' ]
+    case tee_format
+    when :csv
+      tee.andand.puts header.flatten.to_csv
+    when :fixed_width
+      tee.andand.puts header.map { |i| i.to_s.ljust(30) }.join
+    end
+
+    left_records.each do |left_record|
+      begin
+        right_record = left_to_right left_record
+      ensure
+        case tee_format
+        when :csv
+          tee.andand.puts $ltd_1.flatten.to_csv
+        when :fixed_width
+          tee.andand.puts $ltd_1.map { |i| i.to_s.ljust(30) }.join if $ltd_1
+        end
+      end
+    end
+  end
+  
+  def left_to_right(left_record)
+    left = read_left left_record
+    blocking_left = blocking left
+    return if blocking_only? and blocking_left.nil?
+    i_options_left = i_options left
+    t_options_left = t_options left
+    history = Hash.new
+    right_record = right_records.select do |right_record|
+      right = read_right right_record
+      blocking_right = blocking right
+      (not blocking_left and not blocking_right) or
+        (blocking_right and blocking_right.match(left)) or
+        (blocking_left and blocking_left.match(right))
+    end.max do |a_record, b_record|
+      a = read_right a_record
+      b = read_right b_record
+      i_options_a = i_options a
+      i_options_b = i_options b
+      collision_a = collision? i_options_left, i_options_a
+      collision_b = collision? i_options_left, i_options_b
+      if collision_a and collision_b
+        # neither would ever work, so randomly rank one over the other
+        rand(2) == 1 ? -1 : 1
+      elsif collision_a
+        -1
+      elsif collision_b
+        1
+      else
+        t_left_a, t_right_a = optimize t_options_left, t_options(a)
+        t_left_b, t_right_b = optimize t_options_left, t_options(b)
+        a_prefix, a_score = t_left_a.prefix_and_score t_right_a
+        b_prefix, b_score = t_left_b.prefix_and_score t_right_b
+        history[a_record] = [t_left_a.tightened_str, t_right_a.tightened_str, a_prefix ? a_prefix : 'NULL', a_score]
+        history[b_record] = [t_left_b.tightened_str, t_right_b.tightened_str, b_prefix ? b_prefix : 'NULL', b_score]
+        
+        yep_dd = ($ltd_dd_right and $ltd_dd_left and [t_left_a, t_left_b].any? { |f| f.str =~ $ltd_dd_left } and [t_right_a, t_right_b].any? { |f| f.str =~ $ltd_dd_right } and (!$ltd_dd_left_not or [t_left_a, t_left_b].none? { |f| f.str =~ $ltd_dd_left_not }))
+        
+        if $ltd_dd_print and yep_dd
+          logger.andand.debug t_left_a.inspect
+          logger.andand.debug t_right_a.inspect
+          logger.andand.debug t_left_b.inspect
+          logger.andand.debug t_right_b.inspect
+          logger.andand.debug
+        end
+
+        z = 1
+        debugger if yep_dd
+        z = 1
+        
+        if a_score != b_score
+          a_score <=> b_score
+        elsif a_prefix and b_prefix and a_prefix != b_prefix
+          a_prefix <=> b_prefix
+        else
+          b.length <=> a.length
+        end
+      end
+    end
+    $ltd_1 = history[right_record]
+    right = read_right right_record
+    i_options_right = i_options right
+    z = 1
+    debugger if $ltd_left.andand.match(left) or $ltd_right.andand.match(right)
+    z = 1
+    if collision? i_options_left, i_options_right
+      $ltd_0 = nil
+      return
+    else
+      $ltd_0 = right_record
+    end
+    inline_check left_record, right_record
+    right_record
+  end
+  
+  def optimize(t_options_left, t_options_right)
+    cart_prod(t_options_left, t_options_right).max do |a, b|
+      t_left_a, t_right_a = a
+      t_left_b, t_right_b = b
+    
+      a_prefix, a_score = t_left_a.prefix_and_score t_right_a
+      b_prefix, b_score = t_left_b.prefix_and_score t_right_b
+      
+      yep_ddd = ($ltd_ddd_right and $ltd_ddd_left and [t_left_a, t_left_b].any? { |f| f.str =~ $ltd_ddd_left } and [t_right_a, t_right_b].any? { |f| f.str =~ $ltd_ddd_right } and (!$ltd_ddd_left_not or [t_left_a, t_left_b].none? { |f| f.str =~ $ltd_ddd_left_not }))
+      
+      if $ltd_ddd_print and yep_ddd
+        logger.andand.debug t_left_a.inspect
+        logger.andand.debug t_right_a.inspect
+        logger.andand.debug t_left_b.inspect
+        logger.andand.debug t_right_b.inspect
+        logger.andand.debug
+      end
+      
+      z = 1
+      debugger if yep_ddd
+      z = 1
+      
+      if a_score != b_score
+        a_score <=> b_score
+      elsif a_prefix and b_prefix and a_prefix != b_prefix
+        a_prefix <=> b_prefix
+      else
+        # randomly choose
+        # maybe later i can figure out how big the inputs are and apply occam's razor
+        rand(2) == 1 ? -1 : 1
+      end
+    end
+  end
+  
+  def t_options(str)
+    return @_t_options[str] if @_t_options.andand.has_key?(str)
+    @_t_options ||= Hash.new
+    ary = Array.new
+    ary.push T.new(str, str)
+    tightenings.each do |regexp|
+      if match_data = regexp.match(str)
+        ary.push T.new(str, match_data.captures.compact.join)
+      end
+    end
+    @_t_options[str] = ary
+  end
+  
+  class I
+    attr_reader :regexp, :str, :case_sensitive, :identity
+    def initialize(regexp, str, case_sensitive)
+      @regexp = regexp
+      @str = str
+      @identity = regexp.match(str).captures.compact.join
+      @identity = @identity.downcase if case_sensitive
+    end
+  end
+  
+  def collision?(i_options_left, i_options_right)
+    i_options_left.any? do |r_left|
+      i_options_right.any? do |r_right|
+        r_left.regexp == r_right.regexp and r_left.identity != r_right.identity
+      end
+    end
+  end
+  
+  def i_options(str)
+    return @_i_options[str] if @_i_options.andand.has_key?(str)
+    @_i_options ||= Hash.new
+    ary = Array.new
+    identities.each do |regexp|
+      if regexp.match str
+        ary.push I.new(regexp, str, case_sensitive)
+      end
+    end
+    @_i_options[str] = ary
+  end
+  
+  def blocking(str)
+    return @_blocking[str] if @_blocking.andand.has_key?(str)
+    @_blocking ||= Hash.new
+    blockings.each do |regexp|
+      if regexp.match str
+        return @_blocking[str] = regexp
+      end
+    end
+    @_blocking[str] = nil
+  end
+  
+  def literal_regexp(str)
+    return @_literal_regexp[str] if @_literal_regexp.andand.has_key? str
+    @_literal_regexp ||= Hash.new
+    raw_regexp_options = str.split('/').last
+    ignore_case = (!case_sensitive or raw_regexp_options.include?('i')) ? Regexp::IGNORECASE : nil
+    multiline = raw_regexp_options.include?('m') ? Regexp::MULTILINE : nil
+    extended = raw_regexp_options.include?('x') ? Regexp::EXTENDED : nil
+    @_literal_regexp[str] = Regexp.new str.gsub(/\A\/|\/([ixm]*)\z/, ''), (ignore_case||multiline||extended)
+  end
+  
+  def read_left(left_record)
+    return if left_record.nil?
+    if left_reader
+      left_reader.call(left_record)
+    elsif left_record.is_a?(String)
+      left_record
+    else
+      left_record[0]
+    end
+  end
+  
+  def read_right(right_record)
+    return if right_record.nil?
+    if right_reader
+      right_reader.call(right_record)
+    elsif right_record.is_a?(String)
+      right_record
+    else
+      right_record[0]
+    end
+  end
+  
+  # Thanks William James!
+  # http://www.ruby-forum.com/topic/95519#200484
+  def cart_prod(*args)
+    args.inject([[]]){|old,lst|
+      new = []
+      lst.each{|e| new += old.map{|c| c.dup << e }}
+      new
+    }
+  end
+end

data/test/helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'logger'
+require 'ruby-debug'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'loose_tight_dictionary'
+
+class Test::Unit::TestCase
+end

data/test/test_loose_tight_dictionary.rb

@@ -0,0 +1,273 @@
+require 'helper'
+
+require 'remote_table'
+
+# $logger = Logger.new STDERR
+# $logger.level = Logger::INFO
+# $tee = STDOUT
+
+class TestLooseTightDictionary < Test::Unit::TestCase
+  def setup
+    clear_ltd
+    
+    # dh 8 400
+    @a_left = ['DE HAVILLAND CANADA DHC8400 Dash 8']
+    @a_right = ['DEHAVILLAND DEHAVILLAND DHC8-400 DASH-8']
+    # dh 88
+    @b_left = ['ABCDEFG DH88 HIJKLMNOP']
+    # dh 89
+    @c_right = ['ABCDEFG DH89 HIJKLMNOP']
+    # dh 8 200
+    @d_left = ['DE HAVILLAND CANADA DHC8200 Dash 8']
+    @d_right = ['BOMBARDIER DEHAVILLAND DHC8-200Q DASH-8']
+    @d_lookalike = ['ABCD DHC8200 Dash 8']
+    
+    @t_1 = [ '/(dh)c?-?(\d{0,2})-?(\d{0,4})(?:.*?)(dash|\z)/i', 'good tightening for de havilland' ]
+    
+    @r_1 = [ '/(dh)c?-?(\d{0,2})-?(\d{0,4})(?:.*?)(dash|\z)/i', 'good identity for de havilland' ]
+    
+    @left = [
+      @a_left,
+      @b_left,
+      ['DE HAVILLAND DH89 Dragon Rapide'],
+      ['DE HAVILLAND CANADA DHC8100 Dash 8 (E9, CT142, CC142)'],
+      @d_left,
+      ['DE HAVILLAND CANADA DHC8300 Dash 8'],
+      ['DE HAVILLAND DH90 Dragonfly']
+    ]
+    @right = [
+      @a_right,
+      @c_right,
+      @d_right,
+      ['DEHAVILLAND DEHAVILLAND DHC8-100 DASH-8'],
+      ['DEHAVILLAND DEHAVILLAND TWIN OTTER DHC-6']
+    ]
+    @tightenings = []
+    @identities = []
+    @blockings = []
+    @positives = []
+    @negatives = []
+  end
+
+  def clear_ltd
+    @_ltd = nil
+  end
+  
+  def ltd
+    @_ltd ||= LooseTightDictionary.new  @right,
+                                        :tightenings => @tightenings,
+                                        :identities => @identities,
+                                        :blockings => @blockings,
+                                        :positives => @positives,
+                                        :negatives => @negatives,
+                                        :blocking_only => @blocking_only,
+                                        :logger => $logger,
+                                        :tee => $tee
+  end
+
+  if ENV['NEW'] == 'true' or ENV['ALL'] == 'true'
+  end
+  
+  if ENV['OLD'] == 'true' or ENV['ALL'] == 'true'
+    should "optionally only pay attention to things that match blockings" do
+      assert_equal @a_right, ltd.left_to_right(@a_left)
+
+      clear_ltd
+      @blocking_only = true
+      assert_equal nil, ltd.left_to_right(@a_left)
+
+      clear_ltd
+      @blocking_only = true
+      @blockings.push ['/dash/i']
+      assert_equal @a_right, ltd.left_to_right(@a_left)
+    end
+    
+    # the example from the readme, considerably uglier here
+    should "check a simple table" do
+      @right = [ 'seamus', 'andy', 'ben' ]
+      @positives = [ [ 'seamus', 'Mr. Seamus Abshere' ] ]
+      left = [ 'Mr. Seamus Abshere', 'Sr. Andy Rossmeissl', 'Master BenT' ]
+    
+      assert_nothing_raised do
+        ltd.check left
+      end
+    end
+    
+    should "treat a String as a full record if passed through" do
+      dash = 'DHC8-400'
+      b747 = 'B747200/300'
+      dc9 = 'DC-9-10'
+      right_records = [ dash, b747, dc9 ]
+      simple_ltd = LooseTightDictionary.new right_records, :logger => $logger, :tee => $tee
+      assert_equal dash, simple_ltd.left_to_right('DeHavilland Dash-8 DHC-400')
+      assert_equal b747, simple_ltd.left_to_right('Boeing 747-300')
+      assert_equal dc9, simple_ltd.left_to_right('McDonnell Douglas MD81/DC-9')
+    end
+    
+    should "call it a mismatch if you hit a blank positive" do
+      @positives.push [@a_left[0], '']
+      assert_raises(LooseTightDictionary::Mismatch) do
+        ltd.left_to_right @a_left
+      end
+    end
+
+    should "call it a false positive if you hit a blank negative" do
+      @negatives.push [@a_left[0], '']
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.left_to_right @a_left
+      end
+    end
+    
+    should "have a false match without blocking" do
+      # @d_left will be our victim
+      @right.push @d_lookalike
+      @tightenings.push @t_1
+      
+      assert_equal @d_lookalike, ltd.left_to_right(@d_left)
+    end
+    
+    should "do blocking if the left matches a block" do
+      # @d_left will be our victim
+      @right.push @d_lookalike
+      @tightenings.push @t_1
+      @blockings.push ['/(bombardier|de ?havilland)/i']
+      
+      assert_equal @d_right, ltd.left_to_right(@d_left)
+    end
+    
+    should "treat blocks as exclusive" do
+      @right = [ @d_left ]
+      @tightenings.push @t_1
+      @blockings.push ['/(bombardier|de ?havilland)/i']
+
+      assert_equal nil, ltd.left_to_right(@d_lookalike)
+    end
+    
+    should "only use identities if they stem from the same regexp" do
+      @identities.push @r_1
+      @identities.push [ '/(cessna)(?:.*?)(citation)/i' ]
+      @identities.push [ '/(cessna)(?:.*?)(\d\d\d)/i' ]
+      x_left = [ 'CESSNA D-333 CITATION V']
+      x_right = [ 'CESSNA D-333' ]
+      @right.push x_right
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "use the best score from all of the tightenings" do
+      x_left = ["BOEING 737100"]
+      x_right = ["BOEING BOEING 737-100/200"]
+      x_right_wrong = ["BOEING BOEING 737-900"]
+      @right.push x_right
+      @right.push x_right_wrong
+      @tightenings.push ['/(7\d)(7|0)-?\d{1,3}\/(\d\d\d)/i']
+      @tightenings.push ['/(7\d)(7|0)-?(\d{1,3}|[A-Z]{0,3})/i']
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "compare using prefixes if tightened key is shorter than correct match" do
+      x_left = ["BOEING 720"]
+      x_right = ["BOEING BOEING 720-000"]
+      x_right_wrong = ["BOEING BOEING 717-200"]
+      @right.push x_right
+      @right.push x_right_wrong
+      @tightenings.push @t_1
+      @tightenings.push ['/(7\d)(7|0)-?\d{1,3}\/(\d\d\d)/i']
+      @tightenings.push ['/(7\d)(7|0)-?(\d{1,3}|[A-Z]{0,3})/i']
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "use the shortest original input" do
+      x_left = ['De Havilland DHC8-777 Dash-8 Superstar']
+      x_right = ['DEHAVILLAND DEHAVILLAND DHC8-777 DASH-8 Superstar']
+      x_right_long = ['DEHAVILLAND DEHAVILLAND DHC8-777 DASH-8 Superstar/Supernova']
+      
+      @right.push x_right_long
+      @right.push x_right
+      @tightenings.push @t_1
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "perform lookups left to right" do
+      assert_equal @a_right, ltd.left_to_right(@a_left)
+    end
+  
+    should "succeed if there are no checks" do
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+  
+    should "succeed if the positive checks just work" do
+      @positives.push [ @a_left[0], @a_right[0] ]
+    
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+  
+    should "fail if positive checks don't work" do
+      @positives.push [ @d_left[0], @d_right[0] ]
+  
+      assert_raises(LooseTightDictionary::Mismatch) do
+        ltd.check @left
+      end
+    end
+  
+    should "succeed if proper tightening is applied" do
+      @positives.push [ @d_left[0], @d_right[0] ]
+      @tightenings.push @t_1
+  
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+    
+    should "use a Google Docs spreadsheet as a source of tightenings" do
+      @positives.push [ @d_left[0], @d_right[0] ]
+      @tightenings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false
+
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+    
+    should "fail if negative checks don't work" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+    
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.check @left
+      end
+    end
+    
+    should "do inline checking" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+    
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.left_to_right @b_left
+      end
+    end
+  
+    should "fail if negative checks don't work, even with tightening" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+      @tightenings.push @t_1
+    
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.check @left
+      end
+    end
+  
+    should "succeed if proper identity is applied" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+      @positives.push [ @d_left[0], @d_right[0] ]
+      @identities.push @r_1
+    
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification 
+name: loose_tight_dictionary-ruby19
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 8
+  version: 0.0.8
+platform: ruby
+authors: 
+- Seamus Abshere
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-27 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: remote_table
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 0
+        - 2
+        - 19
+        version: 0.2.19
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 3
+        - 4
+        version: 2.3.4
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 5
+        - 3
+        version: 1.5.3
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: andand
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 1
+        - 3
+        - 1
+        version: 1.3.1
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: amatch
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 0
+        - 2
+        - 5
+        version: 0.2.5
+  type: :runtime
+  version_requirements: *id006
+description: Create dictionaries that link rows between two tables (left and right) using loose matching (string similarity) by default and tight matching (regexp) by request.
+email: seamus@abshere.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- examples/first_name_matching.rb
+- examples/icao-bts.rb
+- examples/icao-bts.xls
+- lib/loose_tight_dictionary.rb
+- test/helper.rb
+- test/test_loose_tight_dictionary.rb
+has_rdoc: true
+homepage: http://github.com/seamusabshere/loose_tight_dictionary
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Allows iterative development of dictionaries for big data sets.
+test_files: 
+- test/helper.rb
+- test/test_loose_tight_dictionary.rb
+- examples/first_name_matching.rb
+- examples/icao-bts.rb