loose_tight_dictionary 0.0.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Seamus Abshere
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= loose_tight_dictionary
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Seamus Abshere. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "loose_tight_dictionary"
+    gem.summary = %Q{Allows iterative development of dictionaries for big data sets.}
+    gem.description = %Q{Create dictionaries that link rows between two tables (left and right) using loose matching (string similarity) by default and tight matching (regexp) by request.}
+    gem.email = "seamus@abshere.net"
+    gem.homepage = "http://github.com/seamusabshere/loose_tight_dictionary"
+    gem.authors = ["Seamus Abshere"]
+    gem.add_development_dependency "shoulda"
+    gem.add_development_dependency "remote_table", ">=0.2.16"
+    gem.add_dependency 'activesupport', '>=2.3.4'
+    gem.add_dependency 'fastercsv', '>=1.5.3'
+    gem.add_dependency 'andand', '>=1.3.1'
+    gem.add_dependency 'amatch', '>=0.2.5'
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "loose_tight_dictionary #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/examples/icao-bts.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'remote_table'
+require 'ruby-debug'
+require 'logger'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'loose_tight_dictionary.rb'))
+
+$logger = Logger.new STDERR
+$logger.level = Logger::DEBUG
+$logger.datetime_format = "%H:%M:%S"
+# $tee = File.open('tee.csv', 'w')
+$tee = STDOUT
+
+# $ltd_left = /(super|bonanza)/i
+# $ltd_right = /bonanza d-35/i
+# $ltd_dd_left = /bonanza/i
+# $ltd_dd_right = /musk/i
+# $ltd_dd_left_not = /allison/i
+# $ltd_dd_print = true
+# $ltd_ddd_left = /bonanza/i
+# $ltd_ddd_right = /musk/i
+# $ltd_ddd_left_not = /allison/i
+# $ltd_ddd_print = true
+
+@right = RemoteTable.new :url => 'http://www.bts.gov/programs/airline_information/accounting_and_reporting_directives/csv/number_260.csv',
+                        :select => lambda { |record| record['Aircraft Type'].to_i.between?(1, 998) and record['Manufacturer'].present? }
+
+@tightenings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false
+
+@restrictions = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false
+
+@blockings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false
+
+@positives = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=1&output=csv', :headers => false
+
+@negatives = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=2&output=csv', :headers => false
+
+%w{ tightenings restrictions blockings }.each do |name|
+  $logger.info name
+  $logger.info "\n" + instance_variable_get("@#{name}").to_a.map { |record| record[0] }.join("\n")
+  $logger.info "\n"
+end
+
+('A'..'Z').each do |letter|
+# %w{ E }.each do |letter|
+  @left = RemoteTable.new :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+                          :encoding => 'US-ASCII',
+                          :row_xpath => '//table/tr[2]/td/table/tr',
+                          :column_xpath => 'td'
+
+  d = LooseTightDictionary.new @right, :tightenings => @tightenings, :restrictions => @restrictions, :blockings => @blockings, :logger => $logger, :tee => $tee
+  d.left_reader = lambda { |record| record['Manufacturer'] + ' ' + record['Model'] }
+  d.right_reader = lambda { |record| record['Manufacturer'] + ' ' + record['Long Name'] }
+  d.positives = @positives
+  d.negatives = @negatives
+  d.check @left
+end

data/lib/loose_tight_dictionary.rb

@@ -0,0 +1,270 @@
+require 'active_support'
+require 'active_support/version'
+%w{
+  active_support/core_ext/string
+}.each do |active_support_3_requirement|
+  require active_support_3_requirement
+end if ActiveSupport::VERSION::MAJOR == 3
+require 'amatch'
+require 'andand'
+require 'fastercsv'
+
+class LooseTightDictionary
+  class MissedChecks < RuntimeError; end
+  class Mismatch < RuntimeError; end
+  class FalsePositive < RuntimeError; end
+  
+  class T
+    attr_reader :str, :tightened_str
+    def initialize(str, tightened_str)
+      @str = str
+      @tightened_str = tightened_str
+    end
+    
+    def tightened?
+      str != tightened_str
+    end
+    
+    def prefix_and_score(other)
+      prefix = [ tightened_str.length, other.tightened_str.length ].min if tightened? and other.tightened?
+      score = if prefix
+        tightened_str.first(prefix).pair_distance_similar other.tightened_str.first(prefix)
+      else
+        tightened_str.pair_distance_similar other.tightened_str
+      end
+      [ prefix, score ]
+    end
+  end
+
+  include Amatch
+
+  attr_reader :right_records
+  attr_reader :tightenings
+  attr_reader :restrictions
+  attr_reader :blockings
+  attr_reader :logger
+  attr_reader :tee
+  attr_reader :case_sensitive
+  
+  attr_accessor :positives
+  attr_accessor :negatives
+  attr_accessor :left_reader
+  attr_accessor :right_reader
+
+  def initialize(right_records, options = {})
+    @right_records = right_records
+    @tightenings = options[:tightenings] || Array.new
+    @restrictions = options[:restrictions] || Array.new
+    @blockings = options[:blockings] || Array.new
+    @left_reader = options[:left_reader]
+    @right_reader = options[:right_reader]
+    @positives = options[:positives]
+    @negatives = options[:negatives]
+    @logger = options[:logger]
+    @tee = options[:tee]
+    @case_sensitive = options[:case_sensitive] || false
+  end
+
+  def inline_check(left_record, right_record)
+    return unless positives.present? or negatives.present?
+    
+    left = read_left left_record
+    right = read_right right_record
+    
+    if p = positives.andand.detect { |record| record[0] == left }
+      correct_right = p[1]
+    else
+      correct_right = :ignore
+    end
+    
+    if n = negatives.andand.detect { |record| record[0] == left }
+      incorrect_right = n[1]
+    else
+      incorrect_right = :ignore
+    end
+    
+    if correct_right != :ignore and right != correct_right
+      logger.andand.debug "  Mismatch! (should be #{correct_right})"
+      raise Mismatch
+    end
+    
+    if incorrect_right != :ignore and right == incorrect_right
+      logger.andand.debug "  False positive! (should NOT be #{incorrect_right})"
+      raise FalsePositive
+    end
+  end
+
+  def check(left_records)
+    unless positives.present? or negatives.present?
+      logger.andand.info "You didn't define any positives or negatives, so running check doesn't do anything"
+      return
+    end
+    left_records.each do |left_record|
+      right_record = left_to_right left_record
+      inline_check left_record, right_record
+      tee.andand.puts [ read_left(left_record), read_right(right_record), $ltd_1 ].flatten.to_csv
+    end
+  end
+  
+  def left_to_right(left_record)
+    left = read_left left_record
+    restricted_left = restrict left
+    blocking_left = blocking left
+    t_options_left = t_options left
+    history = Hash.new
+    right_record = right_records.select { |record| blocking_left.nil? or blocking_left.match(read_right(record)) }.max do |a_record, b_record|
+      a = read_right a_record
+      b = read_right b_record
+      restricted_a = restrict a
+      restricted_b = restrict b
+      if restricted_left and restricted_a and restricted_b and restricted_left != restricted_a and restricted_left != restricted_b
+        # neither would ever work, so randomly rank one over the other
+        rand(2) == 1 ? -1 : 1
+      elsif restricted_left and restricted_a and restricted_left != restricted_a
+        -1
+      elsif restricted_left and restricted_b and restricted_left != restricted_b
+        1
+      else
+        t_left_a, t_right_a = optimize t_options_left, t_options(a)
+        t_left_b, t_right_b = optimize t_options_left, t_options(b)
+        a_prefix, a_score = t_left_a.prefix_and_score t_right_a
+        b_prefix, b_score = t_left_b.prefix_and_score t_right_b
+        history[a_record] = [t_left_a.tightened_str, t_right_a.tightened_str, a_prefix ? a_prefix : 'NULL', a_score]
+        history[b_record] = [t_left_b.tightened_str, t_right_b.tightened_str, b_prefix ? b_prefix : 'NULL', b_score]
+        
+        yep_dd = ($ltd_dd_right and $ltd_dd_left and [t_left_a, t_left_b].any? { |f| f.str =~ $ltd_dd_left } and [t_right_a, t_right_b].any? { |f| f.str =~ $ltd_dd_right } and (!$ltd_dd_left_not or [t_left_a, t_left_b].none? { |f| f.str =~ $ltd_dd_left_not }))
+        
+        if $ltd_dd_print and yep_dd
+          logger.andand.debug t_left_a.inspect
+          logger.andand.debug t_right_a.inspect
+          logger.andand.debug t_left_b.inspect
+          logger.andand.debug t_right_b.inspect
+          logger.andand.debug
+        end
+
+        z = 1
+        debugger if yep_dd
+        z = 1
+        
+        if a_score != b_score
+          a_score <=> b_score
+        elsif a_prefix and b_prefix and a_prefix != b_prefix
+          a_prefix <=> b_prefix
+        else
+          b.length <=> a.length
+        end
+      end
+    end
+    $ltd_1 = history[right_record]
+    right = read_right right_record
+    restricted_right = restrict right
+    z = 1
+    debugger if $ltd_left.andand.match(left) or $ltd_right.andand.match(right)
+    z = 1
+    return if restricted_left and restricted_right and restricted_left != restricted_right
+    inline_check left_record, right_record
+    right_record
+  end
+  
+  def optimize(t_options_left, t_options_right)
+    cart_prod(t_options_left, t_options_right).max do |a, b|
+      t_left_a, t_right_a = a
+      t_left_b, t_right_b = b
+    
+      a_prefix, a_score = t_left_a.prefix_and_score t_right_a
+      b_prefix, b_score = t_left_b.prefix_and_score t_right_b
+      
+      yep_ddd = ($ltd_ddd_right and $ltd_ddd_left and [t_left_a, t_left_b].any? { |f| f.str =~ $ltd_ddd_left } and [t_right_a, t_right_b].any? { |f| f.str =~ $ltd_ddd_right } and (!$ltd_ddd_left_not or [t_left_a, t_left_b].none? { |f| f.str =~ $ltd_ddd_left_not }))
+      
+      if $ltd_ddd_print and yep_ddd
+        logger.andand.debug t_left_a.inspect
+        logger.andand.debug t_right_a.inspect
+        logger.andand.debug t_left_b.inspect
+        logger.andand.debug t_right_b.inspect
+        logger.andand.debug
+      end
+      
+      z = 1
+      debugger if yep_ddd
+      z = 1
+      
+      if a_score != b_score
+        a_score <=> b_score
+      elsif a_prefix and b_prefix and a_prefix != b_prefix
+        a_prefix <=> b_prefix
+      else
+        # randomly choose
+        # maybe later i can figure out how big the inputs are and apply occam's razor
+        rand(2) == 1 ? -1 : 1
+      end
+    end
+  end
+  
+  def t_options(str)
+    return @_t_options[str] if @_t_options.andand.has_key?(str)
+    @_t_options ||= Hash.new
+    ary = Array.new
+    ary << T.new(str, str)
+    tightenings.each do |tightening|
+      if literal_regexp(tightening[0]).match str
+        ary << T.new(str, $~.captures.compact.join)
+      end
+    end
+    @_t_options[str] = ary
+  end
+  
+  def blocking(str)
+    return @_blocking[str] if @_blocking.andand.has_key?(str)
+    @_blocking ||= Hash.new
+    blockings.each do |blocking|
+      regexp = literal_regexp blocking[0]
+      if regexp.match str
+        return @_blocking[str] = regexp
+      end
+    end
+    @_blocking[str] = nil
+  end
+  
+  def restrict(str)
+    return @_restrict[str] if @_restrict.andand.has_key?(str)
+    @_restrict ||= Hash.new
+    restrictions.each do |restriction|
+      if literal_regexp(restriction[0]).match str
+        retval = $~.captures.compact.join
+        retval = retval.downcase unless case_sensitive
+        return @_restrict[str] = retval
+      end
+    end
+    @_restrict[str] = nil
+  end
+  
+  def literal_regexp(str)
+    return @_literal_regexp[str] if @_literal_regexp.andand.has_key? str
+    @_literal_regexp ||= Hash.new
+    raw_regexp_options = str.split('/').last
+    i = (!case_sensitive or raw_regexp_options.include?('i')) ? Regexp::IGNORECASE : nil
+    m = raw_regexp_options.include?('m') ? Regexp::MULTILINE : nil
+    x = raw_regexp_options.include?('x') ? Regexp::EXTENDED : nil
+    @_literal_regexp[str] = Regexp.new str.gsub(/\A\/|\/([ixm]*)\z/, ''), (i||m||x), 'U'
+  end
+  
+  def read_left(left_record)
+    return if left_record.nil?
+    left_reader ? left_reader.call(left_record) : left_record[0]
+  end
+  
+  def read_right(right_record)
+    return if right_record.nil?
+    right_reader ? right_reader.call(right_record) : right_record[0]
+  end
+  
+  # Thanks William James!
+  # http://www.ruby-forum.com/topic/95519#200484
+  def cart_prod(*args)
+    args.inject([[]]){|old,lst|
+      new = []
+      lst.each{|e| new += old.map{|c| c.dup << e }}
+      new
+    }
+  end
+end

data/test/helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'logger'
+require 'ruby-debug'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'loose_tight_dictionary'
+
+class Test::Unit::TestCase
+end

data/test/test_loose_tight_dictionary.rb

@@ -0,0 +1,211 @@
+require 'helper'
+
+require 'remote_table'
+
+# $logger = Logger.new STDERR
+# $logger.level = Logger::INFO
+# $tee = STDOUT
+
+class TestLooseTightDictionary < Test::Unit::TestCase
+  def setup
+    clear_ltd
+    
+    # dh 8 400
+    @a_left = ['DE HAVILLAND CANADA DHC8400 Dash 8']
+    @a_right = ['DEHAVILLAND DEHAVILLAND DHC8-400 DASH-8']
+    # dh 88
+    @b_left = ['ABCDEFG DH88 HIJKLMNOP']
+    # dh 89
+    @c_right = ['ABCDEFG DH89 HIJKLMNOP']
+    # dh 8 200
+    @d_left = ['DE HAVILLAND CANADA DHC8200 Dash 8']
+    @d_right = ['BOMBARDIER DEHAVILLAND DHC8-200Q DASH-8']
+    @d_lookalike = ['ABCD DHC8200 Dash 8']
+    
+    @t_1 = [ '/(dh)c?-?(\d{0,2})-?(\d{0,4})(?:.*?)(dash|\z)/i', 'good tightening for de havilland' ]
+    
+    @d_1 = [ '/(dh)c?-?(\d{0,2})-?(\d{0,4})(?:.*?)(dash|\z)/i', 'good restriction for de havilland' ]
+    
+    @left = [
+      @a_left,
+      @b_left,
+      ['DE HAVILLAND DH89 Dragon Rapide'],
+      ['DE HAVILLAND CANADA DHC8100 Dash 8 (E9, CT142, CC142)'],
+      @d_left,
+      ['DE HAVILLAND CANADA DHC8300 Dash 8'],
+      ['DE HAVILLAND DH90 Dragonfly']
+    ]
+    @right = [
+      @a_right,
+      @c_right,
+      @d_right,
+      ['DEHAVILLAND DEHAVILLAND DHC8-100 DASH-8'],
+      ['DEHAVILLAND DEHAVILLAND TWIN OTTER DHC-6']
+    ]
+    @tightenings = []
+    @restrictions = []
+    @blockings = []
+    @positives = []
+    @negatives = []
+  end
+
+  def clear_ltd
+    @_ltd = nil
+  end
+  
+  def ltd
+    @_ltd ||= LooseTightDictionary.new  @right,
+                                        :tightenings => @tightenings,
+                                        :restrictions => @restrictions,
+                                        :blockings => @blockings,
+                                        :positives => @positives,
+                                        :negatives => @negatives,
+                                        :logger => $logger,
+                                        :tee => $tee
+  end
+
+  if ENV['NEW'] == 'true' or ENV['ALL'] == 'true'
+  end
+  
+  if ENV['OLD'] == 'true' or ENV['ALL'] == 'true'
+    should "have a false match without blocking" do
+      # @d_left will be our victim
+      @right.push @d_lookalike
+      @tightenings.push @t_1
+      
+      assert_equal @d_lookalike, ltd.left_to_right(@d_left)
+    end
+    
+    should "do blocking if the left matches a block" do
+      # @d_left will be our victim
+      @right.push @d_lookalike
+      @tightenings.push @t_1
+      @blockings.push ['/(bombardier|de ?havilland)/i']
+      
+      assert_equal @d_right, ltd.left_to_right(@d_left)
+    end
+    
+    should "not do blocking if the left doesn't match any blockings" do
+      @tightenings.push @t_1
+      @blockings.push ['/(bombardier|de ?havilland)/i']
+      
+      assert_equal @d_right, ltd.left_to_right(@d_lookalike)
+    end
+    
+    should "use the best score from all of the tightenings" do
+      x_left = ["BOEING 737100"]
+      x_right = ["BOEING BOEING 737-100/200"]
+      x_right_wrong = ["BOEING BOEING 737-900"]
+      @right.push x_right
+      @right.push x_right_wrong
+      @tightenings.push ['/(7\d)(7|0)-?\d{1,3}\/(\d\d\d)/i']
+      @tightenings.push ['/(7\d)(7|0)-?(\d{1,3}|[A-Z]{0,3})/i']
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "compare using prefixes if tightened key is shorter than correct match" do
+      x_left = ["BOEING 720"]
+      x_right = ["BOEING BOEING 720-000"]
+      x_right_wrong = ["BOEING BOEING 717-200"]
+      @right.push x_right
+      @right.push x_right_wrong
+      @tightenings.push @t_1
+      @tightenings.push ['/(7\d)(7|0)-?\d{1,3}\/(\d\d\d)/i']
+      @tightenings.push ['/(7\d)(7|0)-?(\d{1,3}|[A-Z]{0,3})/i']
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "use the shortest original input" do
+      x_left = ['De Havilland DHC8-777 Dash-8 Superstar']
+      x_right = ['DEHAVILLAND DEHAVILLAND DHC8-777 DASH-8 Superstar']
+      x_right_long = ['DEHAVILLAND DEHAVILLAND DHC8-777 DASH-8 Superstar/Supernova']
+      
+      @right.push x_right_long
+      @right.push x_right
+      @tightenings.push @t_1
+      
+      assert_equal x_right, ltd.left_to_right(x_left)
+    end
+    
+    should "perform lookups left to right" do
+      assert_equal @a_right, ltd.left_to_right(@a_left)
+    end
+  
+    should "succeed if there are no checks" do
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+  
+    should "succeed if the positive checks just work" do
+      @positives.push [ @a_left[0], @a_right[0] ]
+    
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+  
+    should "fail if positive checks don't work" do
+      @positives.push [ @d_left[0], @d_right[0] ]
+  
+      assert_raises(LooseTightDictionary::Mismatch) do
+        ltd.check @left
+      end
+    end
+  
+    should "succeed if proper tightening is applied" do
+      @positives.push [ @d_left[0], @d_right[0] ]
+      @tightenings.push @t_1
+  
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+    
+    should "use a Google Docs spreadsheet as a source of tightenings" do
+      @positives.push [ @d_left[0], @d_right[0] ]
+      @tightenings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false
+
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+    
+    should "fail if negative checks don't work" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+    
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.check @left
+      end
+    end
+    
+    should "do inline checking" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+    
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.left_to_right @b_left
+      end
+    end
+  
+    should "fail if negative checks don't work, even with tightening" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+      @tightenings.push @t_1
+    
+      assert_raises(LooseTightDictionary::FalsePositive) do
+        ltd.check @left
+      end
+    end
+  
+    should "succeed if proper restriction is applied" do
+      @negatives.push [ @b_left[0], @c_right[0] ]
+      @positives.push [ @d_left[0], @d_right[0] ]
+      @restrictions.push @d_1
+    
+      assert_nothing_raised do
+        ltd.check @left
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification 
+name: loose_tight_dictionary
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Seamus Abshere
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-04-28 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: remote_table
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 16
+        version: 0.2.16
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 3
+        - 4
+        version: 2.3.4
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 5
+        - 3
+        version: 1.5.3
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: andand
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 1
+        version: 1.3.1
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: amatch
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 5
+        version: 0.2.5
+  type: :runtime
+  version_requirements: *id006
+description: Create dictionaries that link rows between two tables (left and right) using loose matching (string similarity) by default and tight matching (regexp) by request.
+email: seamus@abshere.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- examples/icao-bts.rb
+- examples/icao-bts.xls
+- lib/loose_tight_dictionary.rb
+- test/helper.rb
+- test/test_loose_tight_dictionary.rb
+has_rdoc: true
+homepage: http://github.com/seamusabshere/loose_tight_dictionary
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Allows iterative development of dictionaries for big data sets.
+test_files: 
+- test/helper.rb
+- test/test_loose_tight_dictionary.rb
+- examples/icao-bts.rb