airport_scraper 0.0.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Jacob Harris
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,36 @@
+= airport_scraper
+
+A gem for extracting Airport mentions from short snippets of text. Just something I threw together as an experiment that's turned into an interesting hobby project.
+
+== Examples
+
+	scraper = AirportScraper.new
+  pdx1 = scrape.extract_airports("On my flight to Portland. Looking forward to kicking back at home.")
+  pdx2 = scrape.extract_airports("Landing in PDX now. See you all soon.")
+
+Both pdx1 and pdx2 would be an array with an Airport hash for Portland International Airport
+
+	multi = scrape.extract_airports("On @SouthwestAir #2992 heading from LIT to DAL and CEO Gary Kelly is on board!")
+	assert_equal ['LIT', 'DAL'], multi.map {|x| x['code']}
+
+== Possible Future Work
+
+* Ways to limit scope to major airports only or specific countries
+* More airport information
+* Geocoding/WOEIDs for airports
+* Ordering of airports in the result array to reflect trip order
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+   bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Jacob Harris. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "airport_scraper"
+    gem.summary = %Q{A gem for extracting airports from mentions in text}
+    gem.description = %Q{A gem for extracting airport codes from text}
+    gem.email = "jharris@nytimes.com"
+    gem.homepage = "http://github.com/harrisj/airport_scraper"
+    gem.authors = ["Jacob Harris"]
+    gem.add_development_dependency "shoulda", ">= 0"
+    gem.add_development_dependency "yard", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/airport_scraper.rb

@@ -0,0 +1,169 @@
+require 'yaml'
+
+class AirportScraper
+  attr_reader :airports, :airport_codes
+
+  def initialize
+    load_airports
+    create_regexes
+  end
+
+  def regex_from_matchers(matchers)
+    if matchers.nil? || matchers.empty?
+      nil
+    else
+      /^(#{matchers.map {|x| x.gsub(".", "\\.")}.join('|')})\b/i
+    end
+  end
+
+  def load_airports
+    @airports = {}
+    
+    %w(ca_airports us_airports intl_airports).each do |file|
+      @airports.merge!(YAML.load_file(File.join(File.dirname(__FILE__), "#{file}.yml")))
+    end
+    
+    @matcher_prefixes = {}
+    
+    @airports.each do |key, value|
+      value['code'] = key
+      value['major'] ||= false
+      value['match_priority'] ||= value['major'] ? 10 : 0
+      value['name'] ||= value['city']
+      value['matchers'] = case value['matchers']
+        when nil
+          []
+        when Array
+          value['matchers']
+        else
+          value['matchers'].to_a
+      end
+      
+      value['regex'] = regex_from_matchers(value['matchers'])
+
+      unless value['matchers'].nil?
+        prefixes = value['matchers'].map {|x| prefix_from_match(x)}.uniq
+        prefixes.each do |p| 
+          @matcher_prefixes[p] ||= []
+          @matcher_prefixes[p] << value
+        end
+      end
+    end
+    
+    @airport_codes = @airports.keys
+
+    @matcher_prefixes.values.each do |airports|
+      airports.sort! {|a, b| b['match_priority'] <=> a['match_priority'] }
+    end
+    
+#    raise @matcher_prefixes.inspect
+  end
+  
+  def prefix_from_match(str)
+    case str 
+    when /\w\w\b/
+      str[0,2].downcase
+    else
+      str[0,3].downcase
+    end
+  end
+
+  def create_regexes
+    @code_match_regex = /\b([A-Z]{3})\b/
+
+    flight_regex = /(flight|flying|plane|jet|turboprop)(\s(back|again|over))?/i
+    
+    @trans_regex = /(\sto\s)|(\s?->\s?)|(\s?>\s?)|(\s?✈\s?)/
+    @via_regex = /,?\s?(via|by way of|on route to)\s/
+
+    @preposition_regex = /\sfrom\s|\sto\s|#{@via_regex}|#{@trans_regex}|\sin\s|\sat\s|@\s|\sout of\s/i
+
+    airport_regex = /(.+)/
+    prep_airport_regex = /(.+)(?=#{@preposition_regex})/
+    
+    @match_regexes = [
+      [/(#{@code_match_regex}(#{@trans_regex}(#{@code_match_regex}))+\b)/, @trans_regex],# (#{@via_regex}#{@code_match_regex}\b)?)/i,
+      /((at|@|in) #{airport_regex} airport)/i,
+      /((boarding|departing) (to|from|in) #{airport_regex})/i,
+      /(touched down in #{airport_regex})\b/i,
+      /((to land)|(land(ed|ing|s)) (in|at) #{airport_regex})\b/i,
+      [/(#{flight_regex}#{@preposition_regex}(#{prep_airport_regex}#{@preposition_regex})*#{airport_regex}+)/i, @trans_regex],
+      #/(#{flight_regex}( (from|in|at|out of) #{airport_regex})? (to|into|towards) #{airport_regex}(#{@via_regex}#{airport_regex})?)/i,
+    ]
+  end
+
+  def airport(code)
+    @airports[code]
+  end
+  
+  def flight_terms
+    %w(touched landed landing land lands plane jet turboprop flying flight boarding departing)
+  end
+
+  def possible_flight?(text)
+    @match_regexes.any? do |regex_pair|
+      if regex_pair.is_a? (Array)
+        regex = regex_pair[0]
+      else
+        regex = regex_pair
+      end
+      
+      regex =~ text
+    end
+  end
+
+  def is_flight(text)
+    true
+  end
+
+  def extract_airports(text)
+    airports = [] 
+    
+    #puts @airport_regex.inspect
+
+    @match_regexes.each do |regex_pair|
+      
+      case regex_pair
+      when Array
+        regex = regex_pair[0]
+        split_regex = regex_pair[1]
+      else
+        regex = regex_pair
+      end
+      
+      text.scan(regex) do |matches|
+        if split_regex
+          matches = matches.compact.map {|m| m.split(split_regex)}.flatten.uniq
+        end
+        
+        #puts "Matches: #{matches.compact.inspect}"
+        # puts "Text: #{text}"
+        # puts "Regex: #{regex.inspect}"
+        matches.compact.each do |match|
+          next if match.nil? || match.length < 2
+
+          if match =~ /^#{@code_match_regex}/
+            #puts "MATCH: #{match}"
+            airport = @airports[$1]
+            airports << airport unless airport.nil?
+          else
+            possible_airports = @matcher_prefixes[prefix_from_match(match)]
+            unless possible_airports.nil?
+              possible_airports.each do |a|
+                next if a['regex'].nil?
+                if match =~ /#{a['regex']}\b/
+                  airports << a
+                  break
+                end
+              end
+            end
+          end
+        end
+        
+        # break
+      end
+    end
+    
+    airports.uniq
+  end
+end

data/lib/ca_airports.yml

@@ -0,0 +1,133 @@
+YCD:
+  city: "Nanaimo, BC"
+  matchers:
+    - Nanaimo
+
+YEG:
+  city: "Edmonton"
+  matchers:
+    - "Edmonton"
+    - "Alberta"
+
+YFC:
+  city: "Fredericton, NB"
+  matchers:
+    - "Fredericton"
+
+YGK:
+  name: "Norman Rogers Airport"
+  city: "Kingston, ON"
+  match_priority: 100
+  matchers:
+    - "Kingston, ON"
+    - "Kingston ON"
+    - "Kingston Ontario"
+    - "Kingston, Ontario"
+
+YHZ:
+  city: "Halifax, NS"
+  matchers:
+    - "Halifax"
+    
+YKF:
+  city: "Kitchener, ON"
+  matchers:
+    - Kitchener
+
+YMQ:
+  city: "Montreal, QC (All Airports)"
+  matchers:
+    - Montreal
+
+YOW:
+  city: Ottawa
+  matchers:
+    - Ottawa
+    
+YUL:
+  name: "Montreal Dorval Airport"
+  city: "Montreal, QC"
+  match_priority: 50
+  matchers:
+    - "Dorval"
+    - "Montreal Dorval"
+
+YQB:
+  city: Quebec City
+  matchers:
+    - Quebec City
+
+YQM:
+  city: "Moncton, NB"
+  matchers:
+    - "Moncton"
+
+YQR:
+  city: "Regina, SK"
+  matchers:
+    - "Regina"
+    
+YQT:
+  city: "Thunder Bay, ON"
+  matchers:
+    - Thunder Bay
+
+YVR:
+  name: Vancouver International Airport
+  city: "Vancouver, BC"
+  matchers:
+    - "Vancouver"
+
+YWG:
+  city: "Winnipeg"
+  matchers:
+    - Winnipeg
+
+YXU:
+  city: "London, ON"
+  match_priority: 100
+  matchers:
+    - "London, ON"
+    - "London Ontario"
+    - "London, Ontario"
+
+YXY:
+  city: "Whitehorse, YT"
+  matchers:
+    - "Whitehorse"
+    
+YYC:
+  city: Calgary
+  county: ca
+  matchers:
+    - Calgary
+    
+YYG:
+  city: "Charlottetown, PEI"
+  country: ca
+  matchers:
+    - "Charlottetown"
+    - "Prince Edward Island"
+    - "P.E.I."
+
+YYJ:
+  city: "Victoria, BC"
+  matchers:
+    - "Victoria"
+
+YYT:
+  city: "Saint Johns, NL"
+  matchers:
+    - "Saint Johns"
+    - "St. Johns"
+    - "St.Johns"
+
+YYZ:
+  city: Toronto
+  matchers:
+    - "Toronto"
+
+YZF:
+  city: "Yellowknife, NT"
+  matchers:
+    - "Yellowknife"