airport_scraper 0.0.1

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/airport_scraper.rb'}"
+puts "Loading airport_scraper gem"
+exec "#{irb} #{libs} --simple-prompt"

data/test/bad_matches.yml

@@ -0,0 +1,10 @@
+- "Taliban factions compete for credit in CIA bombing deaths - It was one of the worst blows ever to America's int.. http://bit.ly/8uo7l4"
+- "@therealjuicyj #wtf does 'goin' ham' mean?! i hear you use it but we don't say that in toronto at all.."
+- "Angelina Jolie and Brad Pitt take their kids to the Mary Poppins show on Broadway in NYC – January 3 â. http://bit.ly/6vcSl1"
+- "back in san diego, ready to boogaaayy. :]"
+- "In SLC Sky Club watching Jets game.  http://myloc.me/2AFzh"
+- "It's been real! See you in ATL shawty! RT @CEOTUFFLUV: it was fun! Last night in Buffalo!"
+- "'What is Justin Beiber's birthday? Justin Bieber (real name Alex Lawrence) was born March 14th, 1994 in London.' -- LMAO! LOSERS!"
+- "Going to climb into bed with a book. Work in Philly resumes tomorrow."
+- "RT @ourpdx: Do you want a chance to win tickets for Xanadu's stage perf in PDX?  Read this:  http://bit.ly/8LnD70"
+

data/test/ca_airports_tests.yml

@@ -0,0 +1,102 @@
+YCD:
+  - "On a flight to Nanaimo"
+
+YEG:
+  - "Flying from NYC to YEG"
+  - "Landing in Edmonton now. Oh Canada!"
+  - "On the plane to Alberta"
+
+YFC:
+  - "Touched down in Fredericton. Now to drive home."
+
+YGK:
+  - "On the plane to Kingston Ontario"
+  - "Landing in Kingston, ON"
+  - "Flying from New York City to Kingston, Ontario"
+
+YHZ:
+  - "Landed in Halifax. Now to find the Trailer Park Boys"
+    
+YKF:
+  - "On a jet to Kitchener."
+
+YMQ:
+  - "Landing in Montreal next stop: Poutine"
+
+YOW:
+  - "Just touched down in Ottawa"
+    
+YUL:
+  - "Landing at Montreal Dorval airport"
+  - "Landed at Dorval and ready to leave the plane"
+
+YQB:
+  - "On a flight to Quebec City"
+
+YQM:
+  - "Landing in Moncton, you probably don't know where this is"
+
+YQR:
+  - "On a turboprop to Regina"
+    
+YQT:
+  - "Landed in Thunder Bay"
+
+YVR:
+  - "Flying to Vancouver tomorrow for the Winter Olympics."
+
+YWG:
+  - "Landing in Winnipeg."
+
+YYZ:
+  - "Good times flying from YYZ to SFO today. Why is our security so reactive? Fear? If terror was Umar Farouk's goal, then Mission Accomplished."
+  - "flying back to Toronto this morning. Sad"
+    
+# YXU:
+#   city: "London, ON"
+#   match_priority: 100
+#   matchers:
+#     - "London, ON"
+#     - "London Ontario"
+#     - "London, Ontario"
+
+# YXY:
+#   city: "Whitehorse, YT"
+#   matchers:
+#     - "Whitehorse"
+#     
+# YYC:
+#   city: Calgary
+#   county: ca
+#   matchers:
+#     - Calgary
+#     
+# YYG:
+#   city: "Charlottetown, PEI"
+#   country: ca
+#   matchers:
+#     - "Charlottetown"
+#     - "Prince Edward Island"
+#     - "P.E.I."
+# 
+# YYJ:
+#   city: "Victoria, BC"
+#   matchers:
+#     - "Victoria"
+# 
+# YYT:
+#   city: "Saint Johns, NL"
+#   matchers:
+#     - "Saint Johns"
+#     - "St. Johns"
+#     - "St.Johns"
+# 
+# YYZ:
+#   city: Toronto
+#   matchers:
+#     - "Toronto"
+# 
+# YZF:
+#   city: "Yellowknife, NT"
+#   matchers:
+#     - "Yellowknife"

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'airport_scraper'
+
+class Test::Unit::TestCase
+end

data/test/intl_airports_tests.yml

@@ -0,0 +1,2 @@
+AMS:
+  - "On board KLM Royal Dutch Airlines KL 3122 from PRG to AMS now #Flight"

data/test/test_airport_scraper.rb

@@ -0,0 +1,143 @@
+require 'helper'
+require 'yaml'
+
+CA_TESTS = YAML.load_file(File.join(File.dirname(__FILE__), "ca_airports_tests.yml"))
+US_TESTS = YAML.load_file(File.join(File.dirname(__FILE__), "us_airports_tests.yml"))
+INTL_TESTS = YAML.load_file(File.join(File.dirname(__FILE__), "intl_airports_tests.yml"))
+BAD_MATCHES = YAML.load_file(File.join(File.dirname(__FILE__), "bad_matches.yml"))
+
+class TestAirportScraper < Test::Unit::TestCase
+  context "new" do
+    setup do
+      @scrape = AirportScraper.new
+    end
+    
+    should "load the airports.yml file into @airports" do
+      airports = @scrape.airports
+      assert_not_nil airports
+      assert_not_nil airports['JFK'], "Didn't find JFK in airports"
+    end
+    
+    should_eventually "create an @code_match_regex to match 3-letter codes" do
+      code_regex = @scrape.instance_variable_get("@code_match_regex")
+      assert_not_nil(code_regex)
+      assert_match(code_regex, 'JFK')
+      assert_no_match(code_regex, 'JFKX')
+      assert_no_match(code_regex, 'jfk')
+    end
+    
+    should_eventually "create an @airport_regex" do
+      name_regex = @scrape.instance_variable_get("@airport_regex")
+      assert_not_nil(name_regex)
+      assert_match(name_regex, "Heathrow")
+      assert_match(name_regex, "heathrow")
+      assert_no_match(name_regex, "HeathrowX")
+    end
+    
+    should "create an @matcher_prefixes array" do
+      by_priority = @scrape.instance_variable_get("@matcher_prefixes")
+      assert_not_nil(by_priority)
+    end
+    
+    should "order @matcher_prefixes values in descending match_priority order" do
+      # Check that PWM comes before PDX
+      pref = @scrape.instance_variable_get("@matcher_prefixes")
+      by_priority = pref[@scrape.prefix_from_match("Portland")]
+      
+      assert_not_nil by_priority
+      pdx = by_priority.detect {|x| x['code'] == 'PDX'}
+      pwm = by_priority.detect {|x| x['code'] == 'PWM'}
+      
+      pdx_idx = by_priority.index(pdx)
+      pwm_idx = by_priority.index(pwm)
+      assert_not_nil pdx_idx
+      assert_not_nil pwm_idx
+      
+      assert(pwm_idx < pdx_idx)
+    end
+  end
+  
+  context "possible_flight?" do
+    setup do
+      @scrape = AirportScraper.new
+    end
+    
+    ["on a flight to Rome", "flying to SFO", "just touched down in Vegas", "EWR to NYC", "EWR -> NYC"].each do |phrase|
+      should "return true for the phrase '#{phrase}'" do
+        assert @scrape.possible_flight?(phrase)
+      end
+    end
+  end
+  
+  context "extract_airports" do
+    setup do
+      @scrape = AirportScraper.new
+    end
+    
+    context "when there are no airports in the text" do
+      should "return an empty_array" do
+        assert_equal [], @scrape.extract_airports("Twas brillig and the slithy toves")
+      end
+    end
+    
+    context "Airport code tests" do
+      setup do
+        @scrape = AirportScraper.new
+      end
+      
+      should_eventually "be able to match the airport codes" do
+        @scrape.airports.each do |airport|
+          assert_contains @scrape.extract_airports("Just landed in #{airport['code']}."), airport
+        end
+      end
+    end
+    
+    context "Freeform name test" do
+      [US_TESTS, CA_TESTS, INTL_TESTS].each do |tests|
+        tests.keys.each do |code|
+          tests[code].each do |str|
+            should "return the airport #{code} for phrase '#{str}'" do
+              airport = @scrape.airport(code)
+              results = @scrape.extract_airports(str)
+              assert_contains results, airport, "Expected #{code}, returned #{results.map {|x| x['code']}.inspect }"
+            end
+          end
+        end
+      end
+    end
+    
+    context "Matchers" do
+      setup do
+        @scape = AirportScraper.new
+      end
+      
+      should "not have duplicate matchers for two airports" do
+        matchers = {}
+        airports = @scrape.airports
+        
+        airports.values.each do |airport|
+          airport['matchers'].each do |matcher|
+            if matchers[matcher].nil?
+              matchers[matcher] = airport
+            else
+#              if matchers[matcher]['code']['match_priority'] == airport['code']['match_priority']
+                flunk "Matcher '#{matcher}' for more than one airport (#{matchers[matcher]['code']}, #{airport['code']}) at same priority"
+#              end
+            end
+          end
+        end
+      end
+      
+      should_eventually "not have shorter matchers for a name with a match_priority greater than a longer variant"
+    end
+    
+    context "Bad matches" do
+      BAD_MATCHES.each do |str|
+        should "not return any airports for phrase '#{str}'" do
+          results = @scrape.extract_airports(str)
+          assert_equal [], results, "Should not have matched anything, returned #{results.map {|x| x['code']}.inspect }"
+        end
+      end
+    end    
+  end
+end

data/test/us_airports_tests.yml

@@ -0,0 +1,71 @@
+ABQ:
+  - "W'sup Dallas? 2 hr flight to albuquerque. There for two days and then to colorado to see the brothers for a week :)"
+  - "Jus landed in new mexico...my 1st time here"
+  - "just landed in ABQ, new mexico! hello coooold!"
+
+BIL:
+  - "Making a quick run to SeaTac to drop off T@. She's flying to Billings but needs be in Bismark"
+
+CLT:
+  - "Landed in Charlotte..."
+
+CMH:
+  - "working a terrible shift, but flying to Columbus in the morning!"
+
+DAL:
+  - "On @SouthwestAir #2992 heading from LIT to DAL and CEO Gary Kelly is on board!"
+
+DEN:
+  - "Landed in Denver, now off to the terminal to await the arrival of my mother. I can't wait to consume mass amounts of turkey on Thursday."
+
+DFW:
+  - "Just landed in Dallas"
+
+EWR:
+  - "EWR Thanksgiving here we come. MIA to EWR to BDL"
+  - "just landed in NJ... its a gud look so far!!"
+
+FLL:
+  - "Landed in Ft. Lauderdale. Driving to Boca."
+
+JFK:
+  - "This little brat just spilled water on me during my 40 minute delayed flight to jfk. FML"
+    
+LAS:
+  - "Touched down in Vegas. Vegas baby!"
+  - "Landed in Vegas, watch out."
+
+LAX:
+  - "Just landed in LA sans DJ equip, but its good to be back."
+
+LGA:
+  - "kind of weird...Karlie Kloss was on my flight to LaGuardia today. haha I was tempted to get a picture with her, but she looked exhausted!"
+
+LIT:
+  - "On @SouthwestAir #2992 heading from LIT to DAL and CEO Gary Kelly is on board!"
+
+OAK:
+  - "flight to Oakland departs at 10:25. And I only just got on a shuttle from the parking lots. Here's hoping I don't miss my flight!"
+
+PDX:
+  - "On my flight to Portland. Looking forward to kicking back at home."
+  - "Landing in PDX now. See you all soon."
+
+PWM:
+  - "Landed in Portland ME"
+
+RDU:
+  - "We just landed in Raleigh. Now its time to pack, go to church, and leave tonight to go to Mom's for Thanksgiving!"
+
+SDF:
+  - "After leaving at 4:30am...just landed in Louisville. And yes, we arrived without a turkey smashing our windshield this year."
+
+SFO:
+  - "@ San Francisco Airport waiting for my flight to Delhi via Beijing.  Delayed by 3 hrs but knew that this morning thanks to Twitter."
+
+SEA:
+  - "On the same Virgin America flight to Seattle with Top Chef's Marcel whose hair was perfectly peaked even at 9 am"
+  - "Alaska/Horizon FTW - on an earlier flight to SeaTac!"
+
+STL:
+  - "Just landed in St.Louis, its raining outside smh I'm still going 2 enjoy my break"

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: airport_scraper
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jacob Harris
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-01 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: A gem for extracting airport codes from text
+email: jharris@nytimes.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/airport_scraper.rb
+- lib/ca_airports.yml
+- lib/intl_airports.yml
+- lib/us_airports.yml
+- script/console
+- test/bad_matches.yml
+- test/ca_airports_tests.yml
+- test/helper.rb
+- test/intl_airports_tests.yml
+- test/test_airport_scraper.rb
+- test/us_airports_tests.yml
+has_rdoc: true
+homepage: http://github.com/harrisj/airport_scraper
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A gem for extracting airports from mentions in text
+test_files: 
+- test/helper.rb
+- test/test_airport_scraper.rb