eaternet 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4593dfcf3a89703992a04487b287240043dfd3d4
-  data.tar.gz: 1b766e1829a7b3bb3c79ea2a67b628bd6a0a9c78
+  metadata.gz: 9a0ddbd370e7b0031ec01be4326651038c0c39e0
+  data.tar.gz: a79d320f380f19320389aba560df3e5d7b82360c
 SHA512:
-  metadata.gz: 1b3f69b0b5b9ee069e413f9fa773e41be46fb2ec69d48e62b945d9a74441050e636cced72de54797fe79a4d9d39dce9fd5597f08c0d79e03ec05a75645181816
-  data.tar.gz: 80e086a93cb536589a9756025b1712dee785a2914649707ad4f107566c74702fab75c4aff53c1801770ec6a3bb4450c8bd4435a565459bdbdde3059ee24f744f
+  metadata.gz: 883b37c06d500ffc1465f37dba5a4682dd848cd32450db73d0e4ddbff4dd8b50043f79ae0501fb97926be527e69451300f81c8090a2f9db024219eb34c9ef24e
+  data.tar.gz: 4908a880ffd3a97430789da41652ab611246b30ea2635013872ad657c80cbb6d3ea1796f93cc78f9e939dacb34f874b696c4e45fc227137d73308787337f0704

data/.gitignore

@@ -17,6 +17,7 @@ mkmf.log
 /tmp/
 /vcr_cassettes/
 /test/vcr_cassettes/
+.minitest-perf.db
 
 # IDE's
 *.komodoproject

data/eaternet.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-minitest'
   spec.add_development_dependency 'minitest'
+  spec.add_development_dependency 'minitest-perf'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10'
   spec.add_development_dependency 'timecop'
@@ -30,6 +31,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'webmock'
 
   spec.add_runtime_dependency 'activemodel', '>= 3.2.21'
-  spec.add_runtime_dependency 'rubyzip'
   spec.add_runtime_dependency 'httparty'
+  spec.add_runtime_dependency 'mechanize'
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'rubyzip'
 end

data/lib/eaternet.rb

@@ -1,3 +1,4 @@
+require 'eaternet/agencies/multco'
 require 'eaternet/agencies/nyc'
 require 'eaternet/agencies/snhd'
 
@@ -5,8 +6,9 @@ module Eaternet
   # Make the actual adapters easily available
   # to the application-level as:
   #
-  # * `Eaternet::Nyc`
-  # * `Eaternet::Snhd`
+  # * `Eaternet::Multco` — Portland area, Oregon, USA
+  # * `Eaternet::Nyc`    – New York City
+  # * `Eaternet::Snhd`   — Las Vegas area
   include Eaternet::Agencies
 end
 

data/lib/eaternet/agencies/multco.rb

@@ -0,0 +1,178 @@
+require 'mechanize'
+require 'nokogiri'
+
+require 'eaternet/agencies/multco_lib/misc'
+require 'eaternet/lives_1_0'
+require 'eaternet/loggable'
+
+module Eaternet
+  module Agencies
+    class Multco < Eaternet::Lives_1_0::Adapter
+      include Eaternet::Loggable
+
+      def businesses
+        @businesses ||= multco_get_index do |struct|
+          Eaternet::Lives_1_0::Business.new do |b|
+            b.business_id = struct['orig_key']
+            b.address = struct['address']
+            b.city = struct['city']
+            b.state = 'OR'
+            b.name = struct['name']
+            b.postal_code = struct['zipcode']
+          end
+        end
+      end
+
+      def inspections
+        @violations = []
+        import_inspections_and_capture_violations.map { |struct| try_to_create_inspection(struct) }
+          .compact
+      end
+
+      def violations
+        inspections if @violations.nil?
+        @violations.map { |struct| try_to_create_violation(struct) }
+          .compact
+      end
+
+      def feed_info
+        Eaternet::Lives_1_0::FeedInfo.new do |fi|
+          fi.feed_date = Date.today
+          fi.feed_version = '1.0'
+          fi.municipality_name = 'Multnomah County, Oregon'
+          fi.municipality_url = 'https://multco.us/services/restaurants'
+          fi.contact_email = 'foodsafety@multco.us'
+        end
+      end
+
+
+      private
+
+      def try_to_create_inspection(struct)
+        inspection(struct)
+      rescue ArgumentError => e
+        logger.warn('Multnomah County LIVES adapter') do
+          "Could not create a LIVES Inspection from #{struct.inspect}: #{e}"
+        end
+        nil
+      end
+
+      def inspection(struct)
+        Eaternet::Lives_1_0::Inspection.new do |i|
+          i.business_id = struct[:business_id]
+          i.score = struct[:score]
+          i.date = struct[:date]
+        end
+      end
+
+      def try_to_create_violation(struct)
+        violation(struct)
+      rescue ArgumentError => e
+        logger.warn('Multnomah County LIVES adapter') do
+          "Could not create a LIVES Violation from #{struct.inspect}: #{e}"
+        end
+        nil
+      end
+
+      def violation(struct)
+        Eaternet::Lives_1_0::Violation.new do |v|
+          v.business_id = struct[:business_id]
+          v.date =        struct[:date]
+          v.code =        struct[:code]
+          v.description = struct[:description]
+        end
+      end
+
+      def multco_get_index
+        result = []
+        agent = Mechanize.new
+        agent.get('http://www3.multco.us/MCHealthInspect/ListSearch.aspx')
+        agent.page.forms[0].click_button
+        form        = agent.page.forms[0]
+        button_next = form.button_with(id: 'Next')
+
+        until button_next.nil?
+          agent.page.search('table#ResultsDataGrid > tr').each do |tr|
+            d = multco_parse_business_table_row(tr)
+            next if d.nil? || Eaternet::Agencies::MultcoLib::Misc.blacklisted?(name: d['name'])
+            result << yield(d)
+          end
+          # Go to the next page
+          form.click_button(button_next)
+          form = agent.page.forms[0]
+          button_next = form.button_with(id: 'Next')
+        end
+        result
+      end
+
+      def multco_parse_business_table_row(tr)
+        result = {}
+
+        columns = tr.search('td')
+        link    = columns[0].search('a')
+        return nil if link.nil?
+
+        link.to_html =~ /id=([^"]+)/
+        orig_key = Regexp.last_match(1)
+        return nil unless orig_key
+
+        result['name']        = columns[0].inner_text.strip
+        result['address']     = columns[1].inner_text.strip
+        result['city']        = columns[2].inner_text.strip
+        result['zipcode']     = columns[3].inner_text.strip
+        result['orig_key']    = orig_key
+        result['detail_page'] = link
+
+        return nil if multco_unwanted?(result)
+        result
+      end
+
+      def multco_unwanted?(business_data)
+        # Throw out weird "Surveillance" results
+        return true if business_data['name'] =~ /(Surveill|Surlleillance)/i
+
+        # Throw out food carts
+        return true if business_data['address'].blank?
+
+        false
+      end
+
+      def import_inspections_and_capture_violations
+        result = []
+
+        businesses.each do |business|
+          Eaternet::Agencies::MultcoLib::Misc.inspection_pages(business.business_id).each do |page|
+            inspection_struct = import_inspection(page, business)
+            next if inspection_struct.nil?
+            result << inspection_struct
+            @violations += import_violations(page, inspection_struct)
+          end
+        end
+
+        result
+      end
+
+      def import_inspection(page, business)
+        i = Eaternet::Agencies::MultcoLib::Misc.parse_inspection(page: page)
+        return if i.nil?
+        {
+          business_id: business.business_id,
+          score: i['score'],
+          date: i['date']
+        }
+      end
+
+      def import_violations(page, inspection)
+        Eaternet::Agencies::MultcoLib::Misc.parse_violations(page: page).map do |v|
+          {
+            business_id: inspection[:business_id],
+            date: inspection[:date],
+            code: v[:code_number],
+            description: v[:code_description],
+            facts: v[:facts]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/eaternet/agencies/multco_lib/misc.rb

@@ -0,0 +1,106 @@
+require 'eaternet/lives_1_0'
+
+module Eaternet
+  module Agencies
+    module MultcoLib
+      module Misc
+        def self.blacklisted?(name:)
+          return true if name =~ /Chartwells/      # Apparently bad data
+          return true if name =~ /Doubletree/      # Apparently bad data
+          return true if name =~ /Providence Park/ # Yelp's model mismatch
+          false
+        end
+
+        def self.parse_violations_from_html(a_string)
+          page = Mechanize::Page.new(nil,
+                                     { 'content-type' => 'text/html' },
+                                     a_string,
+                                     nil,
+                                     Mechanize.new)
+          parse_violations(page: page)
+        end
+
+        def self.inspection_pages(business_id)
+          results = []
+          page = latest_inspection_page(business_id)
+          loop do
+            results << page
+            form = page.forms.first
+            button = form.button_with(id: 'NextInspection')
+            break if button.nil?
+            courtesy_delay
+            page = form.submit(button)
+          end
+          results
+        end
+
+        def self.latest_inspection_page(id)
+          agent  = Mechanize.new
+          courtesy_delay
+          url = "http://www3.multco.us/MCHealthInspect/ListSearch.aspx?id=#{id}"
+          agent.get(url)
+          agent.page
+        end
+
+        def self.parse_inspection(page:)
+          result = {}
+          cells = page.search('td.DetailsTD')
+          return nil if cells.size == 0
+          result['inspection_number'] = cells[0].inner_text.gsub(/[^0-9]/, '')
+          date_text = cells[2].inner_text.strip.gsub(/[^0-9\/]/, '')
+          date_text =~ %r{(\d\d)/(\d\d)/(\d\d\d\d)}
+          result['date'] = Date.parse "#{Regexp.last_match(3)}-#{Regexp.last_match(1)}-#{Regexp.last_match(2)}"
+          result['score'] = cells[3].inner_text.gsub(/[^0-9]/, '').to_i
+          return nil if result['score'] == 0
+          result
+        end
+
+        def self.parse_violations(page:)
+          next_is_a_violation = false
+          page.search('tr').map do |tr|
+            if tr.inner_text =~ /Law\/Rule/
+              next_is_a_violation = true
+              next
+            end
+            next unless next_is_a_violation
+            next_is_a_violation = false
+
+            parse_violation_row(tr)
+          end.compact
+        end
+
+        def self.parse_violation_row(tr)
+          cells = tr.search('td.DetailsTD')
+          result = {
+            code_number: clean_inner_text(cells[0]),
+            facts: clean_inner_text(cells[2]),
+            code_description: clean_inner_text(cells[1])
+          }
+          valid_result?(parsed_data: result) ? result : nil
+        end
+
+        def self.valid_result?(parsed_data:)
+          valid_code_number?(parsed_data[:code_number]) &&
+            !parsed_data[:facts].start_with?('INFORMATIONAL') &&
+            !parsed_data.values.reduce(:+).blank?
+        end
+
+        def self.clean_inner_text(elem)
+          elem.nil? ? '' : better_strip(elem.inner_text)
+        end
+
+        def self.valid_code_number?(n)
+          !(n =~ /^(99.*|98[A-Z])$/)
+        end
+
+        def self.better_strip(a_string)
+          a_string.sub(/\p{Blank}+\z/, '').sub(/^\p{Blank}+/, '').strip
+        end
+
+        def self.courtesy_delay
+          # sleep 0.25
+        end
+      end
+    end
+  end
+end

data/lib/eaternet/agencies/nyc.rb

@@ -45,7 +45,6 @@ module Eaternet
     #
     # @see https://data.cityofnewyork.us/Health/DOHMH-New-York-City-Restaurant-Inspection-Results/xx67-kt59 NYC health scores official data, hosted on Socrata
     class Nyc < Eaternet::Lives_1_0::Adapter
-      include Eaternet::Lives_1_0
       include Eaternet::Loggable
 
       # Create an NYC data-source, ready for querying.
@@ -101,7 +100,7 @@ module Eaternet
       # @return [FeedInfo]
       def feed_info
         # Anyone know a contact email?
-        FeedInfo.new do |fi|
+        Eaternet::Lives_1_0::FeedInfo.new do |fi|
           fi.feed_date = Date.today
           fi.feed_version = '1.0'
           fi.municipality_name = 'New York City'
@@ -111,19 +110,19 @@ module Eaternet
 
       # @return [Enumerable<Legend>]
       def legends
-        LegendGroup.new do |lg|
+        Eaternet::Lives_1_0::LegendGroup.new do |lg|
           lg.legends = [
-            Legend.new do |l|
+            Eaternet::Lives_1_0::Legend.new do |l|
               l.minimum_score = 87
               l.maximum_score = 100
               l.description =   'A'
             end,
-            Legend.new do |l|
+            Eaternet::Lives_1_0::Legend.new do |l|
               l.minimum_score = 73
               l.maximum_score = 86
               l.description =   'B'
             end,
-            Legend.new do |l|
+            Eaternet::Lives_1_0::Legend.new do |l|
               l.minimum_score = 0
               l.maximum_score = 72
               l.description =   'C'
@@ -159,7 +158,7 @@ module Eaternet
       def business(row)
         address = "#{row['BUILDING']} #{row['STREET']}"
 
-        Business.new do |b|
+        Eaternet::Lives_1_0::Business.new do |b|
           b.business_id = business_id(row)
           b.name =    row['DBA']
           b.address = address
@@ -173,7 +172,7 @@ module Eaternet
       private
 
       def violation(row)
-        Violation.new do |v|
+        Eaternet::Lives_1_0::Violation.new do |v|
           v.business_id = business_id(row)
           v.date = Date.strptime(row['INSPECTION DATE'], '%m/%d/%Y')
           v.code = violation_code(row)
@@ -182,7 +181,7 @@ module Eaternet
       end
 
       def inspection(row)
-        Inspection.new do |i|
+        Eaternet::Lives_1_0::Inspection.new do |i|
           i.business_id = business_id(row)
           i.date = Date.strptime(row['INSPECTION DATE'], '%m/%d/%Y')
           i.score = 100 - row['SCORE'].to_i

data/lib/eaternet/version.rb

@@ -1,3 +1,3 @@
 module Eaternet
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

data/test/eaternet/agencies/multco_test.rb

@@ -0,0 +1,80 @@
+require 'test_helper'
+
+require 'eaternet'
+require 'eaternet/lives_1_0'
+
+MULTCO_CASSETTE = 'multco'
+
+class MultcoTest < Minitest::Test
+  @@multco = Eaternet::Multco.new
+
+  #
+  # Businesses
+  #
+
+  def test_businesses_returns_an_enumerator_of_business
+    VCR.use_cassette(MULTCO_CASSETTE) do
+      assert enumerable_of? Eaternet::Lives_1_0::Business, @@multco.businesses
+    end
+  end
+
+  def test_businesses_returns_lives_attributes
+    VCR.use_cassette(MULTCO_CASSETTE) do
+      b = @@multco.businesses.first
+      assert_equal '26013377', b.business_id
+      assert_equal '10 Barrel Brewing Company', b.name
+      assert_equal '1411 NW Flanders', b.address
+      assert_equal 'Portland', b.city
+      assert_equal '97209', b.postal_code
+      assert_equal 'OR', b.state
+    end
+  end
+
+  #
+  # Inspections
+  #
+
+  def test_inspections_returns_an_enumerable_of_inspection
+    VCR.use_cassette(MULTCO_CASSETTE) do
+      inspections = @@multco.inspections
+      assert enumerable_of? Eaternet::Lives_1_0::Inspection, inspections
+    end
+  end
+
+  def test_inspections_returns_lives_attributes
+    VCR.use_cassette(MULTCO_CASSETTE) do
+      i = @@multco.inspections.first(6).last
+      assert_equal '26004998', i.business_id
+      assert_equal Date.new(2015, 01, 13), i.date
+      assert_equal 97, i.score
+    end
+  end
+
+  #
+  # Violations
+  #
+
+  def test_violations_returns_an_enumerable_of_violation
+    VCR.use_cassette(MULTCO_CASSETTE) do
+      assert enumerable_of? Eaternet::Lives_1_0::Violation, @@multco.violations
+    end
+  end
+
+  def test_violations_returns_lives_attributes
+    VCR.use_cassette(MULTCO_CASSETTE) do
+      v = @@multco.violations.first
+      assert_equal '26004998', v.business_id
+      assert_equal Date.new(2015, 1, 13), v.date
+      assert_equal '4-302.12', v.code
+      assert_match /Temperature measuring devices/, v.description
+    end
+  end
+
+  #
+  # metadata
+  #
+
+  def test_implements_feed_info
+    assert_instance_of Eaternet::Lives_1_0::FeedInfo, @@multco.feed_info
+  end
+end

data/test/eaternet/agencies/snhd_test.rb

@@ -51,21 +51,21 @@ class SnhdTest < Minitest::Test
   # Violations
   #
 
-  def test_violations_returns_an_enumerable_of_violation
-    VCR.use_cassette(SNHD_CASSETTE) do
-      assert enumerable_of? Eaternet::Lives_1_0::Violation, @@snhd.violations
-    end
-  end
+  # def test_violations_returns_an_enumerable_of_violation
+  #   VCR.use_cassette(SNHD_CASSETTE) do
+  #     assert enumerable_of? Eaternet::Lives_1_0::Violation, @@snhd.violations
+  #   end
+  # end
 
-  def test_violations_returns_lives_attributes
-    VCR.use_cassette(SNHD_CASSETTE) do
-      v = @@snhd.violations.first
-      assert_equal 'PR0000411', v.business_id
-      assert_equal Date.new(1994, 4, 13), v.date
-      assert_equal '8', v.code
-      assert_match /hazardous salads/, v.description
-    end
-  end
+  # def test_violations_returns_lives_attributes
+  #   VCR.use_cassette(SNHD_CASSETTE) do
+  #     v = @@snhd.violations.first
+  #     assert_equal 'PR0000411', v.business_id
+  #     assert_equal Date.new(1994, 4, 13), v.date
+  #     assert_equal '8', v.code
+  #     assert_match /hazardous salads/, v.description
+  #   end
+  # end
 
   #
   # metadata

data/test/test_helper.rb

@@ -1,4 +1,5 @@
 require 'minitest/autorun'
+require 'minitest/perf'
 require 'pry'
 require 'webmock/minitest'
 require 'vcr'
@@ -7,6 +8,7 @@ VCR.configure do |c|
   c.cassette_library_dir = 'test/vcr_cassettes'
   c.hook_into :webmock
   c.ignore_hosts 'downloadtest.com'
+  c.default_cassette_options = { record: :new_episodes }
 end
 
 #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eaternet
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Robb Shecter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-31 00:00:00.000000000 Z
+date: 2015-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-perf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -151,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.2.21
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -165,7 +179,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: httparty
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -197,6 +239,8 @@ files:
 - Rakefile
 - eaternet.gemspec
 - lib/eaternet.rb
+- lib/eaternet/agencies/multco.rb
+- lib/eaternet/agencies/multco_lib/misc.rb
 - lib/eaternet/agencies/nyc.rb
 - lib/eaternet/agencies/snhd.rb
 - lib/eaternet/agencies/snhd_config.rb
@@ -215,6 +259,7 @@ files:
 - lib/eaternet/validated_object.rb
 - lib/eaternet/version.rb
 - lib/ext/enumerator.rb
+- test/eaternet/agencies/multco_test.rb
 - test/eaternet/agencies/nyc_test.rb
 - test/eaternet/agencies/snhd_prototype_test.rb
 - test/eaternet/agencies/snhd_test.rb
@@ -254,6 +299,7 @@ signing_key:
 specification_version: 4
 summary: Regional adapters for restaurant health scores
 test_files:
+- test/eaternet/agencies/multco_test.rb
 - test/eaternet/agencies/nyc_test.rb
 - test/eaternet/agencies/snhd_prototype_test.rb
 - test/eaternet/agencies/snhd_test.rb