eaternet 0.3.11 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11257485cfdcbda82f42ae36c646ebfc9616203d
-  data.tar.gz: 9f3bbe15c16db1bc73527cbb25aa69c4421ea630
+  metadata.gz: cfd0797bd4c249ea732ef25485f7310ee1d9ba74
+  data.tar.gz: 84e4914a0375a5c84b1d8a48196a19a4f06053ec
 SHA512:
-  metadata.gz: 9bd3634278276974dcecb89c48f1285ff652245dd968d8be8349b7063b853c8acfa6af8c5e7fcea6bb2703c50e4dcd3e2cf993784ff5eff92578f8f327de9e4c
-  data.tar.gz: 2484a3f65c718bd457b6ee56a6c40bb64aa4326950723ee59198c1b05f52f336bde72f3b1035c246638bb606483023c832a81573d5c0bb48e9ae47e4a47cbbf4
+  metadata.gz: 12ba76b4684ae01e31e2b97a8e0b18c6cace0a7cebc381ef4550a8ad13d03784982f8f660064de0f849caac930db81365b65f6d096e77302471deff71d041611
+  data.tar.gz: 5b207a3fb1c0520bf9721a156a36dc008ac1abfa3c592ede5a57548a9f6e18ca5024f440fa6664ff86c23a93b3cd1e5d7735419bc6b728f4cf4c4e1a5b0679df

data/eaternet.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'webmock'
 

data/lib/eaternet/agencies/nyc.rb

@@ -1,7 +1,6 @@
 require 'csv'
 require 'json'
 require 'logger'
-require 'open-uri'
 require 'set'
 require 'tempfile'
 
@@ -14,8 +13,9 @@ require 'eaternet/loggable'
 module Eaternet
   module Agencies
     # A data source for New York City food service health inspections.  It
-    # retrieves the latest info from the official source and makes it easy
-    # to work with.
+    # retrieves the latest info from
+    # [the official source](https://data.cityofnewyork.us/Health/DOHMH-New-York-City-Restaurant-Inspection-Results/xx67-kt59)
+    # and makes it easy to work with.
     #
     # This library conforms to the
     # [LIVES 1.0 standard](http://www.yelp.com/healthscores) developed by
@@ -66,7 +66,6 @@ module Eaternet
       end
 
       # @example Compute the average inspection score for NYC.
-      #
       #   # The library is optimized for memory use at the expense
       #   # of speed. E.g., each call to #inspections will iterate
       #   # through the raw CSV. So here, we first retrieve the
@@ -104,6 +103,7 @@ module Eaternet
       #
       # @return [FeedInfo]
       def feed_info
+        # Anyone know a contact email?
         FeedInfo.new do |fi|
           fi.feed_date = Date.today
           fi.feed_version = '1.0'
@@ -233,29 +233,17 @@ module Eaternet
           .map { |row| block.call(row) }
       end
 
-      def unique(objects)
-        Set.new(objects)
-      end
-
       def table_file
-        if @table_file.nil?
-          @table_file = Tempfile.new('all.csv.')
-          Nyc.download_to(@table_file)
-        end
-        @table_file
-      end
-
-      def self.download_to(a_file)
-        download_via_url(a_file)
+        @table_file ||= Nyc.download_via_url
       end
 
       # Fastest method for downloading all data but may be non-standard
       # for Socrata.
-      def self.download_via_url(a_file)
-        a_file.write(open(csv_url).read)
+      def self.download_via_url
+        Eaternet::Util.download_and_cache(source: csv_url, dest: Tempfile.new('nyc'))
       end
 
-      def csv_url
+      def self.csv_url
         domain = 'data.cityofnewyork.us'
         dataset = 'xx67-kt59'
         "https://#{domain}/api/views/#{dataset}/rows.csv?accessType=DOWNLOAD"

data/lib/eaternet/loggable.rb

@@ -12,6 +12,7 @@ module Eaternet
       logger = Logger.new(ENV['EATERNET_LOG_FILE'] || $stderr)
       logger.datetime_format = '%Y-%m-%d %H:%M:%S'
       logger.progname = 'Eaternet'
+      logger.level = Logger::ERROR
       logger
     end
   end

data/lib/eaternet/util.rb

@@ -1,3 +1,4 @@
+require 'base64'
 require 'open-uri'
 require 'zip'
 
@@ -22,12 +23,24 @@ module Eaternet
       dir
     end
 
+    # Download a file from the network, caching it for 12 hours.
+    #
+    # @param [String] source the URL to retrieve
+    # @param [String] dest pathname in which to save the file
+    def self.download_and_cache(source:, dest:)
+      cache_path = generate_cache_path(source)
+      download(source: source, dest: cache_path) if expired? cache_path
+      FileUtils.cp cache_path, dest
+      File.open dest
+    end
+
     # Download a file from the network.
     #
     # @param [String] source the URL to retrieve
     # @param [String] dest pathname in which to save the file
+    # @return [File] the file
     def self.download(source:, dest:)
-      open(dest, 'wb') { |file| file << open(source).read }
+      File.open(dest, 'wb') { |file| file << open(source).read }
     end
 
     # Extract a Zip archive.
@@ -56,5 +69,29 @@ module Eaternet
       return nil if a_string.nil?
       a_string.strip.gsub(/ +/, ' ')
     end
+
+    def self.file_age_in_days(path)
+      (Time.now - File.mtime(path)).to_i / 86_400.0
+    end
+
+    #
+    # private
+    #
+
+    def self.expired?(cache_path)
+      !File.exist?(cache_path) || file_age_in_days(cache_path) > 0.5
+    end
+
+    def self.generate_cache_path(url)
+      cache_dir = prepare_cache_dir
+      cache_key = Base64.strict_encode64(url)
+      File.join(cache_dir, cache_key)
+    end
+
+    def self.prepare_cache_dir
+      cache_dir = '/tmp/eaternet'
+      `mkdir -p #{cache_dir}`
+      cache_dir
+    end
   end
 end

data/lib/eaternet/version.rb

@@ -1,3 +1,3 @@
 module Eaternet
-  VERSION = '0.3.11'
+  VERSION = '0.3.12'
 end

data/test/eaternet/loggable_test.rb

@@ -9,7 +9,7 @@ class LoggableTest < Minitest::Test
 
   def test_creates_and_sends_log_messages_to_stderr
     assert_output(nil, /Border Collie/) do
-      logger.info('Border Collie')
+      logger.error('Border Collie')
     end
   end
 end

data/test/eaternet/util_test.rb

@@ -0,0 +1,47 @@
+require 'tempfile'
+require 'test_helper'
+require 'timecop'
+
+require 'eaternet/util'
+
+class UtilTest < Minitest::Test
+
+  def setup
+    @file_contents = 'This is the file'
+    @url = "http://downloadtest.com/file-#{rand(1_000_000)}.txt"
+    @file = Tempfile.new('temp')
+    stub_request(:get, @url).to_return(body: @file_contents)
+  end
+
+  def download
+    Eaternet::Util.download(source: @url, dest: @file)
+  end
+
+  def download_and_cache
+    Eaternet::Util.download_and_cache(source: @url, dest: @file)
+  end
+
+  def test_downloads_a_file
+    download
+    assert_equal @file_contents, `cat #{@file.path}`.strip
+    assert_requested :get, @url, times: 1
+  end
+
+  def test_downloads_a_file_with_cache_option
+    download_and_cache
+    assert_equal @file_contents, `cat #{@file.path}`.strip
+    assert_requested :get, @url, times: 1
+  end
+
+  def test_uses_a_file_cache
+    (1..3).each { download_and_cache }
+    assert_requested :get, @url, times: 1
+  end
+
+  def test_expires_a_cache_file
+    thirteen_hours = 13 * 60 * 60
+    download_and_cache
+    Timecop.freeze(Time.now + thirteen_hours) { download_and_cache }
+    assert_requested :get, @url, times: 2
+  end
+end

data/test/test_helper.rb

@@ -1,13 +1,14 @@
 require 'minitest/autorun'
 require 'pry'
+require 'webmock/minitest'
 require 'vcr'
 
 VCR.configure do |c|
   c.cassette_library_dir = 'test/vcr_cassettes'
   c.hook_into :webmock
+  c.ignore_hosts 'downloadtest.com'
 end
 
-
 #
 # Testing helpers
 #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eaternet
 version: !ruby/object:Gem::Version
-  version: 0.3.11
+  version: 0.3.12
 platform: ruby
 authors:
 - Robb Shecter
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -194,6 +208,7 @@ files:
 - test/eaternet/lives_1_0/inspection_test.rb
 - test/eaternet/lives_1_0/legend_test.rb
 - test/eaternet/loggable_test.rb
+- test/eaternet/util_test.rb
 - test/fixtures/morris-park-bake-shop.csv
 - test/script.rb
 - test/test_helper.rb
@@ -230,6 +245,7 @@ test_files:
 - test/eaternet/lives_1_0/inspection_test.rb
 - test/eaternet/lives_1_0/legend_test.rb
 - test/eaternet/loggable_test.rb
+- test/eaternet/util_test.rb
 - test/fixtures/morris-park-bake-shop.csv
 - test/script.rb
 - test/test_helper.rb