grubby 1.2.1 → 3.0.0

data/lib/grubby.rb

@@ -1,13 +1,10 @@
-require "active_support/all"
-require "casual_support"
+require "pathname"
+
 require "gorge"
 require "mechanize"
-require "mini_sanity"
-require "pleasant_path"
 require "ryoba"
 
 require_relative "grubby/version"
-require_relative "grubby/log"
 
 require_relative "grubby/core_ext/string"
 require_relative "grubby/core_ext/uri"
@@ -23,22 +20,37 @@ class Grubby < Mechanize
 
   VERSION = GRUBBY_VERSION
 
-  # The enforced minimum amount of time to wait between requests, in
-  # seconds.  If the value is a Range, a random number within the Range
-  # is chosen for each request.
+  class << self
+    # Logger used by Grubby.
+    #
+    # @return [Logger]
+    def logger
+      @logger ||= Logger.new($stderr).tap do |logger|
+        logger.formatter = -> (severity, time, progname, msg) do
+          "[#{time.strftime "%Y-%m-%d %H:%M:%S"}] #{severity} #{msg}\n"
+        end
+      end
+    end
+
+    attr_writer :logger
+  end
+
+  # The minimum amount of time enforced between requests, in seconds.
+  # If the value is a Range, a random number within the Range is chosen
+  # for each request.
   #
   # @return [Integer, Float, Range<Integer>, Range<Float>]
   attr_accessor :time_between_requests
 
   # Journal file used to ensure only-once processing of resources by
-  # {singleton} across multiple program runs.
+  # {fulfill} across multiple program runs.
   #
   # @return [Pathname, nil]
   attr_reader :journal
 
   # @param journal [Pathname, String]
   #   Optional journal file used to ensure only-once processing of
-  #   resources by {singleton} across multiple program runs.
+  #   resources by {fulfill} across multiple program runs
   def initialize(journal = nil)
     super()
 
@@ -74,26 +86,32 @@ class Grubby < Mechanize
   end
 
   # Sets the journal file used to ensure only-once processing of
-  # resources by {singleton} across multiple program runs.  Setting the
+  # resources by {fulfill} across multiple program runs.  Setting the
   # journal file will clear the in-memory list of previously-processed
   # resources, and, if the journal file exists, load the list from file.
   #
   # @param path [Pathname, String, nil]
   # @return [Pathname]
   def journal=(path)
-    @journal = path&.to_pathname&.touch_file
-    @seen = if @journal
-        require "csv"
-        CSV.read(@journal).map{|row| SingletonKey.new(*row) }.to_set
-      else
-        Set.new
+    if path
+      @journal = Pathname(path)
+      @journal.dirname.mkpath
+      require "csv"
+      @fulfilled = CSV.open(@journal, "a+") do |csv|
+        csv.each.map{|row| FulfilledEntry.new(*row) }.to_set
       end
+    else
+      @journal = nil
+      @fulfilled = Set.new
+    end
+
     @journal
   end
 
-  # Calls +#head+ and returns true if the result has response code
-  # "200".  Unlike +#head+, error response codes (e.g. "404", "500")
-  # do not cause a +Mechanize::ResponseCodeError+ to be raised.
+  # Calls +#head+ and returns true if a response code "200" is received,
+  # false otherwise.  Unlike +#head+, error response codes (e.g. "404",
+  # "500") do not result in a +Mechanize::ResponseCodeError+ being
+  # raised.
   #
   # @param uri [URI, String]
   # @return [Boolean]
@@ -106,7 +124,7 @@ class Grubby < Mechanize
   end
 
   # Calls +#get+ with each of +mirror_uris+ until a successful
-  # ("200 OK") response is recieved, and returns that +#get+ result.
+  # ("200 OK") response is received, and returns that +#get+ result.
   # Rescues and logs +Mechanize::ResponseCodeError+ failures for all but
   # the last mirror.
   #
@@ -114,13 +132,13 @@ class Grubby < Mechanize
   #   grubby = Grubby.new
   #
   #   urls = [
-  #     "http://httpstat.us/404",
-  #     "http://httpstat.us/500",
-  #     "http://httpstat.us/200#foo",
-  #     "http://httpstat.us/200#bar",
+  #     "https://httpstat.us/404",
+  #     "https://httpstat.us/500",
+  #     "https://httpstat.us/200?foo",
+  #     "https://httpstat.us/200?bar",
   #   ]
   #
-  #   grubby.get_mirrored(urls).uri  # == URI("http://httpstat.us/200#foo")
+  #   grubby.get_mirrored(urls).uri  # == URI("https://httpstat.us/200?foo")
   #
   #   grubby.get_mirrored(urls.take(2))  # raise Mechanize::ResponseCodeError
   #
@@ -137,84 +155,101 @@ class Grubby < Mechanize
       if i >= mirror_uris.length
         raise
       else
-        $log.debug("Mirror failed (code #{e.response_code}): #{mirror_uris[i - 1]}")
-        $log.debug("Try mirror: #{mirror_uris[i]}")
+        Grubby.logger.debug("Mirror failed (code #{e.response_code}): #{mirror_uris[i - 1]}")
+        Grubby.logger.debug("Try mirror: #{mirror_uris[i]}")
         retry
       end
     end
   end
 
   # Ensures only-once processing of the resource indicated by +uri+ for
-  # the specified +purpose+.  A list of previously-processed resource
-  # URIs and content hashes is maintained in the Grubby instance.  The
-  # given block is called with the fetched resource only if the
-  # resource's URI and the resource's content hash have not been
-  # previously processed under the specified +purpose+.
+  # the specified +purpose+.  The given block is executed and the result
+  # is returned if and only if the Grubby instance has not recorded a
+  # previous call to +fulfill+ for the same resource and purpose.
+  #
+  # Note that the resource is identified by both its URI and its content
+  # hash.  The latter prevents superfluous and rearranged URI query
+  # string parameters from interfering with only-once processing.
+  #
+  # If {journal} is set, and if the block does not raise an exception,
+  # the resource and purpose are logged to the journal file.  This
+  # enables only-once processing across multiple program runs.  It also
+  # provides a means to resume batch processing after an unexpected
+  # termination.
   #
   # @example
   #   grubby = Grubby.new
   #
-  #   grubby.singleton("https://example.com/foo") do |page|
-  #     # will be executed (first time "/foo")
+  #   grubby.fulfill("https://example.com/posts") do |page|
+  #     "first time"
   #   end
+  #   # == "first time"
   #
-  #   grubby.singleton("https://example.com/foo#bar") do |page|
-  #     # will be skipped (already seen "/foo")
+  #   grubby.fulfill("https://example.com/posts") do |page|
+  #     "already seen" # not evaluated
   #   end
+  #   # == nil
   #
-  #   grubby.singleton("https://example.com/foo", "again!") do |page|
-  #     # will be executed (new purpose for "/foo")
+  #   grubby.fulfill("https://example.com/posts?page=1") do |page|
+  #     "already seen content hash" # not evaluated
   #   end
+  #   # == nil
+  #
+  #   grubby.fulfill("https://example.com/posts", "again!") do |page|
+  #     "already seen, but new purpose"
+  #   end
+  #   # == "already seen, but new purpose"
   #
   # @param uri [URI, String]
   # @param purpose [String]
-  # @yield [resource]
   # @yieldparam resource [Mechanize::Page, Mechanize::File, Mechanize::Download, ...]
-  # @return [Boolean]
-  #   whether the given block was called
+  # @yieldreturn [Object]
+  # @return [Object, nil]
   # @raise [Mechanize::ResponseCodeError]
   #   if fetching the resource results in error (see +Mechanize#get+)
-  def singleton(uri, purpose = "")
+  def fulfill(uri, purpose = "")
     series = []
 
     uri = uri.to_absolute_uri
-    return if try_skip_singleton(uri, purpose, series)
+    return unless add_fulfilled(uri, purpose, series)
 
     normalized_uri = normalize_uri(uri)
-    return if try_skip_singleton(normalized_uri, purpose, series)
+    return unless add_fulfilled(normalized_uri, purpose, series)
 
-    $log.info("Fetch #{normalized_uri}")
+    Grubby.logger.info("Fetch #{normalized_uri}")
     resource = get(normalized_uri)
-    skip = try_skip_singleton(resource.uri, purpose, series) |
-      try_skip_singleton("content hash: #{resource.content_hash}", purpose, series)
+    unprocessed = add_fulfilled(resource.uri, purpose, series) &
+      add_fulfilled("content hash: #{resource.content_hash}", purpose, series)
 
-    yield resource unless skip
+    result = yield resource if unprocessed
 
     CSV.open(journal, "a") do |csv|
-      series.each{|singleton_key| csv << singleton_key }
+      series.each{|entry| csv << entry }
     end if journal
 
-    !skip
+    result
   end
 
 
   private
 
   # @!visibility private
-  SingletonKey = Struct.new(:purpose, :target)
+  FulfilledEntry = Struct.new(:purpose, :target)
 
-  def try_skip_singleton(target, purpose, series)
-    series << SingletonKey.new(purpose, target.to_s)
-    if series.uniq!.nil? && !@seen.add?(series.last)
-      seen_info = series.length > 1 ? "seen #{series.last.target}" : "seen"
-      $log.info("Skip #{series.first.target} (#{seen_info})")
+  def add_fulfilled(target, purpose, series)
+    series << FulfilledEntry.new(purpose, target.to_s)
+    if (series.uniq!) || @fulfilled.add?(series.last)
       true
+    else
+      Grubby.logger.info("Skip #{series.first.target}" \
+        " (seen#{" #{series.last.target}" unless series.length == 1})")
+      false
     end
   end
 
   def normalize_uri(uri)
     uri = uri.dup
-    $log.warn("Ignore ##{uri.fragment} in #{uri}") if uri.fragment
+    Grubby.logger.warn("Ignore ##{uri.fragment} in #{uri}") if uri.fragment
     uri.fragment = nil
     uri.path = uri.path.chomp("/")
     uri

metadata

@@ -1,55 +1,54 @@
 --- !ruby/object:Gem::Specification
 name: grubby
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Jonathan Hefner
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: cgi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: casual_support
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: gorge
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
@@ -66,113 +65,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
-- !ruby/object:Gem::Dependency
-  name: mini_sanity
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: pleasant_path
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: ryoba
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.15'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.15'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: yard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-description: 
 email:
 - jonathan@hefner.pro
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -184,7 +96,6 @@ files:
 - lib/grubby/core_ext/uri.rb
 - lib/grubby/json_parser.rb
 - lib/grubby/json_scraper.rb
-- lib/grubby/log.rb
 - lib/grubby/mechanize/download.rb
 - lib/grubby/mechanize/fetch_with_retry.rb
 - lib/grubby/mechanize/file.rb
@@ -197,8 +108,9 @@ files:
 homepage: https://github.com/jonathanhefner/grubby
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  source_code_uri: https://github.com/jonathanhefner/grubby
+  changelog_uri: https://github.com/jonathanhefner/grubby/blob/master/CHANGELOG.md
 rdoc_options: []
 require_paths:
 - lib
@@ -206,15 +118,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key: 
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Fail-fast web scraping
 test_files: []

data/.gitignore

@@ -1,9 +0,0 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.2.5
-before_install: gem install bundler -v 1.15.1

data/lib/grubby/log.rb

@@ -1,5 +0,0 @@
-$log ||= Logger.new($stderr).tap do |logger|
-  logger.formatter = ->(severity, datetime, progname, msg) do
-    "[#{datetime.to_ymd} #{datetime.to_hms}] #{severity} #{msg}\n"
-  end
-end