spn2 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ec6088e94bed925212bc5aa5abf3211cae5fd0bf5402c49f316d7e4d65f051d
-  data.tar.gz: e00cad0b9921f36052ec81271e69f553979dec1e0b6cd540f1bee1c0f712f337
+  metadata.gz: d73702950316690d62ce910059788093af0ba31b094c1208eb7c7ef72cbaa06e
+  data.tar.gz: 87ba2a61a9f76d86d46043da74418b2a67f6ac584dac2c3b50b2aa7baa8c7aad
 SHA512:
-  metadata.gz: e618df35304407811745df2e7c92106669a01564d4c61c1639fa06a109ffb995fb05e145e12d13efc7cf30fd4c899a959138e0640894bc68c3942efd1035e3d0
-  data.tar.gz: cc5616e4848cd9f5bfceb58e163095f9adfc86e584857d3c5c9674bfbbe8a182262d7c779c7d24a87134e7959e7f3a6fcbfc4588f039efc5184919e193d70ca2
+  metadata.gz: 0ad8a32dc48bd5dbaf5b24428f6a3522c4af9ce3a5a2297df39fb4dd9b36a627565a844cf841ca50e3c3e0200b569c88d73bd27717468ed8b8e2f5444114d316
+  data.tar.gz: b08014dddc62d499a30e5bc8ba1ce20c96a5f01ad8d0d465759897904f297339a9b79ecff1fdd56561c194072e24ba83542b5d0e39f6d6ab8e57a3b55bae8f6d

data/.rubocop.yml

@@ -8,6 +8,3 @@ AllCops:
 
 Style/HashSyntax:
   Enabled: false  # yuk Ruby 3.1
-
-Style/NumericLiterals:
-  Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,17 @@
-## [Unreleased]
-
 ## [0.1.0] - 2022-06-29
 
 - Initial release
+
+## [0.1.1] - 2022-06-30
+
+- Add error handling
+- Add ability to add opts to Spn2.save
+
+## [0.1.2] - 2022-07-02
+
+- Add user_status
+- Add status calls for multiple job_ids and outlinks
+
+## [0.2.0] - 2022-07-03
+
+- Breaking change: Single method 'status' with kwarg :job_ids now used for status of job(s)

data/Guardfile

@@ -4,4 +4,5 @@ guard :minitest do
   watch(%r{^test/(.*)/?test_(.*)\.rb$})
   watch(%r{^test/test_helper\.rb$})      { 'test' }
   watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/lib/#{m[1]}test_#{m[2]}.rb" }
+  watch(%r{^lib/spn2\.rb})               { 'test/lib/spn2' }
 end

data/README.md

@@ -1,3 +1,4 @@
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop/rubocop)
 [![Gem Version](https://badge.fury.io/rb/spn2.svg)](https://badge.fury.io/rb/spn2)
 
 # Spn2
@@ -21,25 +22,44 @@ For the Spn2 namespace do:
 ```rb
 require 'spn2'
 ```
-
 ### Authentication
 
 The API requires authentication, so you will need an account at [archive.org](https://archive.org). There are two methods of authentication; cookies and API key. Presently only the latter is implemented. API keys may be generated at https://archive.org/account/s3.php. Ensure your access key and secret key are set in environment variables SPN2_ACCESS_KEY and SPN2_SECRET_KEY respectively.
 
+```rb
+> Spn2.access_key
+=> <your access key>
+> Spn2.secret_key
+=> <your secret key>
+```
 ### Save a page
 
-Save a url in the Wayback Machine. This method returns the job_id in a hash.
+Save (capture) a url in the Wayback Machine. This method returns the job_id in a hash.
 ```rb
 > Spn2.save(url: 'example.com') # returns a job_id
 
-=> {job_id: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14'}
+=> {"url"=>"http://example.com","job_id"=>"spn2-9c17e047f58f9220a7008d4f18152fee4d111d14"} # may include a "message" key too
+```
+Various options are available, as detailed in the [specification](https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA/edit) in the section "Capture request". These may be passed like so:
+```rb
+> Spn2.save(url: 'example.com', opts: { capture_all: 1, capture_outlinks: 1 })
+
+=> {"url"=>"http://example.com","job_id"=>"spn2-9c17e047f58f9220a7008d4f18152fee4d111d14"}
+```
+Page save errors will raise an error and look like this:
+```rb
+=> {"status"=>"error", "status_ext"=>"error:too-many-daily-captures", "message"=>"This URL has been already captured 10 times today. 
+    Please try again tomorrow. Please email us at \"info@archive.org\" if you would like to discuss this more."} (Spn2::Spn2ErrorFailedCapture)
 ```
+The key "status_ext" contains an explanatory message - see the API [specification](https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA/edit).
+
+
 
 ### View the status of a job
 
 Use the job_id.
 ```rb
-> Spn2.status(job_id: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14')
+> Spn2.status(job_ids: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14')
 
 => {"counters"=>{"outlinks"=>1, "embeds"=>2}, "job_id"=>"spn2-9c17e047f58f9220a7008d4f18152fee4d111d14",
     "original_url"=>"http://example.com/", "resources"=>["http://example.com/", "http://example.com/favicon.ico"],
@@ -48,13 +68,56 @@ Use the job_id.
 ```
 "status" => "success" is what you are looking for.
 
+Care is advised for domains/urls which are frequently saved into the Wayback Machine as the job_id is merely "spn2-" followed by a hash of the url\*. A status request will show the status of _the most recent capture by anyone_ of the url in question.
+
+\* Usually an sha1 hash of the url in the form http://\<domain\>/\<path\>/ e.g:
+```sh
+$ echo "http://example.com/"|tr -d "\n"|shasum
+9c17e047f58f9220a7008d4f18152fee4d111d14  -
+```
+
+The status of an array of job_id's can be obtained with:
+```rb
+> Spn2.status(job_ids: ['spn2-9c17e047f58f9220a7008d4f18152fee4d111d14', 'spn2-...'])
+
+=> [.. # an array of status hashes
+```
+
+Finally, the status of any outlinks captured by using the save option `capture_outlinks: 1` is available by supplying the parent job_id to:
+```rb
+> Spn2.status(job_ids: 'spn2-cce034d987e1d72d8cbf1770bcf99024fe20dddf', outlinks: true)
+
+=> [.. # an array of outlink job status hashes
+```
+### User status
+
+Information about the user is available via:
+```rb
+> Spn2.user_status
+=> {"daily_captures_limit"=>100000, "available"=>8, "processing"=>0, "daily_captures"=>10}
+```
+
 ### System status
 
 The status of Wayback Machine itself is available.
 ```rb
 > Spn2.system_status
-=> {"status"=>"ok"}
+=> {"status"=>"ok"} # if not "ok" captures may be delayed
+```
+### Error handling
+
+To facilitate graceful error handling, a full list of all error classes is provided by:
+```rb
+> Spn2.error_classes
+=> [Spn2::Spn2Error, Spn2::Spn2ErrorBadAuth,.. ..]
 ```
+## Testing
+
+Just run `bundle exec rake` to run the test suite.
+
+Valid API keys must be held in SPN2_ACCESS_KEY and SPN2_SECRET_KEY for testing. Go to https://archive.org/account/s3.php to set up API keys if you need them. If you have your live keys stored in these env vars just do:
+
+ `export SPN2_ACCESS_KEY=<valid access test key> && export SPN2_SECRET_KEY=<valid secret test key>` immediately before the above command.
 
 ## Development
 

data/lib/curlable.rb

@@ -12,7 +12,7 @@ module Curlable
   end
 
   def post(url:, headers: {}, params: {})
-    Curl::Easy.http_post("#{url}?#{Curl.postalize(params)}", params) do |http|
+    Curl::Easy.http_post("#{url}?#{Curl.postalize(params)}", Curl.postalize(params)) do |http|
       http.follow_location = true
       headers.each { |k, v| http.headers[k] = v }
     end.body_str

data/lib/spn2/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spn2
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/spn2.rb

@@ -1,61 +1,134 @@
 # frozen_string_literal: true
 
+require 'date'
 require 'json'
 require 'nokogiri'
 
 require_relative 'curlable'
+require_relative 'spn2_errors'
 
-# namespace
+# Design decison to not use a class as only 'state' is in 2 env vars
 module Spn2
-  extend Curlable # for .system_status
-  include Curlable
+  extend Curlable
 
-  class Spn2Error < StandardError; end
-
-  FIND_JOB_ID_REGEXP = /(spn2-([a-f]|\d){40})/
+  ESSENTIAL_STATUS_KEYS = %w[job_id resources status].freeze
+  JOB_ID_REGEXP = /^(spn2-([a-f]|\d){40})$/
   WEB_ARCHIVE = 'https://web.archive.org'
 
-  def self.access_key
-    ENV.fetch('SPN2_ACCESS_KEY', nil)
-  end
+  BINARY_OPTS = %w[capture_all capture_outlinks capture_screenshot delay_wb_availabilty force_get skip_first_archive
+                   outlinks_availability email_result].freeze
+  OTHER_OPTS = %w[if_not_archived_within js_behavior_timeout capture_cookie target_username target_password].freeze
 
-  def self.secret_key
-    ENV.fetch('SPN2_SECRET_KEY', nil)
-  end
+  class << self
+    def error_classes
+      Spn2.constants.map { |e| Spn2.const_get(e) }.select { |e| e.is_a?(Class) && e < Exception }
+    end
 
-  def self.system_status
-    JSON.parse get(url: "#{WEB_ARCHIVE}/save/status/system")
-  end
+    def access_key
+      ENV.fetch('SPN2_ACCESS_KEY')
+    end
 
-  def self.save(url:)
-    job_id(json(auth_post(url: "#{WEB_ARCHIVE}/save/#{url}", params: "url=#{url}")))
-  end
+    def secret_key
+      ENV.fetch('SPN2_SECRET_KEY')
+    end
 
-  def self.status(job_id:)
-    json auth_get(url: "#{WEB_ARCHIVE}/save/status/#{job_id}?_t=#{Time.now.to_i}")
-  end
+    def system_status
+      json get(url: "#{WEB_ARCHIVE}/save/status/system") # no auth
+    end
 
-  def self.auth_get(url:)
-    get(url: url, headers: accept_header.merge(auth_header))
-  end
+    def user_status
+      json auth_get(url: "#{WEB_ARCHIVE}/save/status/user?t=#{DateTime.now.strftime('%Q').to_i}")
+    end
 
-  def self.auth_post(url:, params:)
-    post(url: url, headers: accept_header.merge(auth_header), params:)
-  end
+    def save(url:, opts: {})
+      raise Spn2ErrorInvalidOption, "One or more invalid options: #{opts}" unless options_valid?(opts)
 
-  def self.accept_header
-    { 'Accept' => 'application/json' }
-  end
+      json = json(auth_post(url: "#{WEB_ARCHIVE}/save/#{url}", params: { url: url }.merge(opts)))
+      raise Spn2ErrorBadAuth, json.inspect if json['message']&.== BAD_AUTH_MSG
 
-  def self.auth_header
-    { 'Authorization' => "LOW #{Spn2.access_key}:#{Spn2.secret_key}" }
-  end
+      raise Spn2ErrorFailedCapture, json.inspect unless json['job_id']
 
-  def self.job_id(hash)
-    { job_id: hash['job_id'] }
-  end
+      json
+    end
+    alias capture save
+
+    def status(job_ids:, outlinks: false)
+      params = status_params(job_ids: job_ids, outlinks: outlinks)
+      json = json(auth_post(url: "#{WEB_ARCHIVE}/save/status", params: params))
+      return json if json.is_a? Array # must be valid response
+
+      handle_status_errors(job_ids: job_ids, json: json, outlinks: outlinks)
+      json
+    end
+
+    private
+
+    def status_params(job_ids:, outlinks:)
+      return { job_ids: job_ids.join(',') } if job_ids.is_a?(Array)
+      return { job_id_outlinks: job_ids } if outlinks
+
+      { job_id: job_ids } # single job_id
+    end
+
+    def handle_status_errors(job_ids:, json:, outlinks:)
+      raise Spn2ErrorBadAuth, json.inspect if json['message']&.== BAD_AUTH_MSG
+      raise Spn2ErrorNoOutlinks, json.inspect if outlinks
+      raise Spn2ErrorMissingKeys, json.inspect unless (ESSENTIAL_STATUS_KEYS - json.keys).empty?
+      raise Spn2Error, json.inspect if job_ids.is_a?(Array)
+    end
+
+    def auth_get(url:)
+      get(url: url, headers: accept_header.merge(auth_header))
+    end
+
+    def auth_post(url:, params: {})
+      post(url: url, headers: accept_header.merge(auth_header), params: params)
+    end
+
+    def accept_header
+      { Accept: 'application/json' }
+    end
+
+    def auth_header
+      { Authorization: "LOW #{Spn2.access_key}:#{Spn2.secret_key}" }
+    end
+
+    def doc(html_string)
+      Nokogiri::HTML html_string
+    end
+
+    def json(html_string)
+      JSON.parse(doc = doc(html_string))
+    rescue JSON::ParserError # an html response
+      parse_error_code_from_page_title(doc.title) if doc.title
+      parse_error_from_page_body(html_string)
+    end
+
+    def parse_error_code_from_page_title(title_string)
+      raise_code_response_error_if_code_in_string(title_string)
+      raise Spn2ErrorUnknownResponseCode, title_string # code found but doesn't match any known error classes
+    end
+
+    def parse_error_from_page_body(html_string)
+      h1_tag_text = h1_tag_text(html_string)
+      raise_code_response_error_if_code_in_string h1_tag_text
+      raise Spn2ErrorTooManyRequests if h1_tag_text == TOO_MANY_REQUESTS
+
+      raise Spn2ErrorUnknownResponse, html_string # fall through
+    end
+
+    def h1_tag_text(html_string)
+      doc(html_string).xpath('//h1')&.text || ''
+    end
+
+    def raise_code_response_error_if_code_in_string(string)
+      return unless ERROR_CODES.include? code = string.to_i
+
+      raise Spn2.const_get("Spn2Error#{code}")
+    end
 
-  def self.json(json_string)
-    JSON.parse Nokogiri::HTML(json_string)
+    def options_valid?(opts)
+      opts.keys.all? { |k| (BINARY_OPTS + OTHER_OPTS).include? k.to_s }
+    end
   end
 end

data/lib/spn2_errors.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# namespace
+module Spn2
+  BAD_AUTH_MSG = 'You need to be logged in to use Save Page Now.'
+  ERROR_CODES = [400, 502].freeze
+  TOO_MANY_REQUESTS = 'Too Many Requests'
+
+  class Spn2Error < StandardError; end
+  class Spn2ErrorBadAuth < Spn2Error; end
+  class Spn2ErrorBadParams < Spn2Error; end
+  class Spn2ErrorFailedCapture < Spn2Error; end
+  class Spn2ErrorInvalidOption < Spn2Error; end
+  class Spn2ErrorMissingKeys < Spn2Error; end
+  class Spn2ErrorNoOutlinks < Spn2Error; end
+  class Spn2ErrorTooManyRequests < Spn2Error; end
+  class Spn2ErrorUnknownResponse < Spn2Error; end
+  class Spn2ErrorUnknownResponseCode < Spn2Error; end
+  ERROR_CODES.each { |i| const_set("Spn2Error#{i}", Class.new(Spn2Error)) }
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spn2
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - MatzFan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-29 00:00:00.000000000 Z
+date: 2022-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curb
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.16'
+- !ruby/object:Gem::Dependency
+  name: minitest-parallel_fork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +150,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-description: Atomate the process of saving web pages to archive.org
+description: Automate the process of saving web pages to archive.org
 email: 
 executables: []
 extensions: []
@@ -152,8 +166,9 @@ files:
 - lib/curlable.rb
 - lib/spn2.rb
 - lib/spn2/version.rb
+- lib/spn2_errors.rb
 - sig/spn2.rbs
-homepage: https://gitlab.com/matxfan/spn2
+homepage: https://gitlab.com/matzfan/spn2
 licenses:
 - MIT
 metadata:
@@ -177,8 +192,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.16
+rubygems_version: 3.3.17
 signing_key: 
 specification_version: 4
-summary: Gem for the Save Page Now API of the Wayback Machine
+summary: Gem for the Save Page Now 2 API of the Wayback Machine
 test_files: []