spn2 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 744fc63b21023c96ff40a72561d80f8e277be132b369c10b7196f273178177d8
-  data.tar.gz: 26c923d7ac606c16c0f56e98cff3497878771899a7fc23947c57d864316de4a7
+  metadata.gz: e968c75da93882e48ac210e17bb599497971ee1f065035758279529b23c53f1b
+  data.tar.gz: 9fa1b6f6125d9347d2418254b8e8838572fd8a4efcf0c6bbabc258f3784c097b
 SHA512:
-  metadata.gz: ca70afc978bd3766ebdc4234575ea1ca5e19202757c353bd917b6b822293764fbde204ad3e6753188eaab0c2123abf2b71ae8fee6c827df41fb836662f90f788
-  data.tar.gz: 23a245ea4c896b034da511fa7dab95d81fda3cd8cec778c1e0be27eddf4f49e46452efb7e10feeb59af0b7ad02ee95fc352f2cbc34c8e79f245c5a8efd283b52
+  metadata.gz: bacfeda95f8a40e132496cb69078e8229767de20c99615db7da4bb1d5f35a9a4c7e3e688a1fe3d31d6dd9e5f026d0d6fc8e328aae90915de55e6810db675732f
+  data.tar.gz: e62fe104f074cb9ab5e4a85b89050e8dc0f198613dd9ff62fd56d3c645428e080699e14d57b8b6a655b68ed3be00704ff67ba5e9a58508083fe0f961cdc1ac55

data/.rubocop.yml

@@ -8,6 +8,3 @@ AllCops:
 
 Style/HashSyntax:
   Enabled: false  # yuk Ruby 3.1
-
-Style/NumericLiterals:
-  Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,3 @@
-## [Unreleased]
-
 ## [0.1.0] - 2022-06-29
 
 - Initial release
@@ -8,3 +6,8 @@
 
 - Add error handling
 - Add ability to add opts to Spn2.save
+
+## [0.1.2] - 2022-07-02
+
+- Add user_status
+- Add status calls for multiple job_ids and outlinks

data/Guardfile

@@ -4,4 +4,5 @@ guard :minitest do
   watch(%r{^test/(.*)/?test_(.*)\.rb$})
   watch(%r{^test/test_helper\.rb$})      { 'test' }
   watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/lib/#{m[1]}test_#{m[2]}.rb" }
+  watch(%r{^lib/spn2\.rb})               { 'test/lib/spn2' }
 end

data/README.md

@@ -1,3 +1,4 @@
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop/rubocop)
 [![Gem Version](https://badge.fury.io/rb/spn2.svg)](https://badge.fury.io/rb/spn2)
 
 # Spn2
@@ -37,19 +38,28 @@ Save (capture) a url in the Wayback Machine. This method returns the job_id in a
 ```rb
 > Spn2.save(url: 'example.com') # returns a job_id
 
-=> {job_id: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14'} # json may include "url" and "message" keys too
+=> {"url"=>"http://example.com","job_id"=>"spn2-9c17e047f58f9220a7008d4f18152fee4d111d14"} # may include a "message" key too
 ```
 Various options are available, as detailed in the [specification](https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA/edit) in the section "Capture request". These may be passed like so:
 ```rb
 > Spn2.save(url: 'example.com', opts: { capture_all: 1, capture_outlinks: 1 })
 
-=> {url: 'example.com', job_id: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14'}
+=> {"url"=>"http://example.com","job_id"=>"spn2-9c17e047f58f9220a7008d4f18152fee4d111d14"}
 ```
+Page save errors will raise an error and look like this:
+```rb
+=> {"status"=>"error", "status_ext"=>"error:too-many-daily-captures", "message"=>"This URL has been already captured 10 times today. 
+    Please try again tomorrow. Please email us at \"info@archive.org\" if you would like to discuss this more."} (Spn2::Spn2ErrorFailedCapture)
+```
+The key "status_ext" contains an explanatory message - see the API [specification](https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA/edit).
+
+
+
 ### View the status of a job
 
 Use the job_id.
 ```rb
-> Spn2.status(job_id: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14')
+> Spn2.status_job_id(job_id: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14')
 
 => {"counters"=>{"outlinks"=>1, "embeds"=>2}, "job_id"=>"spn2-9c17e047f58f9220a7008d4f18152fee4d111d14",
     "original_url"=>"http://example.com/", "resources"=>["http://example.com/", "http://example.com/favicon.ico"],
@@ -58,6 +68,35 @@ Use the job_id.
 ```
 "status" => "success" is what you are looking for.
 
+Care is advised for domains/urls which are frequently saved into the Wayback Machine as the job_id is merely "spn2-" followed by a hash of the url\*. A status request will show the status of _the most recent capture by anyone_ of the url in question.
+
+\* Usually an sha1 hash of the url in the form http://\<domain\>/\<path\>/ e.g:
+```sh
+$ echo "http://example.com/"|tr -d "\n"|shasum
+9c17e047f58f9220a7008d4f18152fee4d111d14  -
+```
+
+The status of a comma-separated list of job_id's can be obtained with:
+```rb
+> Spn2.status_job_ids(job_ids: 'spn2-9c17e047f58f9220a7008d4f18152fee4d111d14,spn2-...')
+
+=> [.. # an array of status hashes
+```
+
+Finally, the status of any outlinks captured by using the save option `capture_outlinks: 1` is available by supplying the parent job_id to:
+```rb
+> Spn2.status_job_id_outlinks(job_id: 'spn2-cce034d987e1d72d8cbf1770bcf99024fe20dddf')
+
+=> [.. # an array of outlink job status hashes
+```
+### User status
+
+Information about the user is available via:
+```rb
+> Spn2.user_status
+=> {"daily_captures_limit"=>100000, "available"=>8, "processing"=>0, "daily_captures"=>10}
+```
+
 ### System status
 
 The status of Wayback Machine itself is available.
@@ -67,11 +106,19 @@ The status of Wayback Machine itself is available.
 ```
 ### Error handling
 
-To fascilitate graceful error handling, a full list of all error classes is provided by:
+To facilitate graceful error handling, a full list of all error classes is provided by:
 ```rb
 > Spn2.error_classes
 => [Spn2::Spn2Error, Spn2::Spn2ErrorBadAuth,.. ..]
 ```
+## Testing
+
+Just run `bundle exec rake` to run the test suite.
+
+Valid API keys must be held in SPN2_ACCESS_KEY and SPN2_SECRET_KEY for testing. Go to https://archive.org/account/s3.php to set up API keys if you need them. If you have your live keys stored in these env vars just do:
+
+ `export SPN2_ACCESS_KEY=<valid access test key> && export SPN2_SECRET_KEY=<valid secret test key>` immediately before the above command.
+
 ## Development
 
 ~~After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.~~

data/lib/spn2/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spn2
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/lib/spn2.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'date'
 require 'json'
 require 'nokogiri'
 
@@ -7,20 +8,22 @@ require_relative 'curlable'
 
 # Design decison to not use a class as only 'state' is in 2 env vars
 module Spn2
-  extend Curlable # for system_status
-  include Curlable
+  extend Curlable
 
+  BAD_AUTH_MSG = 'You need to be logged in to use Save Page Now.'
   ERROR_CODES = [502].freeze
 
   class Spn2Error < StandardError; end
   class Spn2ErrorBadAuth < Spn2Error; end
-  class Spn2ErrorBadAuth < Spn2Error; end
-  class Spn2ErrorBadResponse < Spn2Error; end
+  class Spn2ErrorFailedCapture < Spn2Error; end
   class Spn2ErrorInvalidOption < Spn2Error; end
+  class Spn2ErrorMissingKeys < Spn2Error; end
+  class Spn2ErrorNoOutlinks < Spn2Error; end
+  class Spn2ErrorTooManyRequests < Spn2Error; end
+  class Spn2ErrorUnknownResponse < Spn2Error; end
   class Spn2ErrorUnknownResponseCode < Spn2Error; end
   ERROR_CODES.each { |i| Spn2.const_set("Spn2Error#{i}", Class.new(Spn2Error)) }
 
-  BAD_AUTH_MSG = 'You need to be logged in to use Save Page Now.'
   ESSENTIAL_STATUS_KEYS = %w[job_id resources status].freeze
   JOB_ID_REGEXP = /^(spn2-([a-f]|\d){40})$/
   WEB_ARCHIVE = 'https://web.archive.org'
@@ -35,37 +38,58 @@ module Spn2
     end
 
     def access_key
-      ENV.fetch('SPN2_ACCESS_KEY', nil)
+      ENV.fetch('SPN2_ACCESS_KEY')
     end
 
     def secret_key
-      ENV.fetch('SPN2_SECRET_KEY', nil)
+      ENV.fetch('SPN2_SECRET_KEY')
     end
 
     def system_status
       json get(url: "#{WEB_ARCHIVE}/save/status/system") # no auth
     end
 
+    def user_status
+      json auth_get(url: "#{WEB_ARCHIVE}/save/status/user?t=#{DateTime.now.strftime('%Q').to_i}")
+    end
+
     def save(url:, opts: {})
       raise Spn2ErrorInvalidOption, "One or more invalid options: #{opts}" unless options_valid?(opts)
 
-      hash = json(auth_post(url: "#{WEB_ARCHIVE}/save/#{url}", params: { url: url }.merge(opts)))
-      raise Spn2ErrorBadAuth, hash.inspect if hash['message']&.== BAD_AUTH_MSG
+      json = json(auth_post(url: "#{WEB_ARCHIVE}/save/#{url}", params: { url: url }.merge(opts)))
+      raise Spn2ErrorBadAuth, json.inspect if json['message']&.== BAD_AUTH_MSG
 
-      raise Spn2ErrorBadResponse, "Bad response: #{hash.inspect}" unless hash['job_id']
+      raise Spn2ErrorFailedCapture, json.inspect unless json['job_id']
 
-      hash
+      json
     end
     alias capture save
 
-    def status(job_id:)
-      hash = json(auth_get(url: "#{WEB_ARCHIVE}/save/status/#{job_id}"))
-      raise Spn2ErrorBadAuth, hash.inspect if hash['message']&.== BAD_AUTH_MSG
+    def status_job_id(job_id:)
+      json = json(auth_post(url: "#{WEB_ARCHIVE}/save/status", params: { job_id: job_id }))
+      raise Spn2ErrorBadAuth, json.inspect if json['message']&.== BAD_AUTH_MSG
 
-      raise Spn2ErrorBadResponse, "Bad response: #{hash.inspect}" unless (ESSENTIAL_STATUS_KEYS - hash.keys).empty?
+      raise Spn2ErrorMissingKeys, json.inspect unless (ESSENTIAL_STATUS_KEYS - json.keys).empty?
 
-      hash
+      json
     end
+    alias status status_job_id
+
+    def status_job_ids(job_ids:)
+      json = json(auth_post(url: "#{WEB_ARCHIVE}/save/status", params: { job_ids: job_ids }))
+      raise Spn2Error, json.inspect unless json.is_a? Array
+
+      json
+    end
+    alias statuses status_job_ids
+
+    def status_job_id_outlinks(job_id:)
+      json = json(auth_post(url: "#{WEB_ARCHIVE}/save/status", params: { job_id_outlinks: job_id }))
+      raise Spn2ErrorNoOutlinks, json.inspect unless json.is_a? Array
+
+      json
+    end
+    alias status_outlinks status_job_id_outlinks
 
     private
 
@@ -85,19 +109,29 @@ module Spn2
       { Authorization: "LOW #{Spn2.access_key}:#{Spn2.secret_key}" }
     end
 
-    def json(html_string)
-      JSON.parse(doc = Nokogiri::HTML(html_string))
-    rescue JSON::ParserError # an html response
-      raise Spn2ErrorBadResponse, "No title in: #{html_string}" unless (title = doc.title)
+    def doc(html_string)
+      Nokogiri::HTML html_string
+    end
 
-      parse_error_code_from_page_title(title)
+    def json(html_string)
+      JSON.parse(doc = doc(html_string))
+    rescue JSON::ParserError # an html response & therefore an error
+      parse_error_code_from_page_title(doc.title) if doc.title
+      parse_error_from_page_body(html_string) # if no title parse body
     end
 
-    def parse_error_code_from_page_title(string)
-      code = string.to_i
+    def parse_error_code_from_page_title(title_string)
+      code = title_string.to_i
       raise Spn2.const_get("Spn2Error#{code}") if ERROR_CODES.include? code
 
-      raise Spn2ErrorUnknownResponseCode, string
+      raise Spn2ErrorUnknownResponseCode
+    end
+
+    def parse_error_from_page_body(html_string)
+      h1 = doc(html_string).xpath('//h1')
+      raise Spn2ErrorTooManyRequests if !h1.empty? && h1.text == 'Too Many Requests'
+
+      raise Spn2ErrorUnknownResponse, html_string # fall through
     end
 
     def options_valid?(opts)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spn2
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - MatzFan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-30 00:00:00.000000000 Z
+date: 2022-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curb
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.16'
+- !ruby/object:Gem::Dependency
+  name: minitest-parallel_fork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +150,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-description: Atomate the process of saving web pages to archive.org
+description: Automate the process of saving web pages to archive.org
 email: 
 executables: []
 extensions: []
@@ -153,7 +167,7 @@ files:
 - lib/spn2.rb
 - lib/spn2/version.rb
 - sig/spn2.rbs
-homepage: https://gitlab.com/matxfan/spn2
+homepage: https://gitlab.com/matzfan/spn2
 licenses:
 - MIT
 metadata:
@@ -180,5 +194,5 @@ requirements: []
 rubygems_version: 3.3.17
 signing_key: 
 specification_version: 4
-summary: Gem for the Save Page Now API of the Wayback Machine
+summary: Gem for the Save Page Now 2 API of the Wayback Machine
 test_files: []