archive_today 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad131a62edbb1b182f34c8beccaff07cb2e6f73414cd7391c8c8487bfe5ba3b4
-  data.tar.gz: a30538c8a90aeec34e003c51de6efcefc4f2b8985a3c66c8bb47355ce2fcbd28
+  metadata.gz: d37b8ae11abd0b7a09160e3dc7fa47b7e395a9e9b0ee56c6a0c0f6d686e06dd4
+  data.tar.gz: 88d084d91aeaefb0033e105ee23348f55335acbf92dc8bf4de07f9efcb894a1d
 SHA512:
-  metadata.gz: 92b10f61076a7b1e2c44b93bdad340ed71a5be366b3766f00743e33eec0ca4b3515758fda76a3ee6c140006528cbfd9dc97837bb06a8f466d72ac9e671d82a38
-  data.tar.gz: fb4d2613c9417dcf16cfce535596574f367a8da48ad25dda967667d918b5ad32c49585de8b32eb1072988de8693e321d85995366760b6215724e1448f5ba7a7b
+  metadata.gz: 16a9e59b313e65a34d06657ee3a450ce8fe6e03c8f1f1ceb436db7ebce3fe5fd85b6a30f47818b49c2872bbe6cf641456d14d5d6e6e5323e1d908055abde6ca3
+  data.tar.gz: a484ae8056b1e643b0e2852ac25d79ad8e97597b8920cf9fb43947ddf9981fb94fbc4e773e7370c26d84bbf7543ffc31e92841e9aaf08adf5e5f5d7d75230a0e

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 0.2.0
+
+Also retrieve screenshot URL if available
+
+## Breaking Changes
+
+- Method renamed from `#submit` to `#capture`
+- Return type is now a hash with keys `:url` and `:screenshot_url`
+
 # 0.1.0
 
 Initial commit

data/Gemfile

@@ -1,9 +1,12 @@
 source 'https://rubygems.org'
 
+group :development, :test do
+  gem 'amazing_print'
+  gem 'httplog'
+  gem 'irbtools'
+  gem 'pry'
+  gem 'rake'
+end
+
 # Specify your gem's dependencies in archive_today.gemspec
 gemspec
-
-gem 'faraday', '~> 1.0'
-gem 'faraday_middleware', '~> 1.0'
-gem 'nokogiri', '~> 1.10'
-gem 'rake', '~> 12.0'

data/Gemfile.lock

@@ -1,30 +1,102 @@
 PATH
   remote: .
   specs:
-    archive_today (0.1.0)
+    archive_today (0.2.0)
+      faraday (~> 1.0)
+      faraday_middleware (~> 1.0)
+      nokogiri (~> 1.10)
+      rake (~> 12.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    alias (0.2.3)
+    amazing_print (1.2.1)
+    binding.repl (3.0.0)
+    boson (1.3.0)
+    boson-more (0.3.1)
+      boson (>= 1.3.0)
+    cd (1.0.1)
+    clipboard (1.0.6)
+    coderay (1.1.2)
+    debugging (1.1.1)
+      binding.repl (~> 3.0)
+      paint (>= 0.9, < 3.0)
+    every_day_irb (2.1.0)
+      cd (~> 1.0)
+    fancy_irb (1.2.1)
+      paint (>= 0.9, < 3.0)
+      unicode-display_width (~> 1.1)
     faraday (1.0.1)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (1.0.0)
       faraday (~> 1.0)
+    ffi (1.13.1)
+    g (1.7.2)
+    hirb (0.7.3)
+    httplog (1.4.2)
+      rack (>= 1.0)
+      rainbow (>= 2.0.0)
+    interactive_editor (0.0.11)
+      spoon (>= 0.0.1)
+    irbtools (1.7.1)
+      alias (~> 0.2.3)
+      binding.repl (~> 3.0)
+      boson (~> 1.3.0)
+      boson-more (~> 0.3.0)
+      clipboard (~> 1.0.5)
+      coderay (~> 1.1.0)
+      debugging (~> 1.0)
+      every_day_irb (>= 1.7.1)
+      fancy_irb (>= 0.7.3)
+      g (>= 1.7.2)
+      hirb (~> 0.7, >= 0.7.3)
+      interactive_editor (>= 0.0.10)
+      method_locator (>= 0.0.4)
+      method_source (>= 0.8.2)
+      methodfinder (~> 2.0)
+      ori (~> 0.1.0)
+      os (~> 0.9)
+      paint (>= 0.8.7)
+      ruby_engine (~> 1.0)
+      ruby_info (~> 1.0)
+      ruby_version (~> 1.0)
+      wirb (>= 1.0.3)
+    method_locator (0.0.4)
+    method_source (1.0.0)
+    methodfinder (2.2.1)
     mini_portile2 (2.4.0)
     multipart-post (2.1.1)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
+    ori (0.1.0)
+    os (0.9.6)
+    paint (2.2.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.2)
+    rainbow (3.0.0)
     rake (12.3.2)
+    ruby_engine (1.0.1)
+    ruby_info (1.0.1)
+    ruby_version (1.0.2)
+    spoon (0.0.6)
+      ffi
+    unicode-display_width (1.7.0)
+    wirb (2.2.1)
+      paint (>= 0.9, < 3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  amazing_print
   archive_today!
-  faraday (~> 1.0)
-  faraday_middleware (~> 1.0)
-  nokogiri (~> 1.10)
-  rake (~> 12.0)
+  httplog
+  irbtools
+  pry
+  rake
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -20,18 +20,51 @@ Or install it yourself as:
 
 ## Usage
 
+### Class Method
+
+`ArchiveToday#capture`
+
+Returns a Hash with keys `:url` and `:screenshot_url`. Note that if the page is in the process of being archived, the screenshot is not generated yet, so `nil` is returned for the screenshot URL.
+
+#### Args
+
+- `url` (required) - the target URL for archival
+- `debug` (optional) - when set to true, this will log HTTP requests and responses
+
+```ruby
+require 'archive_today'
+
+ArchiveToday.capture(url: 'https://example.com')
+
+# => { url: 'https://archive.is/a1b2c3, screenshot_url: 'https://archive.is/[...].jpg' }
+```
+
+### Instance Method
+
+`ArchiveToday::Archiver`
+
+This class exposes the same `#capture` method, but you can also query the instance for the cached URLs once the capture response is received.
+
+#### Args
+
+- `url` (required) - the target URL for archival
+- `debug` (optional) - when set to true, this will log HTTP requests and responses
+
+
 ```ruby
 require 'archive_today'
 
-ArchiveToday.submit(url: 'https://example.com')
+a = ArchiveToday::Archiver.new(url: 'https://example.com')
+a.capture
+
+puts a.screenshot_url
 
-# => 'https://archive.is/a1b2c3
+# => 'https://archive.is/[...].jpg
 ```
 
 ## Roadmap
 
 - proxies
-- optionally return URL of captured screenshot
 
 ## Development
 

data/archive_today.gemspec

@@ -12,12 +12,15 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
+  spec.add_dependency 'faraday', '~> 1.0'
+  spec.add_dependency 'faraday_middleware', '~> 1.0'
+  spec.add_dependency 'nokogiri', '~> 1.10'
+  spec.add_dependency 'rake', '~> 12.0'
+
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "#{spec.homepage}/CHANGELOG.md"
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md"
 
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end

data/lib/archive_today.rb

@@ -1,3 +1,6 @@
+require 'faraday'
+require 'faraday_middleware'
+require 'nokogiri'
 require 'archive_today/version'
 require 'archive_today/archiver'
 
@@ -5,8 +8,8 @@ module ArchiveToday
   class Error < StandardError; end
 
   class << self
-    def submit(url:, debug: false)
-      Archiver.new(url: url, debug: debug).submit
+    def capture(url:, debug: false)
+      Archiver.new(url: url, debug: debug).capture
     end
   end
 end

data/lib/archive_today/archiver.rb

@@ -1,6 +1,3 @@
-require 'faraday'
-require 'faraday_middleware'
-require 'nokogiri'
 require_relative 'version'
 
 module ArchiveToday
@@ -8,41 +5,69 @@ module ArchiveToday
     BASE_URL = 'https://archive.today/'.freeze
     DEFAULT_USER_AGENT = "archive_today #{ArchiveToday::VERSION}".freeze
 
-    attr_reader :debug, :url, :user_agent
+    attr_accessor :response
+    attr_reader :debug, :target_url, :user_agent
 
     def initialize(url:, user_agent: DEFAULT_USER_AGENT, debug: false)
       @debug = debug
-      @url = url
+      @target_url = url
       @user_agent = user_agent
     end
 
-    def submit
-      puts 'Submitting URL ...'
+    def capture
+      puts 'Submitting URL ...' if debug
       response = connection.post('/submit/') do |req|
         req.body = submission_body
       end
       raise unless response.success?
 
-      handle_response(response)
+      self.response = response
+
+      {
+        url: finalized_url,
+        screenshot_url: screenshot_url
+      }
     end
 
     private
 
-    def handle_response(response)
-      headers = response.headers
+    def finalized_url
+      archived_url.gsub('/wip', '')
+    end
+
+    def archived_url
+      @archived_url ||= begin
+        headers = response.headers
 
-      return headers[:location] if headers.has_key?('location')
-      return headers[:refresh].split(';url=').last if headers.has_key?('refresh')
+        return headers[:location] if headers.has_key?('location')
+        return headers[:refresh].split(';url=').last if headers.has_key?('refresh')
+
+        # TODO: handle the history case mentioned here?
+        # https://github.com/pastpages/archiveis/blob/master/archiveis/api.py#L81
+        response.env.url
+      end
+    end
 
-      # TODO: handle the history case mentioned here?
-      # https://github.com/pastpages/archiveis/blob/master/archiveis/api.py#L81
-      response.env.url
+    def screenshot_url
+      return nil unless archived_url
+      return nil if archived_url.include? '/wip/'
+
+      response = connection.get do |req|
+        req.url "#{archived_url}/image"
+      end
+      html = Nokogiri::HTML(response.body)
+      node = html.at_css('img[itemprop="contentUrl"]')
+      url = node.attr('src')
+      puts "Got screenshot URL: #{url}" if debug && url
+      return url if url
+
+      nil
     end
 
     def submission_body
       URI.encode_www_form(
         {
-          url: url,
+          url: target_url,
           anyway: 1,
           submitid: unique_submission_id
         }
@@ -50,14 +75,15 @@ module ArchiveToday
     end
 
     def unique_submission_id
-      puts 'Getting unique submission ID ...'
+      puts 'Getting unique submission ID ...' if debug
       response = connection.get('/')
       raise unless response.success?
 
       html = Nokogiri::HTML(response.body)
       node = html.at_css('input[name="submitid"]')
       id = node.attr('value')
-      puts "Got ID: #{id}" and return if id
+      puts "Got ID: #{id}" if debug && id
+      return id if id
 
       nil
     end

data/lib/archive_today/version.rb

@@ -1,3 +1,3 @@
 module ArchiveToday
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archive_today
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - tomholford
@@ -9,7 +9,63 @@ autorequire:
 bindir: exe
 cert_chain: []
 date: 2020-07-13 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
 description: Submit a URL to the Archive.today service to preserve it's contents in
   a Memento-compatible format
 email:
@@ -37,7 +93,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/tomholford/archive-today
   source_code_uri: https://github.com/tomholford/archive-today
-  changelog_uri: https://github.com/tomholford/archive-today/CHANGELOG.md
+  changelog_uri: https://github.com/tomholford/archive-today/blob/master/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths: