archaeo 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24c6b37575e8f673a8e6acb7aba38264ac811236cb91e905663914d08a283289
-  data.tar.gz: 632f36d31ee83b23f727dd8eb8217929d3dad26c94c6948fb0621aff2b937701
+  metadata.gz: 8b82c7c8f2e568697056587556ac01bee92f51376af390aa26ed9c76221200d8
+  data.tar.gz: 411cf04b07219a02e5b049120bb74528dc17cf5fcb8424acd3c7c5ee69659bea
 SHA512:
-  metadata.gz: 33cf0ea6c5317be5aafba988e8652555a8e7a77620e93571a51c1537ae4cdd455e76d170fe78549a5e3068ec82db12138159cb87a31c1aafda5c036a6ca2511e
-  data.tar.gz: 2c0ae7a6a461913ed475dd3084e495f057db995ebf9ea8a1e6c9595aae74cd6bc65a0cdc6f2b2a3a831d9cc0e45e92e06a13a4f7653e89c0a8bbcd7226207677
+  metadata.gz: ec87c66817c1ce3f318e45d5b26221ca44c1958dfa9009fbe1541a1b3bbdb8e4585f6baf66e73c6a839a10d938e189b61dbea43afccf74a6f55476ad8719e18e
+  data.tar.gz: 0c33454fa368f009a7383ebc8a6a79f3638615740bd312eaab568c21d90cb20a0c4c6615324b20634f00614021016cf6d464910679fcf61fdf8d663097935cd7

data/lib/archaeo/availability_api.rb

@@ -16,6 +16,7 @@ module Archaeo
     end
 
     def near(url, timestamp: nil)
+      url = UrlNormalizer.normalize(url)
       params = { "url" => url }
       params["timestamp"] = timestamp.to_s if timestamp
 

data/lib/archaeo/bulk_downloader.rb

@@ -15,6 +15,7 @@ module Archaeo
     end
 
     def download(url, from: nil, to: nil, resume: false)
+      url = UrlNormalizer.normalize(url)
       FileUtils.mkdir_p(@output_dir)
       state = DownloadState.new(@output_dir)
 

data/lib/archaeo/cdx_api.rb

@@ -47,6 +47,7 @@ module Archaeo
     # Returns an Enumerator of Snapshot objects, auto-paginating
     # via resume key unless an explicit page is requested.
     def snapshots(url, **options)
+      url = UrlNormalizer.normalize(url)
       validate_options!(options)
 
       Enumerator.new do |yielder|
@@ -59,9 +60,15 @@ module Archaeo
     end
 
     def near(url, timestamp:)
+      url = UrlNormalizer.normalize(url)
       ts = Timestamp.coerce(timestamp)
       result = snapshots(url, sort: "closest",
                               closest: ts.to_s, limit: 1).first
+      if result&.blocked?
+        raise BlockedSiteError,
+              "Site is blocked: #{url}"
+      end
+
       result || raise(NoSnapshotFound,
                       "No snapshot found near #{ts} for #{url}")
     end
@@ -94,6 +101,7 @@ module Archaeo
 
     # Returns the number of pages for a paginated query.
     def num_pages(url, **options)
+      url = UrlNormalizer.normalize(url)
       params = { "url" => url, "showNumPages" => "true" }
       merge_scalar_params!(params, options)
       response = @client.get(
@@ -109,6 +117,7 @@ module Archaeo
 
     # Returns all unique original URLs under a domain.
     def known_urls(domain, match_type: "domain")
+      domain = UrlNormalizer.normalize(domain)
       snapshots(domain, match_type: match_type,
                         collapse: ["urlkey"]).map(&:original_url).uniq
     end

data/lib/archaeo/fetcher.rb

@@ -14,6 +14,7 @@ module Archaeo
     end
 
     def fetch(url, timestamp:, identity: false)
+      url = UrlNormalizer.normalize(url)
       ts = Timestamp.coerce(timestamp)
       archive_url = ArchiveUrl.new(url, timestamp: ts,
                                         identity: identity)
@@ -21,6 +22,13 @@ module Archaeo
       build_page(response, archive_url.to_s, url, ts)
     end
 
+    def fetch_page_with_assets(url, timestamp:)
+      page = fetch(url, timestamp: timestamp)
+      assets = AssetExtractor.new(page.content,
+                                  base_url: page.archive_url).extract
+      PageBundle.new(page: page, assets: assets)
+    end
+
     private
 
     def build_page(response, archive_url, url, timestamp)

data/lib/archaeo/page_bundle.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Archaeo
+  # A fetched page together with all its extracted asset URLs.
+  #
+  # Bundles a Page with the AssetList discovered from its HTML,
+  # providing a single object for complete page archival.
+  class PageBundle
+    attr_reader :page, :assets
+
+    def initialize(page:, assets:)
+      @page = page
+      @assets = assets
+    end
+  end
+end

data/lib/archaeo/save_api.rb

@@ -17,6 +17,7 @@ module Archaeo
     end
 
     def save(url)
+      url = UrlNormalizer.normalize(url)
       save_url = "#{ENDPOINT}/#{url}"
       start_time = Time.now.utc
       attempt_save(save_url, start_time, url)

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/archaeo.rb

@@ -19,6 +19,7 @@ module Archaeo
   autoload :ArchiveUrl, "archaeo/archive_url"
   autoload :Snapshot, "archaeo/snapshot"
   autoload :Page, "archaeo/page"
+  autoload :PageBundle, "archaeo/page_bundle"
   autoload :SaveResult, "archaeo/save_result"
   autoload :AvailabilityResult, "archaeo/availability_result"
   autoload :UrlNormalizer, "archaeo/url_normalizer"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Ribose Inc.
@@ -84,6 +84,7 @@ files:
 - lib/archaeo/fetcher.rb
 - lib/archaeo/http_client.rb
 - lib/archaeo/page.rb
+- lib/archaeo/page_bundle.rb
 - lib/archaeo/save_api.rb
 - lib/archaeo/save_result.rb
 - lib/archaeo/snapshot.rb