resync-client 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 339acc57f21bd27647e8f9484157b6b4d6f34007
+  data.tar.gz: 293b87bfd6ec673c9e01701d1976626aef06994b
+SHA512:
+  metadata.gz: 5b30f4db7f6ff8c3a7c8652edc4458dc991593f6e1d14ccf49e7e7d28d16f03fdb94853a24d70ff57d4e32fb927ef447e0fd78145a442b41533e9b11668188d4
+  data.tar.gz: f37a86a97b4b3018d79b6a64031f3a0e36f6d5a165a19ff8a88284c490061d0876cfea283989321c3f4b7837e2203d262dff22d57796c793c981791bc5368cc0

data/.gitignore

@@ -0,0 +1,42 @@
+# Ruby defaults
+
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+# Rails engine
+
+.bundle/
+log/*.log
+spec/dummy/db/*.sqlite3
+spec/dummy/db/*.sqlite3-journal
+spec/dummy/log/*.log
+spec/dummy/tmp/
+spec/dummy/.sass-cache
+
+# IntellJ
+
+*.iml
+*.ipr
+*.iws
+.rakeTasks
+.idea
+
+# Emacs
+
+*~
+
+# Mac OS
+
+.DS_Store

data/.rubocop.yml

@@ -0,0 +1,23 @@
+# Disable line-length check; it's too easy for the cure to be worse than the disease
+Metrics/LineLength:
+  Enabled: False
+
+# Disable problematic module documentation check (see https://github.com/bbatsov/rubocop/issues/947)
+Style/Documentation:
+  Enabled: false
+
+# Allow one line around class body (Style/EmptyLines will still disallow two or more)
+Style/EmptyLinesAroundClassBody:
+  Enabled: false
+
+# Allow one line around module body (Style/EmptyLines will still disallow two or more)
+Style/EmptyLinesAroundModuleBody:
+  Enabled: false
+
+# Allow one line around block body (Style/EmptyLines will still disallow two or more)
+Style/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+# Allow %r notation for regexes with a single / character
+Style/RegexpLiteral:
+  MaxSlashes: 0

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/.travis.yml

@@ -0,0 +1,2 @@
+language: ruby
+

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.md

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 The Regents of the University of California
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# resync-client
+
+A gem providing a Ruby client for the [ResourceSync](http://www.openarchives.org/rs/1.0/resourcesync) web synchronization framework, based on the [resync](https://github.com/dmolesUC3/resync) gem and [Net::HTTP](http://ruby-doc.org/stdlib-2.2.2/libdoc/net/http/rdoc/Net/HTTP.html).
+
+## Status
+
+This is a work in progress. We welcome bug reports and feature requests.
+
+## Usage
+
+Retrieving the [Source Description](http://www.openarchives.org/rs/1.0/resourcesync#wellknown) for a site:
+
+```ruby
+    client = Resync::Client.new
+
+    source_desc_uri = 'http://example.org/.well-known/resourcesync'
+    source_desc = client.get_and_parse(source_desc_uri) # => Resync::SourceDescription
+```
+
+Retrieving a [Capability List](http://www.openarchives.org/rs/1.0/resourcesync#CapabilityList) from the source description:
+
+```ruby
+    cap_list_resource = source_desc.resource_for(capability: 'capabilitylist')
+    cap_list = cap_list_resource.get_and_parse # => Resync::CapabilityList
+```
+
+Retrieving a [Change List](http://www.openarchives.org/rs/1.0/resourcesync#ChangeList) and downloading the latest revision of a known resource to a file
+
+```ruby
+    change_list_resource = cap_list.resource_for(capability: 'changelist')
+    change_list = change_list_resource.get_and_parse # => Resync::ChangeList
+    latest_rev_resource = change_list.latest_for(uri: URI('http://example.com/my-resource'))
+    latest_rev_resource.download_to_file('/tmp/my-resource.txt')
+```
+
+Retrieving a [Change Dump](http://www.openarchives.org/rs/1.0/resourcesync#ChangeDump), searching through its manifests for changes to a specified URL, downloading the ZIP package containing that resource, and extracting it from the ZIP package:
+
+```ruby
+    change_dump_resource = cap_list.resource_for(capability: 'changedump')
+    change_dump = change_dump_resource.get_and_parse # => Resync::ChangeDump
+    change_dump.resources.each do |package|
+      manifest_link = package.link_for(rel: 'contents')
+      if manifest_link
+        manifest = manifest_link.get_and_parse # => Resync::ChangeDumpManifest
+        latest_resource = manifest.latest_for(uri: URI('http://example.com/my-resource'))
+        if latest_resource
+          timestamp = latest_resource.modified_time.strftime('%s%3N')
+          zip_package = package.zip_package # => Resync::ZipPackage (downloaded to temp file)
+          bitstream = zip_package.bitstream_for(latest_resource) # => Resync::Bitstream
+          content = bitstream.content # => String (extracted from ZIP file)
+          File.open("/tmp/my-resource-#{timestamp}.txt") { |f| f.write(content) }
+        end
+      end
+    end
+```
+
+## Limitations
+
+`resync-client` hasn't really been tested except with [resync-simulator](https://github.com/resync/resync-simulator), and that not much beyond what you'll find in [example.rb](example.rb), so expect trouble.

data/Rakefile

@@ -0,0 +1,56 @@
+# ------------------------------------------------------------
+# RSpec
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+namespace :spec do
+
+  desc 'Run all unit tests'
+  RSpec::Core::RakeTask.new(:unit) do |task|
+    task.rspec_opts = %w(--color --format documentation --order default)
+    task.pattern = 'unit/**/*_spec.rb'
+  end
+
+  desc 'Run all acceptance tests'
+  RSpec::Core::RakeTask.new(:acceptance) do |task|
+    ENV['COVERAGE'] = nil
+    task.rspec_opts = %w(--color --format documentation --order default)
+    task.pattern = 'acceptance/**/*_spec.rb'
+  end
+
+  task all: [:unit, :acceptance]
+end
+
+desc 'Run all tests'
+task spec: 'spec:all'
+
+# ------------------------------------------------------------
+# Coverage
+
+desc 'Run all unit tests with coverage'
+task :coverage do
+  ENV['COVERAGE'] = 'true'
+  Rake::Task['spec:unit'].execute
+end
+
+# ------------------------------------------------------------
+# RuboCop
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+# ------------------------------------------------------------
+# TODOs
+
+desc 'List TODOs (from spec/todo.rb)'
+RSpec::Core::RakeTask.new(:todo) do |task|
+  task.rspec_opts = %w(--color --format documentation --order default)
+  task.pattern = 'todo.rb'
+end
+
+# ------------------------------------------------------------
+# Defaults
+
+desc 'Run unit tests, check test coverage, run acceptance tests, check code style'
+task default: [:coverage, 'spec:acceptance', :rubocop]

data/example.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+# Note: This assumes we're running from the root of the resync-client project
+$LOAD_PATH << File.dirname(__FILE__)
+require 'lib/resync/client'
+
+client = Resync::Client.new
+
+# Note: this URI is from resync-simulator: https://github.com/resync/resync-simulator
+source_desc_uri = 'http://localhost:8888/.well-known/resourcesync'
+puts "Source: #{source_desc_uri}"
+source_desc = client.get_and_parse(source_desc_uri) # Resync::SourceDescription
+
+cap_list_resource = source_desc.resource_for(capability: 'capabilitylist')
+cap_list = cap_list_resource.get_and_parse # Resync::CapabilityList
+
+change_list_resource = cap_list.resource_for(capability: 'changelist')
+change_list = change_list_resource.get_and_parse # Resync::ChangeList
+puts "  from:    #{change_list.metadata.from_time}"
+puts "  until:   #{change_list.metadata.until_time}"
+
+changes = change_list.resources # Array<Resync::Resource>
+puts "  changes: #{changes.size}"
+puts
+
+n = changes.size > 5 ? 5 : changes.size
+puts "last #{n} changes of any kind:"
+changes.slice(-n, n).each do |r|
+  puts "  #{r.uri}"
+  puts "    modified at: #{r.modified_time}"
+  puts "    change type: #{r.metadata.change}"
+  puts "    md5:         #{r.metadata.hash('md5')}"
+end
+
+last_update = changes.select { |r| r.metadata.change == Resync::Types::Change::UPDATED }[-1]
+puts 'last update:'
+puts "  #{last_update.uri}"
+puts "    modified at: #{last_update.modified_time}"
+puts "    change type: #{last_update.metadata.change}"
+puts "    md5:         #{last_update.metadata.hash('md5')}"
+
+last_update_response = last_update.get
+puts last_update_response.class
+puts "    content:      #{last_update_response}"
+
+last_update_file = last_update.download_to_temp_file
+last_update_file_contents = File.read(last_update_file)
+puts "    as file:      #{last_update_file}"
+puts "    file content: #{last_update_file_contents}"

data/lib/resync/client/bitstream.rb

@@ -0,0 +1,79 @@
+module Resync
+
+  # A single entry in a ZIP package.
+  class Bitstream
+
+    # ------------------------------------------------------------
+    # Attributes
+
+    # @return [String] the path to the entry within the ZIP file
+    attr_reader :path
+
+    # @return [Resource] the resource describing this bitstream
+    attr_reader :resource
+
+    # @return [Metadata] the metadata for this bitstream
+    attr_reader :metadata
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # Creates a new bitstream for the specified resource.
+    #
+    # @param zipfile [Zip::File] The zipfile to read the bitstream from.
+    # @param resource [Resource] The resource describing the bitstream.
+    def initialize(zipfile:, resource:)
+      self.resource = resource
+      @zip_entry = zipfile.find_entry(@path)
+    end
+
+    # ------------------------------------------------------------
+    # Convenience accessors
+
+    # The (uncompressed) size of the bitstream.
+    def size
+      @size ||= @zip_entry.size
+    end
+
+    # The bitstream, as an +IO+-like object. Subsequent
+    # calls will return the same stream.
+    def stream
+      @stream ||= @zip_entry.get_input_stream
+    end
+
+    # The content of the bitstream. The content will be
+    # read only once.
+    def content
+      @content ||= stream.read
+    end
+
+    # The content type of the bitstream, as per {#metadata}.
+    def mime_type
+      @mime_type ||= metadata.mime_type
+    end
+
+    # ------------------------------------------------------------
+    # Private methods
+
+    private
+
+    def resource=(value)
+      fail ArgumentError, 'nil is not a resource' unless value
+      self.metadata = value.metadata
+      @resource = value
+    end
+
+    def metadata=(value)
+      fail 'no metadata found' unless value
+      self.path = value.path
+      @metadata = value
+    end
+
+    def path=(value)
+      fail 'no path found in metadata' unless value
+      @path = value.start_with?('/') ? value.slice(1..-1) : value
+    end
+
+  end
+
+end

data/lib/resync/client/client.rb

@@ -0,0 +1,58 @@
+require_relative 'version'
+require_relative 'http_helper'
+
+module Resync
+
+  # Utility class for retrieving HTTP content and parsing it as ResourceSync documents.
+  class Client
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # Creates a new +Client+
+    # @param helper [HTTPHelper] the HTTP helper. Defaults to a new HTTP helper with
+    #   +resync-client VERSION+ as the User-Agent string.
+    def initialize(helper: HTTPHelper.new(user_agent: "resync-client #{VERSION}"))
+      @helper = helper
+    end
+
+    # ------------------------------------------------------------
+    # Public methods
+
+    # Gets the content of the specified URI and parses it as a ResourceSync document.
+    def get_and_parse(uri)
+      uri = Resync::XML.to_uri(uri)
+      raw_contents = get(uri)
+      doc = XMLParser.parse(raw_contents)
+      doc.client = self
+      doc
+    end
+
+    # Gets the content of the specified URI as a string.
+    # @param uri [URI, String] the URI to download
+    # @return [String] the content of the URI
+    def get(uri)
+      uri = Resync::XML.to_uri(uri)
+      @helper.fetch(uri: uri)
+    end
+
+    # Gets the content of the specified URI and saves it to a temporary file.
+    # @param uri [URI, String] the URI to download
+    # @return [String] the path to the downloaded file
+    def download_to_temp_file(uri)
+      uri = Resync::XML.to_uri(uri)
+      @helper.fetch_to_file(uri: uri)
+    end
+
+    # Gets the content of the specified URI and saves it to the specified file,
+    # overwriting it if it exists.
+    # @param uri [URI, String] the URI to download
+    # @param path [String] the path to save the download to
+    # @return [String] the path to the downloaded file
+    def download_to_file(uri:, path:)
+      uri = Resync::XML.to_uri(uri)
+      @helper.fetch_to_file(path: path, uri: uri)
+    end
+
+  end
+end

data/lib/resync/client/downloadable.rb

@@ -0,0 +1,35 @@
+require 'resync'
+
+module Resync
+
+  # Adds +get+, +get_raw+, and +get_file+ methods, delegating
+  # to the injected client.
+  #
+  # @see Augmented#client
+  module Downloadable
+    # Delegates to {Client#get_and_parse} to get the contents of
+    # +:uri+ as a ResourceSync document
+    def get_and_parse # rubocop:disable Style/AccessorMethodName
+      client.get_and_parse(uri)
+    end
+
+    # Delegates to {Client#get} to get the contents of this +:uri+
+    def get # rubocop:disable Style/AccessorMethodName
+      client.get(uri)
+    end
+
+    # Delegates to {Client#download_to_temp_file} to download the
+    # contents of +:uri+ to a file.
+    def download_to_temp_file # rubocop:disable Style/AccessorMethodName
+      client.download_to_temp_file(uri)
+    end
+
+    # Delegates to {Client#download_to_file} to download the
+    # contents of +:uri+ to the specified path.
+    # @param path [String] the path to download to
+    def download_to_file(path)
+      client.download_to_file(uri: uri, path: path)
+    end
+  end
+
+end

data/lib/resync/client/dump.rb

@@ -0,0 +1,25 @@
+require_relative 'zip_package'
+
+module Resync
+  # Extends {ChangeDump} and {ResourceDump} to provide
+  # transparent access to the linked bitstream packages
+  module Dump
+    # Injects a +:zip_package+ method into each resource,
+    # downloading the (presumed) bitstream package to a
+    # temp file and returning it as a {ZipPackage}
+    def resources=(value)
+      super
+      resources.each do |r|
+        def r.zip_package
+          @zip_package ||= ZipPackage.new(download_to_temp_file)
+        end
+      end
+    end
+
+    # A list of the {ZipPackage}s for each resource
+    # @return [Array<ZipPackage>] the zip packages for each resource
+    def zip_packages
+      resources.map(&:zip_package)
+    end
+  end
+end

data/lib/resync/client/http_helper.rb

@@ -0,0 +1,111 @@
+require 'net/http'
+require 'tempfile'
+require 'uri'
+require 'mime-types'
+
+module Resync
+
+  # Utility class simplifying GET requests for HTTP/HTTPS resources.
+  #
+  class HTTPHelper
+
+    # ------------------------------------------------------------
+    # Constants
+
+    # The default number of redirects to follow before erroring out.
+    DEFAULT_MAX_REDIRECTS = 5
+
+    # ------------------------------------------------------------
+    # Accessors
+
+    # @!attribute [rw] user_agent
+    #   @return [String] the User-Agent string to send when making requests
+    attr_accessor :user_agent
+
+    # @!attribute [rw] redirect_limit
+    #   @return [Integer] the number of redirects to follow before erroring out
+    attr_accessor :redirect_limit
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # Creates a new +HTTPHelper+
+    #
+    # @param user_agent [String] the User-Agent string to send when making requests
+    # @param redirect_limit [Integer] the number of redirects to follow before erroring out
+    #   (defaults to {DEFAULT_MAX_REDIRECTS})
+    def initialize(user_agent:, redirect_limit: DEFAULT_MAX_REDIRECTS)
+      @user_agent = user_agent
+      @redirect_limit = redirect_limit
+    end
+
+    # ------------------------------------------------------------
+    # Public methods
+
+    # Gets the content of the specified URI as a string.
+    # @param uri [URI] the URI to download
+    # @param limit [Integer] the number of redirects to follow (defaults to {#redirect_limit})
+    # @return [String] the content of the URI
+    def fetch(uri:, limit: redirect_limit)
+      make_request(uri, limit) do |success|
+        # not 100% clear why we need an explicit return here; it
+        # doesn't show up in unit tests but it does in example.rb
+        return success.body
+      end
+    end
+
+    # Gets the content of the specified URI and saves it to a file. If no
+    # file path is provided, saves it to a temporary file.
+    # @param uri [URI] the URI to download
+    # @param path [String] the path to save the download to (optional)
+    # @return [String] the path to the downloaded file
+    def fetch_to_file(uri:, path: nil, limit: redirect_limit)
+      make_request(uri, limit) do |success|
+        file = path ? File.new(path, 'w+') : Tempfile.new(['resync-client', ".#{extension_for(success)}"])
+        open file, 'w' do |out|
+          success.read_body { |chunk| out.write(chunk) }
+        end
+        return file.path
+      end
+    end
+
+    # ------------------------------------------------------------
+    # Private methods
+
+    private
+
+    def make_request(uri, limit, &block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      fail "Redirect limit (#{redirect_limit}) exceeded retrieving URI #{uri}" if limit <= 0
+      req = Net::HTTP::Get.new(uri, 'User-Agent' => user_agent)
+      Net::HTTP.start(uri.hostname, uri.port, use_ssl: (uri.scheme == 'https')) do |http|
+        http.request(req) do |response|
+          case response
+          when Net::HTTPSuccess
+            block.call(response)
+          when Net::HTTPInformation, Net::HTTPRedirection
+            make_request(redirect_uri_for(response, uri), limit - 1, &block)
+          else
+            fail "Error #{response.code}: #{response.message} retrieving URI #{uri}"
+          end
+        end
+      end
+    end
+
+    def extension_for(response)
+      content_type = response['Content-Type']
+      mime_type = MIME::Types[content_type].first || MIME::Types['application/octet-stream'].first
+      mime_type.preferred_extension || 'bin'
+    end
+
+    def redirect_uri_for(response, original_uri)
+      if response.is_a?(Net::HTTPInformation)
+        original_uri
+      else
+        location = response['location']
+        new_uri = URI(location)
+        new_uri.relative? ? original_uri + location : new_uri
+      end
+    end
+
+  end
+end

data/lib/resync/client/resync_extensions.rb

@@ -0,0 +1,85 @@
+require 'resync'
+require_relative 'downloadable'
+require_relative 'dump'
+
+# Extensions to the core Resync classes to simplify retrieval
+module Resync
+
+  # ------------------------------------------------------------
+  # Base classes
+
+  # Injects a {Client} that subclasses can use to fetch
+  # resources and links
+  #
+  # @!attribute [rw] client
+  #   @return [Client] the injected {Client}. Defaults to
+  #     a new {Client} instance.
+  class Augmented
+    attr_writer :client
+
+    def client
+      @client ||= Client.new
+    end
+
+    alias_method :base_links=, :links=
+    private :base_links=
+
+    # Adds a +:client+ method to each link, delegating
+    # to {#client}
+    def links=(value)
+      self.base_links = value
+      self.base_links = value
+      parent = self
+      links.each do |l|
+        l.define_singleton_method(:client) do
+          parent.client
+        end
+      end
+    end
+  end
+
+  # Adds a +:client+ method to each resource, delegating
+  # to {Augmented#client}
+  class BaseResourceList
+    alias_method :base_resources=, :resources=
+    private :base_resources=
+
+    # Adds a +:client+ method to each resource, delegating
+    # to {Augmented#client}
+    def resources=(value)
+      self.base_resources = value
+      parent = self
+      resources.each do |r|
+        r.define_singleton_method(:client) do
+          parent.client
+        end
+      end
+    end
+  end
+
+  # ------------------------------------------------------------
+  # Resource and Link
+
+  # Includes the {Downloadable} module
+  class Resource
+    include Downloadable
+  end
+
+  # Includes the {Link} module
+  class Link
+    include Downloadable
+  end
+
+  # ------------------------------------------------------------
+  # ResourceDump and ChaneDump
+
+  # Includes the {Dump} module
+  class ResourceDump
+    include Dump
+  end
+
+  # Includes the {Dump} module
+  class ChangeDump
+    include Dump
+  end
+end

data/lib/resync/client/version.rb

@@ -0,0 +1,6 @@
+module Resync
+  class Client
+    # The version of this gem.
+    VERSION = '0.1.0'
+  end
+end