resync 0.1.0

data/lib/resync/metadata.rb

@@ -0,0 +1,112 @@
+require_relative 'shared/descriptor'
+require_relative 'xml'
+
+module Resync
+
+  # Metadata about a resource or ResourceSync document. See section 7,
+  # {http://www.openarchives.org/rs/1.0/resourcesync#DocumentFormats Sitemap Document Formats},
+  # in the ResourceSync specification.
+  #
+  # @!attribute [rw] at_time
+  #   @return [Time] the datetime at which assembling a resource list
+  #     began (including resource list indices, resource dumps, etc.)
+  # @!attribute [rw] from_time
+  #   @return [Time] the beginning of the time range represented by
+  #     a change list (including change list indices, change dumps, etc.)
+  # @!attribute [rw] until_time
+  #   @return [Time] the end of the time range represented by
+  #     a change list (including change list indices, change dumps, etc.)
+  # @!attribute [rw] completed_time
+  #   @return the datetime at which assembling a resource list
+  #     ended (including resource list indices, resource dumps, etc.)
+  # @!attribute [rw] change
+  #   @return [Change] the type of change to a resource reported in
+  #     a change list (including change list indices, change dumps, etc.)
+  # @!attribute [rw] capability
+  #   @return [String] identifies the type of a ResourceSync document.
+  class Metadata < Descriptor
+    include ::XML::Mapping
+
+    # ------------------------------------------------------------
+    # Attributes
+
+    root_element_name 'md'
+
+    time_node :at_time, '@at', default_value: nil
+    time_node :from_time, '@from', default_value: nil
+    time_node :until_time, '@until', default_value: nil
+    time_node :completed_time, '@completed', default_value: nil
+    change_node :change, '@change', default_value: nil
+    text_node :capability, '@capability', default_value: nil
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # @param at_time [Time] the datetime at which assembling a resource list
+    #   began (including resource list indices, resource dumps, etc.)
+    # @param from_time [Time] the beginning of the time range represented by
+    #   a change list (including change list indices, change dumps, etc.)
+    # @param until_time [Time] the end of the time range represented by
+    #   a change list (including change list indices, change dumps, etc.)
+    # @param completed_time the datetime at which assembling a resource list
+    #   ended (including resource list indices, resource dumps, etc.)
+    # @param modified_time [Time] The date and time when the referenced resource was last modified.
+    # @param length [Integer] The content length of the referenced resource.
+    # @param mime_type [MIME::Type] The media type of the referenced resource.
+    # @param encoding [String] Any content encoding (if any) applied to the data in the
+    #   referenced resource (e.g. for compression)
+    # @param hashes [Hash<String, String>] Fixity information for the referenced
+    #   resource, as a map from hash algorithm tokens (e.g. +md5+, +sha-256+)
+    #   to hex-encoded digest values.
+    # @param change [Change] the type of change to a resource reported in
+    #   a change list (including change list indices, change dumps, etc.)
+    # @param capability [String] identifies the type of a ResourceSync document.
+    # @param path [String] For +ResourceDumpManifests+ and +ChangeDumpManifests+,
+    #   the path to the referenced resource within the dump ZIP file.
+    def initialize( # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+        at_time: nil,
+        from_time: nil,
+        until_time: nil,
+        completed_time: nil,
+        modified_time: nil,
+
+        length: nil,
+        mime_type: nil,
+        encoding: nil,
+        hashes: {},
+
+        change: nil,
+        capability: nil,
+        path: nil
+    )
+      super(modified_time: modified_time, length: length, mime_type: mime_type, encoding: encoding, hashes: hashes, path: path)
+
+      self.at_time = at_time
+      self.from_time = from_time
+      self.until_time = until_time
+      self.completed_time = completed_time
+
+      self.change = change
+      self.capability = capability
+    end
+
+    # ------------------------------------------------------------
+    # Custom setters
+
+    def at_time=(value)
+      @at_time = time_or_nil(value)
+    end
+
+    def from_time=(value)
+      @from_time = time_or_nil(value)
+    end
+
+    def until_time=(value)
+      @until_time = time_or_nil(value)
+    end
+
+    def completed_time=(value)
+      @completed_time = time_or_nil(value)
+    end
+  end
+end

data/lib/resync/resource.rb

@@ -0,0 +1,72 @@
+require_relative 'shared/augmented'
+require_relative 'xml'
+require_relative 'metadata'
+
+module Resync
+  # A resource (i.e., +<url>+ or +<sitemap>+). See section 7,
+  # {http://www.openarchives.org/rs/1.0/resourcesync#DocumentFormats Sitemap Document Formats},
+  # in the ResourceSync specification.
+  class Resource < Augmented
+    include ::XML::Mapping
+
+    # ------------------------------------------------------------
+    # Attributes
+
+    root_element_name 'url'
+
+    uri_node :uri, 'loc', default_value: nil
+    time_node :modified_time, 'lastmod', default_value: nil
+    changefreq_node :changefreq, 'changefreq', default_value: nil
+    numeric_node :priority, 'priority', default_value: nil
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # @param modified_time [Time] The date and time when the referenced resource was last modified.
+    # @param changefreq [ChangeFrequency] how frequently the referenced resource is likely to change.
+    # @param priority [Number] the priority of this resource relative to other resources from the
+    #   same provider. Allows robots to decide which resources to crawl or harvest.
+    #   Values should be in the range 0-1.0 (inclusive), where 0 is the lowest priority
+    #   and 1.0 is the highest.
+    # @param links [Array<Link>] related links (i.e. +<rs:ln>+).
+    # @param metadata [Metadata] metadata about this resource.
+    def initialize( # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+        uri:,
+        modified_time: nil,
+        changefreq: nil,
+        priority: nil,
+        links: [],
+        metadata: nil
+    )
+      super(links: links)
+      self.uri = uri
+      self.modified_time = modified_time
+      self.changefreq = changefreq
+      self.priority = priority
+      self.metadata = metadata
+    end
+
+    # ------------------------------------------------------------
+    # Custom setters
+
+    def uri=(value)
+      @uri = XML.to_uri(value)
+    end
+
+    # ------------------------------------------------------------
+    # Public methods
+
+    def capability
+      metadata ? metadata.capability : nil
+    end
+
+    # ------------------------------------------------------------
+    # Overrides
+
+    # ResourceSync schema requires '##other' elements to appear last
+    def self.all_xml_mapping_nodes(options = { mapping: nil, create: true })
+      xml_mapping_nodes(options) + superclass.all_xml_mapping_nodes(options)
+    end
+
+  end
+end

data/lib/resync/resource_dump.rb

@@ -0,0 +1,15 @@
+require_relative 'shared/base_resource_list'
+require_relative 'xml'
+
+module Resync
+  # A resource dump. See section 11.1,
+  # "{http://www.openarchives.org/rs/1.0/resourcesync#ResourceDump Resource Dump}",
+  # in the ResourceSync specification.
+  class ResourceDump < BaseResourceList
+    include ::XML::Mapping
+
+    # The capability provided by this type.
+    CAPABILITY = 'resourcedump'
+
+  end
+end

data/lib/resync/resource_dump_manifest.rb

@@ -0,0 +1,15 @@
+require_relative 'shared/base_resource_list'
+require_relative 'xml'
+
+module Resync
+  # A resource dump. See section 11.2,
+  # "{http://www.openarchives.org/rs/1.0/resourcesync#ResourceDumpManifest Resource Dump Manifest}",
+  # in the ResourceSync specification.
+  class ResourceDumpManifest < BaseResourceList
+    include ::XML::Mapping
+
+    # The capability provided by this type.
+    CAPABILITY = 'resourcedump-manifest'
+
+  end
+end

data/lib/resync/resource_list.rb

@@ -0,0 +1,15 @@
+require_relative 'shared/base_resource_list'
+require_relative 'xml'
+
+module Resync
+  # A resource list. See section 10.1,
+  # "{http://www.openarchives.org/rs/1.0/resourcesync#ResourceList Resource List}",
+  # in the ResourceSync specification.
+  class ResourceList < BaseResourceList
+    include ::XML::Mapping
+
+    # The capability provided by this type.
+    CAPABILITY = 'resourcelist'
+
+  end
+end

data/lib/resync/resource_list_index.rb

@@ -0,0 +1,15 @@
+require_relative 'shared/base_resource_list'
+require_relative 'shared/sitemap_index'
+
+module Resync
+  # A resource list index. See section 10.2,
+  # "{http://www.openarchives.org/rs/1.0/resourcesync#ResourceListIndex Resource List Index}",
+  # in the ResourceSync specification.
+  class ResourceListIndex < BaseResourceList
+    include ::XML::Mapping
+    include SitemapIndex
+
+    # The capability provided by this type.
+    CAPABILITY = 'resourcelist'
+  end
+end

data/lib/resync/shared/augmented.rb

@@ -0,0 +1,76 @@
+require_relative '../link'
+require_relative '../metadata'
+
+module Resync
+
+  # Base class for elements augmented with ResourceSync-specific child elements.
+  #
+  # @!attribute [rw] links
+  #   @return [Array<Link>] related links.
+  # @!attribute [rw] metadata
+  #   @return [Metadata] metadata about this object.
+  class Augmented
+    include ::XML::Mapping
+
+    # ------------------------------------------------------------
+    # Class methods
+
+    # ResourceSync-specific tags needing to be prefixed with +rs+ on output
+    RS_TAGS = Set['ln', 'md']
+    private_constant :RS_TAGS
+
+    # Adds the +rs+ namespace prefix
+    def self.prefix_rewriter(obj, xml, default_obj_to_xml)
+      default_obj_to_xml.call(obj, xml)
+      xml.each_element do |e|
+        e.name = "rs:#{e.name}" if RS_TAGS.include?(e.name)
+      end
+    end
+    private_class_method :prefix_rewriter
+
+    # ------------------------------------------------------------
+    # Attributes
+
+    array_node :links, 'ln', class: Link, default_value: [], writer: method(:prefix_rewriter)
+    object_node :metadata, 'md', class: Metadata, default_value: nil, writer: method(:prefix_rewriter)
+
+    use_mapping :sitemapindex
+    array_node :links, 'ln', class: Link, default_value: [], writer: method(:prefix_rewriter), sub_mapping: :_default
+    object_node :metadata, 'md', class: Metadata, default_value: nil, writer: method(:prefix_rewriter), sub_mapping: :_default
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # Creates a new +Augmented+ instance with the specified links and metadata.
+    #
+    # @param links [Array<Link>] related links (i.e. +<rs:ln>+).
+    # @param metadata [Metadata] metadata about this resource.
+    def initialize(links: [], metadata: nil)
+      self.links = links
+      self.metadata = metadata
+    end
+
+    # ------------------------------------------------------------
+    # Custom accessors
+
+    # Sets the +links+ list. +nil+ is treated as an empty list.
+    def links=(value)
+      @links = value || []
+    end
+
+    # Finds links with the specified relation.
+    # @param rel [String] the relation.
+    # @return [Array<Link>] those links having that relation, or an empty array if none exist.
+    def links_for(rel:)
+      links.select { |l| l.rel == rel }
+    end
+
+    # Shortcut to find the first link with the specified relation (in ResourceSync there often
+    # should be only one link with a particular relation)
+    # @param rel [String] the relation.
+    # @return [Link] the first link having that relation, or nil if none exists.
+    def link_for(rel:)
+      links.find { |l| l.rel == rel }
+    end
+  end
+end

data/lib/resync/shared/base_resource_list.rb

@@ -0,0 +1,117 @@
+require_relative 'augmented'
+require_relative '../resource'
+require_relative '../metadata'
+
+module Resync
+  # Base class for root elements containing a list of resources
+  # (i.e., +<urlset>+ and +<sitemapindex>+ elements). Subclasses
+  # must define a +CAPABILITY+ constant identifying the capability
+  # they represent (e.g. +resourcelist+, +changelist+).
+  #
+  # @!attribute [rw] resources
+  #   @return [Array<Resource>] the +<url>+ or +<sitemap>+ elements contained in this list.
+  class BaseResourceList < Augmented
+    include ::XML::Mapping
+
+    # ------------------------------------------------------------
+    # Attributes
+
+    root_element_name 'urlset'
+    array_node :resources, 'url', class: Resource, default_value: []
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # Creates a new +BaseResourceList+.
+    #
+    # @param resources [Array<Resource>] The +<url>+ or +<sitemap>+ elements contained in this list.
+    # @param links [Array<Link>] Related links (+<rs:ln>+).
+    # @param metadata [Metadata] Metadata about this list. The +capability+ of the metadata must match this
+    #   implementation class' +CAPABILITY+ constant.
+    # @raise [ArgumentError] if the specified metadata does not have the correct +capability+ attribute for this list type.
+    def initialize(resources: [], links: [], metadata: nil)
+      super(links: links)
+      self.resources = resources
+      self.metadata = metadata
+    end
+
+    # ------------------------------------------------------------
+    # Custom setters
+
+    # Sets the +resources+ list. +nil+ is treated as an empty list.
+    def resources=(value)
+      @resources = value || []
+    end
+
+    # Sets the metadata.
+    #
+    # @raise [ArgumentError] if the specified metadata does not have the correct +capability+ attribute for
+    #   this list type.
+    def metadata=(value)
+      @metadata = metadata_with_correct_capability(value)
+    end
+
+    # ------------------------------------------------------------
+    # Custom getters
+
+    def capability
+      @metadata.capability
+    end
+
+    # Finds resources with the specified capability.
+    # @param capability [String] the capability.
+    # @return [Array<Resource>] those resources having that capability, or an empty array if none exist.
+    def resources_for(capability:)
+      resources.select { |l| l.capability == capability }
+    end
+
+    # Shortcut to find the first resource with the specified capability (in ResourceSync there often
+    # should be only one resource with a particular capability)
+    # @param capability [String] the capability.
+    # @return [Resource] the first resource having that capability, or nil if none exists.
+    def resource_for(capability:)
+      resources.find { |l| l.capability == capability }
+    end
+
+    # ------------------------------------------------------------
+    # Overrides
+
+    # Overrides +::XML::Mapping.pre_save+ to declare the Sitemap and ResourceSync namespaces.
+    # Used for writing.
+    def pre_save(options = { mapping: :_default })
+      xml = super(options)
+      xml.add_namespace('http://www.sitemaps.org/schemas/sitemap/0.9')
+      xml.add_namespace('rs', 'http://www.openarchives.org/rs/terms/')
+      xml
+    end
+
+    # Initializes the +:_default+ and +:sitemapindex+ mappings on all subclasses, and sets the corresponding
+    # root element names (+<urlset>+ and +<sitemapindex>+)
+    def self.inherited(base)
+      base.use_mapping :_default
+      base.root_element_name 'urlset'
+      base.use_mapping :sitemapindex
+      base.root_element_name 'sitemapindex'
+    end
+
+    # ------------------------------------------------------------
+    # Private methods
+
+    private
+
+    # ------------------------------
+    # Parameter validators
+
+    # Validates the +capability+ attribute in the specified metadata.
+    # @raise [ArgumentError] if the specified metadata does not have the correct +capability+ attribute for this list type.
+    def metadata_with_correct_capability(metadata)
+      capability = self.class::CAPABILITY
+      fail ArgumentError, "Missing constant #{self.class}::CAPABILITY" unless capability
+      return Metadata.new(capability: capability) unless metadata
+      fail ArgumentError, "#{metadata} does not appear to be metadata" unless metadata.respond_to?('capability')
+      fail ArgumentError, "Wrong capability for #{self.class.name} metadata; expected '#{capability}', was '#{metadata.capability}'" unless metadata.capability == capability
+      metadata
+    end
+
+  end
+end

data/lib/resync/shared/descriptor.rb

@@ -0,0 +1,135 @@
+require 'mime/types'
+require_relative '../xml'
+
+module Resync
+  # Base class for ResourceSync-specific elements describing a
+  # resource or link.
+  #
+  # @!attribute [rw] modified_time
+  #   @return [Time] the date and time when the referenced resource was last modified.
+  # @!attribute [rw] length
+  #   @return [Integer] the content length of the referenced resource.
+  # @!attribute [rw] mime_type
+  #   @return [MIME::Type] the media type of the referenced resource.
+  # @!attribute [rw] encoding
+  #   @return [String] the content encoding (if any) applied to the data in the
+  #     referenced resource (e.g. for compression)
+  # @!attribute [rw] hashes
+  #   @return [Hash<String, String>] fixity information for the referenced
+  #     resource, as a map from hash algorithm tokens (e.g. +md5+, +sha-256+)
+  #     to hex-encoded digest values.
+  # @!attribute [rw] path
+  #   @return [String] for +ResourceDumpManifests+ and +ChangeDumpManifests+,
+  #     the path to the referenced resource within the dump ZIP file.
+  class Descriptor
+    include ::XML::Mapping
+
+    # ------------------------------------------------------------
+    # Attributes
+
+    time_node :modified_time, '@modified', default_value: nil
+    numeric_node :length, '@length', default_value: nil
+    mime_type_node :mime_type, '@type', default_value: nil
+    text_node :encoding, '@encoding', default_value: nil
+    hash_codes_node :hashes, '@hash', default_value: nil
+    text_node :path, '@path', default_value: nil
+
+    # ------------------------------------------------------------
+    # Initializer
+
+    # Creates a new +Descriptor+ instance with the specified fields.
+    #
+    # @param modified_time [Time] The date and time when the referenced resource was last modified.
+    # @param length [Integer] The content length of the referenced resource.
+    # @param mime_type [MIME::Type] The media type of the referenced resource.
+    # @param encoding [String] Any content encoding (if any) applied to the data in the
+    #   referenced resource (e.g. for compression)
+    # @param hashes [Hash<String, String>] Fixity information for the referenced
+    #   resource, as a map from hash algorithm tokens (e.g. +md5+, +sha-256+)
+    #   to hex-encoded digest values.
+    # @param path [String] For +ResourceDumpManifests+ and +ChangeDumpManifests+,
+    #   the path to the referenced resource within the dump ZIP file.
+    def initialize( # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+        modified_time: nil,
+        length: nil,
+        mime_type: nil,
+        encoding: nil,
+        hashes: nil,
+        path: nil
+    )
+      self.modified_time = modified_time
+      self.length = length
+      self.mime_type = mime_type
+      self.encoding = encoding
+      self.hashes = hashes
+      self.path = path
+    end
+
+    # ------------------------------------------------------------
+    # Custom setters
+
+    def modified_time=(value)
+      @modified_time = time_or_nil(value)
+    end
+
+    def length=(value)
+      @length = natural_number_or_nil(value)
+    end
+
+    def mime_type=(value)
+      @mime_type = mime_type_or_nil(value)
+    end
+
+    def hashes=(value)
+      @hashes = Descriptor.hash_of_hashcodes(value)
+    end
+
+    # ------------------------------------------------------------
+    # Public methods
+
+    # Gets the hash value for the specified algorithm.
+    #
+    # @param algorithm [String] The token (e.g. +md5+, +sha-256+) for the hash algorithm.
+    # @return [String] The hex-encoded digest value.
+    def hash(algorithm)
+      hashes[algorithm]
+    end
+
+    # ------------------------------------------------------------
+    # Private methods
+
+    private
+
+    # ------------------------------
+    # Parameter validators
+
+    def time_or_nil(time)
+      fail ArgumentError, "time #{time} is not a Time" if time && !time.is_a?(Time)
+      time
+    end
+
+    def natural_number_or_nil(value)
+      fail ArgumentError, "value #{value} must be a non-negative integer" if value && (!value.is_a?(Integer) || value < 0)
+      value
+    end
+
+    def mime_type_or_nil(mime_type)
+      return nil unless mime_type
+      return mime_type if mime_type.is_a?(MIME::Type)
+
+      mt = MIME::Types[mime_type].first
+      return mt if mt
+
+      MIME::Type.new(mime_type)
+    end
+
+    # ------------------------------
+    # Conversions
+
+    def self.hash_of_hashcodes(hashes)
+      return {} unless hashes
+      return hashes if hashes.is_a?(Hash)
+      hashes.split(/[[:space:]]+/).map { |hash| hash.split(':') }.to_h
+    end
+  end
+end

data/lib/resync/shared/sitemap_index.rb

@@ -0,0 +1,32 @@
+require 'xml/mapping'
+require_relative '../resource'
+
+module Resync
+  module SitemapIndex
+    include ::XML::Mapping
+
+    def self.included(base)
+      base.extend(ClassMethods)
+
+      base.use_mapping :sitemapindex
+      base.root_element_name 'sitemapindex'
+      base.array_node :resources, 'sitemap', class: Resource, default_value: [], sub_mapping: :_default
+    end
+
+    # Ensures that an index is always written as a +<sitemapindex>+.
+    # Overrides +::XML::Mapping.save_to_xml+.
+    def save_to_xml(options = { mapping: :_default })
+      options = options.merge(mapping: :sitemapindex)
+      super(options)
+    end
+
+    # Ensures that an index is always read as a +<sitemapindex>+.
+    # Overrides +::XML::Mapping::ClassMethods.load_from_xml+.
+    module ClassMethods
+      def load_from_xml(xml, options = { mapping: :_default })
+        options = options.merge(mapping: :sitemapindex)
+        super(xml, options)
+      end
+    end
+  end
+end

data/lib/resync/shared/sorted_resource_list.rb

@@ -0,0 +1,60 @@
+require_relative 'base_resource_list'
+
+module Resync
+  # An extension to +BaseResourceList+ for resource lists that
+  # should be sorted by modification time.
+  class SortedResourceList < BaseResourceList
+
+    # ------------------------------------------------------------
+    # Custom setters
+
+    # Sets the +resources+ list, sorting the resources by modification
+    # time. (+nil+ is treated as an empty list.) Resources without
+    # modification times will be sorted to the end.
+    def resources=(value)
+      @resources = sorted(value)
+      @resources_by_uri = by_uri(@resources)
+    end
+
+    # ------------------------------------------------------------
+    # Custom accessors
+
+    attr_reader :resources_by_uri
+
+    def latest_for(uri:)
+      uri = XML.to_uri(uri)
+      @resources_by_uri[uri].last
+    end
+
+    def all_uris
+      @resources_by_uri.keys
+    end
+
+    # ------------------------------------------------------------
+    # Private methods
+
+    private
+
+    # ------------------------------
+    # Conversions
+
+    def sorted(value)
+      return [] unless value
+      value.sort do |left, right|
+        if left.modified_time && right.modified_time
+          left.modified_time <=> right.modified_time
+        else
+          right.modified_time ? 1 : -1
+        end
+      end
+    end
+
+    def by_uri(resources)
+      by_uri = {}
+      resources.each do |r|
+        (by_uri[r.uri] ||= []) << r
+      end
+      by_uri
+    end
+  end
+end

data/lib/resync/source_description.rb

@@ -0,0 +1,14 @@
+require_relative 'shared/base_resource_list'
+
+module Resync
+  # A change list. See section 8,
+  # "{http://www.openarchives.org/rs/1.0/resourcesync#SourceDesc Describing the Source}",
+  # in the ResourceSync specification.
+  class SourceDescription < BaseResourceList
+    include ::XML::Mapping
+
+    # The capability provided by this type.
+    CAPABILITY = 'description'
+
+  end
+end