ecfr 1.0.0

data/lib/ecfr/client.rb

@@ -0,0 +1,318 @@
+module Ecfr
+  #
+  # Provides a configurable Faraday connection
+  # (via {Ecfr::Configuration} options) and returns an
+  # instance of the calling class.
+  #
+  # The request can be additionally modified before flight
+  # using the {Ecfr.config.request_hook} - this is useful
+  # for adding additional headers like request tracking IDs, etc.
+  #
+  class Client
+    include Ecfr::ParallelClient
+
+    class Error < StandardError
+      attr_reader :record
+
+      def initialize(record = nil, message = "")
+        @record = record
+        super(message)
+      end
+    end
+
+    class BadRequest < Error; end
+    class Forbidden < Error; end
+    class InvalidRequest < Error; end
+    class RecordNotFound < Error; end
+    class ResponseError < Error; end
+    class ServerError < Error; end
+    class Unauthorized < Error; end
+    class UnknownStatusCode < Error; end
+
+    #
+    # Faraday client configured for eCFR usage.
+    #
+    # @param [<Hash>] client_options
+    # @option client_options [<Hash>] :basic_auth :username
+    #   and :password basic auth credentials
+    # @option client_options [<Hash>] :timeout value in
+    #   seconds for a custom timeout value
+    # @param [<Proc>] &block the faraday connection is yielded
+    #   to the provided block allowing additional customization
+    #
+    # @return [<Faraday::Connection>] Faraday connection object
+    #
+    # @note: base_url is defined in each service, but we allow an override
+    #   for use cases that need to use the raw client/parallel_client
+    #
+    def self.client(base_url: nil, client_options: {}, &block)
+      client_pool(base_url || self.base_url, client_options)
+    end
+
+    #
+    # Returns a client connection, either new or from cache, based
+    # on origin and client_options. This allows connection sharing
+    # across all requests to the same origin with the current thread.
+    #
+    # @param [<String>] base_url a well formed url
+    # @param [<Hash>] client_options see Ecfr::Client.client
+    #
+    # @return [<Faraday::Connection>] Faraday connection object
+    #
+    def self.client_pool(base_url, client_options)
+      uri = URI.parse(base_url)
+
+      cache_key = "ecfr-client-pool-#{uri.host}-#{Digest::MD5.hexdigest(client_options.to_s)}"
+
+      RequestStore.fetch(cache_key) do
+        Faraday.new(url: uri.to_s) do |faraday|
+          if client_options[:adapter]
+            faraday.adapter client_options[:adapter]
+          else
+            faraday.adapter :net_http_persistent, pool_size: 5
+          end
+
+          faraday.request :url_encoded # form-encode POST params
+          faraday.options[:timeout] = client_options[:timeout] || Ecfr.config.timeout
+
+          faraday.use Faraday::UserAgent::Middleware, Ecfr.config.user_agent
+
+          if Ecfr.config.log_http_requests
+            # https://lostisland.github.io/faraday/middleware/instrumentation
+            faraday.request :instrumentation
+
+            faraday.response :logger, Ecfr.config.logger, Ecfr.config.logger_options
+          end
+
+          if client_options[:basic_auth]
+            faraday.request(:basic_auth,
+              client_options.dig(:basic_auth, :username),
+              client_options.dig(:basic_auth, :password))
+          end
+
+          yield(faraday) if block_given?
+        end
+      end
+    end
+
+    #
+    # A wrapper around Faraday connection methods that provides
+    # support for more complex handling of the response to support
+    # our classes. This is the recommended method for most endpoints
+    # that return JSON.
+    #
+    # @param [<Symbol, String>] method lowercase HTTP verb -
+    #   :get, :post, :delete, and :purge are supported
+    # @param [<String>] path added to the service base url to form
+    #   the full URL for the API endpoint
+    # @param [<Hash>] params parameters to be added to the request
+    # @param [<Hash>] client_options see {Ecfr::Client.client} for
+    #   valid options
+    # @param [<Hash>] perform_options
+    # @option perform_options [<Boolean>] :parse_response (true) whether
+    #   return a parsed response (via JSON.parse) or just return the
+    #   response body.
+    # @option perform_options [<Symbol, String>] :attributes_key the
+    #   key from the JSON response to return as the results. Provides
+    #   support for API endpoints that may not use a standard key for
+    #   for every request path.
+    #
+    # @return [<Object>] instantiated Ecfr class with the
+    #   response_status set.
+    #
+    def self.perform(method, path, params: {}, client_options: {}, perform_options: {})
+      path = [self::SERVICE_PATH, path].compact.join("/")
+
+      default_perform_options = {parse_response: true}
+      perform_options = default_perform_options.merge(perform_options)
+
+      # cache response data, not the instantiation of the class - this allows
+      # for multiple user classes to inherit from a single gem defined class
+      if Ecfr.config.cache_responses
+        cache_key = cache_key(method, path, params)
+
+        response = instrument("Ecfr::Perform #{cache_key.to_s.tr("\"", "'")}") do
+          RequestStore.fetch(cache_key) do
+            puts "Request not in eCFR gem cache, fetching..."
+
+            response = fetch(method, path, params: params, client_options: client_options)
+
+            cache_base_response(response, path, params) if respond_to?(:cache_base_response)
+
+            response
+          end
+        end
+      else
+        response = instrument("Ecfr::Perform #{path}/#{params}") do
+          fetch(method, path, params: params, client_options: client_options)
+        end
+      end
+
+      build(
+        response: response,
+        request_data: {
+          method: method,
+          path: path,
+          params: params
+        },
+        build_options: perform_options
+      )
+    end
+
+    def self.instrument(step, &block)
+      if defined?(Rack::MiniProfiler)
+        Rack::MiniProfiler.step(step) do
+          yield(block)
+        end
+      else
+        yield(block)
+      end
+    end
+    private_class_method :instrument
+
+    def self.cache_key(method, path, params)
+      "#{method}:#{path}:#{Digest::MD5.hexdigest(params.sort.to_s)}"
+    end
+
+    def self.fetch(method, path, params:, client_options:)
+      instrument("Ecfr::Fetch") do
+        send(method, path, params, client_options)
+      end
+    end
+    private_class_method :fetch
+
+    #
+    # Transforms the response into our objects. Primarily exists
+    # to support testing in the upstream user applications.
+    #
+    # See the .perform/.fetch method for argument signatures
+    #
+    def self.build(response:, request_data: {}, build_options: {})
+      default_build_options = {parse_response: true}
+      build_options = default_build_options.merge(build_options)
+
+      if build_options[:parse_response]
+        results = JSON.parse(response.body)
+
+        results = results[build_options[:attributes_key].to_s] if build_options[:attributes_key]
+      else
+        results = response.body
+      end
+
+      options = {
+        response_status: response.status,
+        request_data: request_data
+      }
+
+      options.merge!(build_options[:init_data]) if build_options[:init_data]
+
+      # json results are dynamically cast into subclasses -
+      # in order support upstream inheritance based overrides
+      # of these subclasses, we capture when inheritance happens
+      # and instantiate that class instead
+      if const_defined?("#{self}::KLASS")
+        self::KLASS.new(results, options)
+      else
+        new(results, options)
+      end
+    end
+
+    def self.get(path, params = {}, client_options = {})
+      c = client(client_options: client_options)
+
+      execute do
+        c.get(path, params) do |req|
+          Ecfr.config.request_hook.call(req)
+        end
+      end
+    end
+
+    def self.post(path, params = {}, client_options = {})
+      c = client(client_options: client_options)
+
+      execute do
+        c.post(path, params) do |req|
+          Ecfr.config.request_hook.call(req)
+        end
+      end
+    end
+
+    def self.delete(path, params = {}, client_options = {})
+      c = client(client_options: client_options)
+
+      execute do
+        c.delete(path, params) do |req|
+          Ecfr.config.request_hook.call(req)
+        end
+      end
+    end
+
+    Faraday::Connection::METHODS << :purge
+    def self.purge(path, params = {}, client_options = {})
+      c = client(client_options: client_options)
+
+      execute do
+        c.run_request(:purge, path, nil, nil) do |request|
+          request.params.update(params)
+          Ecfr.config.request_hook.call(request)
+        end
+      end
+    end
+
+    #
+    # Wrapper around a Faraday response to handle known response
+    # statuses and wrap them in Ecfr specific errors that can
+    # be handled upstream in a consistent manner.
+    #
+    # @param [<Proc>] &block Faraday response
+    #
+    # @return [<Faraday::Response, Client::Error>]
+    #
+    def self.execute(&block)
+      handle_response(
+        yield(block)
+      )
+    rescue Faraday::ConnectionFailed
+      raise ResponseError.new(nil, "Hostname lookup failed")
+    rescue Faraday::TimeoutError
+      raise ResponseError.new(nil, "Request timed out")
+    end
+
+    #
+    # Wrapper to return 4XX-5XX status codes as error objects.
+    # We attempt to parse the response body to check for error
+    # messages in JSON form and pass those along.
+    #
+    # @param [<Faraday::Response>] response
+    #
+    # @return [<Error>]
+    #
+    def self.handle_response(response)
+      return response if [200, 201, 204].include?(response.status)
+
+      # handle errors
+      begin
+        errors = JSON.parse(response.body)
+      rescue
+        errors = nil
+      end
+
+      case response.status
+      when 400
+        raise BadRequest.new(errors)
+      when 401
+        raise Unauthorized.new(errors)
+      when 403
+        raise Forbidden.new(errors)
+      when 404
+        raise RecordNotFound
+      when 422
+        raise InvalidRequest
+      when 500
+        raise ServerError.new(errors)
+      else
+        raise UnknownStatusCode.new(errors, "Status code: #{response.status}")
+      end
+    end
+  end
+end

data/lib/ecfr/common/hierarchy.rb

@@ -0,0 +1,35 @@
+module Ecfr
+  module Common
+    class Hierarchy
+      include AttributeMethodDefinition
+      extend Extensible
+
+      attribute :title,
+        desc: "Title number"
+      attribute :subtitle,
+        desc: "Subtitle identifier"
+      attribute :chapter,
+        desc: "Chapter indentifier"
+      attribute :subchapter,
+        desc: "Subchapter identifier"
+      attribute :part,
+        desc: "Part identifier"
+      attribute :subpart,
+        desc: "Subpart identifier"
+      attribute :subject_group,
+        desc: "Subject group identifier"
+      attribute :section,
+        desc: "Section identifier"
+      attribute :appendix,
+        desc: "Appendix identifier"
+      attribute :paragraph,
+        desc: "Paragraph citation"
+
+      def to_hash
+        @attributes.each_with_object({}) { |attr, hsh|
+          hsh[attr[0]] = attr[1]
+        }.compact.symbolize_keys
+      end
+    end
+  end
+end

data/lib/ecfr/configuration.rb

@@ -0,0 +1,58 @@
+module Ecfr
+  #
+  # Provides configuration of various aspects of the gem
+  #
+  class Configuration
+    class MissingGem < StandardError; end
+    class MissingUserAgent < StandardError; end
+
+    CONFIG_DEFAULTS = {
+      base_url: "https://www.ecfr.gov",
+      ofr_profile_service_base_url: "https://www.federalregister.gov/my/profile",
+      # service url overrides
+      admin_service_url: nil,
+      diff_service_url: nil,
+      ofr_profile_service_url: nil,
+      prince_xml_service_url: nil,
+      renderer_service_url: nil,
+      search_service_url: nil,
+      subscriptions_service_url: nil,
+      versioner_service_url: nil,
+      # basic auth - some endpoints require auth
+      ecfr_basic_auth_username: nil,
+      ecfr_basic_auth_password: nil,
+      # request modification
+      request_hook: ->(req) { req },
+      user_agent: nil,
+      # client modifications
+      timeout: 10,
+      prince_xml_service_pdf_timeout: 10,
+      # response modification
+      log_http_requests: false,
+      logger: nil,
+      logger_options: {},
+      # caching
+      cache_responses: false
+    }
+
+    attr_accessor(*CONFIG_DEFAULTS.each.map { |k, v| k })
+
+    def initialize
+      CONFIG_DEFAULTS.each do |k, v|
+        instance_variable_set("@#{k}", v)
+      end
+    end
+
+    def validate!
+      raise MissingUserAgent, "A user agent must be provided" if user_agent.blank?
+
+      if cache_responses
+        begin
+          Gem::Specification.find_by_name("request_store")
+        rescue Gem::LoadError
+          raise MissingGem, "the RequestStore gem must be installed if response caching is enabled"
+        end
+      end
+    end
+  end
+end

data/lib/ecfr/constants.rb

@@ -0,0 +1,21 @@
+module Ecfr
+  module Constants
+    module ChangeTypes
+      KNOWN_CHANGE_TYPES = {
+        cross_references: "Regulations linked from the above-dated Federal Register:",
+        delayed: "Regulations delayed in the above-dated Federal Register:",
+        delayed_withdrawn: "Regulations delayed or withdrawn in the above-dated Federal Register:",
+        delayed_withdrawn_extended: "Regulations delayed, withdrawn, or extended in the above-dated Federal Register:",
+        effective: "Effective regulations inserted from the above-dated Federal Register:",
+        effective_cross_references: "Regulations inserted that were previously linked and became effective on the date listed above:",
+        expired: "Effective dates that expire on this date:",
+        extended: "Regulations extended in the above-dated Federal Register",
+        initial: "Initial import of this content - change type is indeterminate"
+      }
+    end
+
+    module Hierarchy
+      HIERARCHY_LEVELS = %w[title subtitle chapter subchapter part subpart subject_group section appendix]
+    end
+  end
+end

data/lib/ecfr/default_documentation_setup.rb

@@ -0,0 +1,39 @@
+module Ecfr
+  module DefaultDocumentationSetup
+    module ClassMethods
+      DOCUMENTATION_PATH = "documentation.json"
+
+      def api_documentation(args = {})
+        perform(
+          documentation_config[:method],
+          DOCUMENTATION_PATH,
+          params: args.compact
+        )
+      end
+
+      # .documentation_config is provided to support use cases
+      # in which the user wants to construct their own
+      # handling of status checks, e.g. to wrap in custom
+      # errors or run in parallel
+      def documentation_config
+        {
+          url: [
+            base_url,
+            self::SERVICE_PATH,
+            "/#{DOCUMENTATION_PATH}"
+          ].reject(&:blank?).join,
+          method: :get
+        }
+      end
+    end
+
+    def self.included(base)
+      base.instance_eval do
+        attribute :description
+        attribute :paths
+      end
+
+      base.extend(ClassMethods)
+    end
+  end
+end

data/lib/ecfr/default_status_setup.rb

@@ -0,0 +1,46 @@
+module Ecfr
+  module DefaultStatusSetup
+    module ClassMethods
+      STATUS_PATH = "status.json"
+
+      def status
+        perform(
+          status_config[:method],
+          STATUS_PATH
+        )
+      end
+
+      # .status_config is provided to support use cases
+      # in which the user wants to construct their own
+      # handling of status checks, e.g. to wrap in custom
+      # errors or run in parallel
+      def status_config
+        url = [
+          base_url,
+          self::SERVICE_PATH,
+          STATUS_PATH
+        ].reject(&:blank?)
+          .join("/")
+          .gsub(/(?<!:)\/\//, "/") # remove extraneous double slashes '//'
+
+        {
+          url: url,
+          method: :get
+        }
+      end
+    end
+
+    def self.included(base)
+      base.instance_eval do
+        attribute :checks
+        attribute :status
+      end
+
+      base.extend(ClassMethods)
+    end
+
+    def status_code
+      response_status
+    end
+  end
+end

data/lib/ecfr/diff_service/base.rb

@@ -0,0 +1,17 @@
+module Ecfr
+  module DiffService
+    class Base < Ecfr::Base
+      require_relative "status"
+
+      SERVICE_PATH = "/api/diff"
+
+      def self.base_url
+        Ecfr.config.diff_service_url || Ecfr.config.base_url
+      end
+
+      def self.service_name
+        "Diff Service"
+      end
+    end
+  end
+end

data/lib/ecfr/diff_service/status.rb

@@ -0,0 +1,34 @@
+module Ecfr
+  module DiffService
+    class Status < Base
+      include DefaultStatusSetup
+
+      STATUS_PATH = "v1"
+
+      def self.status
+        perform(
+          status_config[:method],
+          STATUS_PATH,
+          params: status_config[:options],
+          perform_options: {parse_response: false}
+        )
+      end
+
+      # .status_config is provided to support use cases
+      # in which the user wants to construct their own
+      # handling of status checks, e.g. to wrap in custom
+      # errors or run in parallel
+      def self.status_config
+        {
+          url: "#{base_url}/#{SERVICE_PATH}/#{STATUS_PATH}",
+          method: :post,
+          response_type: "html",
+          options: {
+            new: "old test",
+            old: "new test"
+          }
+        }
+      end
+    end
+  end
+end

data/lib/ecfr/extensible.rb

@@ -0,0 +1,45 @@
+module Ecfr
+  # Users of this gem may want to subclass our API client classes to add
+  # app-specific functionality; for example, adding to functionality of the
+  # `Ancestor` or `NodeSummary` classes.
+  #
+  # Because our the gem's client classes instantiate other objects (eg Ancestors
+  # creates NodeSummary objects), we need to make it possible for the user to
+  # get instances of their own NodeSummary subclass created when they use their
+  # Ancestor subclass
+  #
+  # To support upstream modification of these classes we record when inheritance
+  # happens and use that to instantiate the expected class at run time.
+  #
+  # This means that these subclasses can only be inherited from / modified by a
+  # single upstream class (and will always be instantiated as that class).
+  # We skip this bookkeeping when the inheritance is from one of the classes
+  # that directly inherits from a service Base class as it is reasonable
+  # to expect that inheritance in that scenario works as all other Ruby
+  # inheritance.
+  #
+  module Extensible
+    class AlreadyExtendedError < StandardError; end
+
+    def inherited(subclass)
+      # Skip for class inheritance within the gem.
+      return if subclass.to_s.starts_with?("Ecfr::")
+
+      # Skip for inheritance tracking of top level gem classes
+      return if superclass.to_s.ends_with?("Service::Base")
+
+      # Only set if not already defined (constants can't be redefined)
+      if const_defined?(:KLASS)
+        # during autoreloading in dev type environments subclassing will appear
+        # to happen twice - we can safely ignore that - if the class differs
+        # from that already defined then we want to notify the user
+        if self::KLASS.to_s != subclass.to_s
+          raise AlreadyExtendedError,
+            "#{name} has already been extended by '#{self::KLASS}'. Subclasses can not be extended by inheritance multiple times. #{subclass}"
+        end
+      else
+        const_set(:KLASS, subclass)
+      end
+    end
+  end
+end

data/lib/ecfr/facet_attribute_method_definition.rb

@@ -0,0 +1,47 @@
+module Ecfr
+  module FacetAttributeMethodDefinition
+    module ClassMethods
+      def attribute(*attribute)
+        as = attribute.last.delete(:as)
+
+        case as
+        when :key
+          self.fk = attribute.first.to_s
+        when :value
+          self.fv = attribute.first.to_s
+        else
+          raise "must set :as to either :key or :value, was: #{as}"
+        end
+
+        super(*attribute)
+      end
+    end
+
+    def self.included(base)
+      base.instance_eval do
+        class_attribute :fk
+        class_attribute :fv
+      end
+
+      base.extend(ClassMethods)
+    end
+
+    def initialize(results, options = {})
+      default_options = {base: true}
+      options = default_options.merge(options)
+
+      if !options[:base]
+        key, value = results
+        results = {}
+        results[self.class.fk] = key
+        results[self.class.fv] = value
+      end
+
+      super(results, options)
+    end
+
+    def count_for(key)
+      attributes[self.class.result_root][key]
+    end
+  end
+end

data/lib/ecfr/faraday/user_agent/middleware.rb

@@ -0,0 +1,14 @@
+module Faraday
+  module UserAgent
+    class Middleware < Faraday::Middleware
+      def initialize(app, user_agent)
+        super(app)
+        @user_agent = user_agent
+      end
+
+      def on_request(env)
+        env[:request_headers]["User-Agent"] = @user_agent
+      end
+    end
+  end
+end

data/lib/ecfr/ofr_profile_service/base.rb

@@ -0,0 +1,20 @@
+module Ecfr
+  module OfrProfileService
+    class Base < Ecfr::Base
+      require_relative "status"
+
+      # path is relative because we expect the ofr profile
+      # api to be mounted at a subpath and an absolute url
+      # here will cause the path in the config to be lost
+      SERVICE_PATH = "api/profile"
+
+      def self.base_url
+        Ecfr.config.ofr_profile_service_url || Ecfr.config.ofr_profile_service_base_url
+      end
+
+      def self.service_name
+        "OFR Profile"
+      end
+    end
+  end
+end