berkeley_library-util 0.1.4 → 0.1.6

data/lib/berkeley_library/util/uris/requester.rb

@@ -16,39 +16,66 @@ module BerkeleyLibrary
           # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
           # @param headers [Hash] the request headers.
           # @return [String] the body as a string.
-          # @raise [RestClient::Exception] in the event of an error.
+          # @raise [RestClient::Exception] in the event of an unsuccessful request.
           def get(uri, params: {}, headers: {})
-            resp = make_get_request(uri, params, headers)
+            resp = make_request(:get, uri, params, headers)
             resp.body
           end
 
-          # Performs a GET request and returns the response.
+          # Performs a HEAD request and returns the response status as an integer.
+          # Note that unlike {Requester#get}, this does not raise an error in the
+          # event of an unsuccessful request.
+          #
+          # @param uri [URI, String] the URI to HEAD
+          # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+          # @param headers [Hash] the request headers.
+          # @return [Integer] the response code as an integer.
+          def head(uri, params: {}, headers: {})
+            head_response(uri, params: params, headers: headers).code
+          end
+
+          # Performs a GET request and returns the response, even in the event of
+          # a failed request.
           #
           # @param uri [URI, String] the URI to GET
           # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
           # @param headers [Hash] the request headers.
           # @return [RestClient::Response] the body as a string.
           def get_response(uri, params: {}, headers: {})
-            make_get_request(uri, params, headers)
+            make_request(:get, uri, params, headers)
+          rescue RestClient::Exception => e
+            e.response
+          end
+
+          # Performs a HEAD request and returns the response, even in the event of
+          # a failed request.
+          #
+          # @param uri [URI, String] the URI to HEAD
+          # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+          # @param headers [Hash] the request headers.
+          # @return [RestClient::Response] the body as a string.
+          def head_response(uri, params: {}, headers: {})
+            make_request(:head, uri, params, headers)
           rescue RestClient::Exception => e
             e.response
           end
 
           private
 
-          def make_get_request(uri, params, headers)
+          # @return [RestClient::Response]
+          def make_request(method, uri, params, headers)
             url_str = url_str_with_params(uri, params)
-            get_or_raise(url_str, headers)
+            req_resp_or_raise(method, url_str, headers)
           end
 
-          def url_str_with_params(uri, params)
-            raise ArgumentError, 'uri cannot be nil' unless (url_str = Validator.url_str_or_nil(uri))
+          def url_str_with_params(url, params)
+            raise ArgumentError, 'url cannot be nil' unless (uri = Validator.uri_or_nil(url))
 
             elements = [].tap do |ee|
-              ee << url_str
+              ee << uri
               next if params.empty?
 
-              ee << '?' unless url_str.include?('?')
+              ee << (uri.query ? '&' : '?')
               ee << URI.encode_www_form(params)
             end
 
@@ -57,15 +84,15 @@ module BerkeleyLibrary
           end
 
           # @return [RestClient::Response]
-          def get_or_raise(url_str, headers)
-            resp = RestClient.get(url_str, headers)
+          def req_resp_or_raise(method, url_str, headers)
+            resp = RestClient::Request.execute(method: method, url: url_str, headers: headers)
             begin
               return resp if (status = resp.code) == 200
 
               raise(exception_for(resp, status))
             ensure
               # noinspection RubyMismatchedReturnType
-              logger.info("GET #{url_str} returned #{status}")
+              logger.info("#{method.to_s.upcase} #{url_str} returned #{status}")
             end
           end
 

data/lib/berkeley_library/util/uris.rb

@@ -8,6 +8,8 @@ module BerkeleyLibrary
     module URIs
       include BerkeleyLibrary::Logging
 
+      UTF_8 = Encoding::UTF_8
+
       class << self
         include URIs
       end
@@ -24,18 +26,31 @@ module BerkeleyLibrary
         Appender.new(uri, *elements).to_uri
       end
 
-      # Performs a GET request.
+      # Performs a GET request and returns the response body as a string.
       #
       # @param uri [URI, String] the URI to GET
       # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
       # @param headers [Hash] the request headers.
       # @return [String] the body as a string.
-      # @raise [RestClient::Exception] in the event of an error.
+      # @raise [RestClient::Exception] in the event of an unsuccessful request.
       def get(uri, params: {}, headers: {})
         Requester.get(uri, params: params, headers: headers)
       end
 
-      # Performs a GET request and returns the response.
+      # Performs a HEAD request and returns the response status as an integer.
+      # Note that unlike {Requester#get}, this does not raise an error in the
+      # event of an unsuccessful request.
+      #
+      # @param uri [URI, String] the URI to HEAD
+      # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+      # @param headers [Hash] the request headers.
+      # @return [Integer] the response code as an integer.
+      def head(uri, params: {}, headers: {})
+        Requester.head(uri, params: params, headers: headers)
+      end
+
+      # Performs a GET request and returns the response, even in the event of
+      # a failed request.
       #
       # @param uri [URI, String] the URI to GET
       # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
@@ -45,6 +60,17 @@ module BerkeleyLibrary
         Requester.get_response(uri, params: params, headers: headers)
       end
 
+      # Performs a HEAD request and returns the response, even in the event of
+      # a failed request.
+      #
+      # @param uri [URI, String] the URI to HEAD
+      # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+      # @param headers [Hash] the request headers.
+      # @return [RestClient::Response] the body as a string.
+      def head_response(uri, params: {}, headers: {})
+        Requester.head_response(uri, params: params, headers: headers)
+      end
+
       # Returns the specified URL as a URI, or `nil` if the URL is `nil`.
       # @param url [String, URI, nil] the URL.
       # @return [URI] the URI, or `nil`.
@@ -54,6 +80,19 @@ module BerkeleyLibrary
         Validator.uri_or_nil(url)
       end
 
+      # Escapes the specified string so that it can be used as a URL path segment,
+      # replacing disallowed characters (including /) with percent-encodings as needed.
+      def path_escape(s)
+        raise ArgumentError, "Can't escape #{s.inspect}: not a string" unless s.respond_to?(:encoding)
+        raise ArgumentError, "Can't escape #{s.inspect}: expected #{UTF_8}, was #{s.encoding}" unless s.encoding == UTF_8
+
+        ''.tap do |escaped|
+          s.bytes.each do |b|
+            escaped << (should_escape?(b, :path_segment) ? '%%%02X' % b : b.chr)
+          end
+        end
+      end
+
       # Returns the specified URL as a URI, or `nil` if the URL cannot
       # be parsed.
       # @param url [Object, nil] the URL.
@@ -65,6 +104,32 @@ module BerkeleyLibrary
         logger.warn("Error parsing URL #{url.inspect}", e)
         nil
       end
+
+      private
+
+      # TODO: extend to cover other modes - host, zone, path, password, query, fragment
+      #       cf. https://github.com/golang/go/blob/master/src/net/url/url.go
+      ALLOWED_BYTES_BY_MODE = {
+        path_segment: [0x24, 0x26, 0x2b, 0x3a, 0x3d, 0x40] # @ & = + $
+      }.freeze
+
+      def should_escape?(b, mode)
+        return false if unreserved?(b)
+        return false if ALLOWED_BYTES_BY_MODE[mode].include?(b)
+
+        true
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      def unreserved?(byte)
+        return true if byte >= 0x41 && byte <= 0x5a # A-Z
+        return true if byte >= 0x61 && byte <= 0x7a # a-z
+        return true if byte >= 0x30 && byte <= 0x39 # 0-9
+        return true if [0x2d, 0x2e, 0x5f, 0x7e].include?(byte) # - . _ ~
+
+        false
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity
     end
   end
 end

data/rakelib/.rubocop.yml

@@ -0,0 +1,4 @@
+inherit_from: ../.rubocop.yml
+
+require:
+  - rubocop-rake

data/spec/.rubocop.yml

@@ -1,5 +1,8 @@
 inherit_from: ../.rubocop.yml
 
+require:
+  - rubocop-rspec
+
 AllCops:
   # Exclude generated files
   Exclude:
@@ -38,3 +41,89 @@ Metrics/MethodLength:
 # Sometimes we're testing the operator
 Lint/BinaryOperatorWithIdenticalOperands:
   Enabled: false
+
+############################################################
+# rubocop-rspec
+
+# believe me, it wasn't by choice
+RSpec/AnyInstance:
+  Enabled: false
+
+# we meant to do that
+RSpec/BeforeAfterAll:
+  Enabled: false
+
+# more words != more readable
+RSpec/ContextWording:
+  Enabled: false
+
+# explicit >>> implicit
+RSpec/DescribedClass:
+  Enabled: false
+
+# more punctuation != more readable
+RSpec/DescribeSymbol:
+  Enabled: false
+
+# setup cost / time >>> failure granularity
+RSpec/ExampleLength:
+  Max: 15
+  CountAsOne:
+    - array
+    - hash
+    - heredoc
+
+# we meant to do that
+RSpec/ExpectInHook:
+  Enabled: false
+
+# your naming scheme is not in possession of all the facts
+RSpec/FilePath:
+  Enabled: false
+
+# explicit >>> implicit
+RSpec/InstanceVariable:
+  Enabled: false
+
+# maybe when 'all' has a corresponding 'none' matcher
+RSpec/IteratedExpectation:
+  Enabled: false
+
+# we meant to do that
+RSpec/MessageSpies:
+  Enabled: false
+
+# too late now
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+# setup cost / time >>> failure granularity
+RSpec/MultipleExpectations:
+  Enabled: false
+
+# cure is worse than the disease
+RSpec/NestedGroups:
+  Enabled: false
+
+# more quotation marks != more readable
+RSpec/SharedExamples:
+  Enabled: false
+
+# we meant to do that
+RSpec/StubbedMock:
+  Enabled: false
+
+# we meant to do that
+RSpec/VerifiedDoubles:
+  Enabled: false
+
+############################################################
+# rubocop-rspec
+
+# enable newer rubocop-rspec cops
+
+RSpec/IdenticalEqualityAssertion: # new in 2.4
+  Enabled: true
+
+RSpec/Rails/AvoidSetupHook: # new in 2.4
+  Enabled: true

data/spec/berkeley_library/util/arrays_spec.rb

@@ -113,12 +113,13 @@ module BerkeleyLibrary::Util
         ]
         sources.each do |source|
           expect(Arrays.find_indices(for_array: source, in_array: target)).to be_nil
+          expect(Arrays.find_indices(for_array: source, in_array: target) { |s, t| t == s.to_s }).to be_nil
         end
       end
 
       it 'takes a comparison block' do
         sub = %i[a c e]
-        expect(Arrays.find_indices(for_array: sub, in_array: target) { |source, target| target == source.to_s }).to eq([0, 2, 4])
+        expect(Arrays.find_indices(for_array: sub, in_array: target) { |s, t| t == s.to_s }).to eq([0, 2, 4])
       end
     end
 
@@ -145,6 +146,10 @@ module BerkeleyLibrary::Util
         expect(Arrays.find_index(in_array: arr, start_index: 2) { |x| x < 4 }).to be_nil
       end
 
+      it 'raises ArgumentError if given extra arguments' do
+        expect { Arrays.find_index(1, 2, 3, in_array: [1, 2, 3]) }.to raise_error(ArgumentError)
+      end
+
       # rubocop:disable Lint/Void
       it 'returns an enumerator if given no arguments' do
         e = Arrays.find_index(in_array: arr)
@@ -327,6 +332,10 @@ module BerkeleyLibrary::Util
         expect(Arrays.invert([0, 2, 3])).to eq([0, nil, 1, 2])
       end
 
+      it 'returns nil if given nil' do
+        expect(Arrays.invert(nil)).to be_nil
+      end
+
       it 'fails if values are not ints' do
         # noinspection RubyYardParamTypeMatch
         expect { Arrays.invert(%i[a b c]) }.to raise_error(TypeError)

data/spec/berkeley_library/util/files_spec.rb

@@ -8,6 +8,7 @@ module BerkeleyLibrary
       attr_reader :tmpdir
 
       before { @tmpdir = Dir.mktmpdir(basename) }
+
       after { FileUtils.remove_entry(tmpdir) }
 
       describe :file_exists? do

data/spec/berkeley_library/util/stringios_spec.rb

@@ -16,6 +16,13 @@ module BerkeleyLibrary
           end
         end
 
+        it 'reads from the end if given a negative index' do
+          bytes.reverse.each_with_index do |b, i|
+            end_offset = i + 1
+            expect(StringIOs.getbyte(sio, -end_offset)).to eq(b)
+          end
+        end
+
         it 'resets the current offset' do
           StringIOs.getbyte(sio, bytes.size / 2)
           expect(sio.pos).to eq(0)

data/spec/berkeley_library/util/strings_spec.rb

@@ -41,6 +41,10 @@ module BerkeleyLibrary
           expect(Strings.diff_index(s, s)).to be_nil
         end
 
+        it 'returns nil for non-strings' do
+          expect(Strings.diff_index(2, ['2'])).to be_nil
+        end
+
         it 'returns the index for different strings' do
           s1 = 'elvis aaron presley'
           s2 = 'elvis nikita presley'

data/spec/berkeley_library/util/times_spec.rb

@@ -33,6 +33,10 @@ module BerkeleyLibrary
           # noinspection RubyYardParamTypeMatch
           expect { Times.ensure_utc(Object.new) }.to raise_error(ArgumentError)
         end
+
+        it 'returns nil for nil' do
+          expect(Times.ensure_utc(nil)).to be_nil
+        end
       end
     end
   end

data/spec/berkeley_library/util/uris/requester_spec.rb

@@ -66,6 +66,85 @@ module BerkeleyLibrary
             expect(result).to eq(expected_body)
           end
         end
+
+        describe :head do
+          it 'returns an HTTP status code for a URL string' do
+            url = 'https://example.org/'
+            expected_status = 203
+            stub_request(:head, url).to_return(status: expected_status)
+
+            result = Requester.head(url)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'returns an HTTP response body for a URI' do
+            uri = URI.parse('https://example.org/')
+            expected_status = 203
+            stub_request(:head, uri).to_return(status: expected_status)
+
+            result = Requester.head(uri)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'appends query parameters' do
+            url = 'https://example.org/'
+            params = { p1: 1, p2: 2 }
+            url_with_query = "#{url}?#{URI.encode_www_form(params)}"
+            expected_status = 203
+            stub_request(:head, url_with_query).to_return(status: expected_status)
+
+            result = Requester.head(url, params: params)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'appends query parameters to URL with existing params' do
+            url = 'https://example.org/endpoint?foo=bar'
+            params = { p1: 1, p2: 2 }
+            url_with_query = "#{url}&#{URI.encode_www_form(params)}"
+            expected_status = 203
+            stub_request(:head, url_with_query).to_return(status: expected_status)
+
+            result = Requester.head(url, params: params)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'sends request headers' do
+            url = 'https://example.org/'
+            headers = { 'X-help' => 'I am trapped in a unit test' }
+            expected_status = 203
+            stub_request(:head, url).with(headers: headers).to_return(status: expected_status)
+
+            result = Requester.head(url, headers: headers)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'returns the status even for unsuccessful requests' do
+            aggregate_failures 'responses' do
+              [207, 400, 401, 403, 404, 405, 418, 451, 500, 503].each do |expected_status|
+                url = "http://example.edu/#{expected_status}"
+                stub_request(:head, url).to_return(status: expected_status)
+
+                result = Requester.head(url)
+                expect(result).to eq(expected_status)
+              end
+            end
+          end
+
+          it 'handles redirects' do
+            url1 = 'https://example.org/'
+            url2 = 'https://example.edu/'
+            stub_request(:head, url1).to_return(status: 302, headers: { 'Location' => url2 })
+            expected_status = 203
+            stub_request(:head, url2).to_return(status: expected_status)
+
+            result = Requester.head(url1)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'rejects a nil URI' do
+            expect { Requester.head(nil) }.to raise_error(ArgumentError)
+          end
+        end
       end
     end
   end