berkeley_library-util 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57ff7fc89570ad61496f1587934e40dc18752df9cd99d766f591c4fe03fcde51
-  data.tar.gz: e0bc0403c067c04801683e12de79a569721332ac23c770a05a0a0fde41842214
+  metadata.gz: 9eac060e7b0956b965455bddb8cc6c9721d5af784ac421766c2d791d99632927
+  data.tar.gz: a68fecb0b685ddffa3cb0fff6a46180e377efeb71b1cc5d23a389117c27be72d
 SHA512:
-  metadata.gz: 6f7786f78fd64266dab9e424f61c03a95dc5a7eb294990bfef1d124699d5e0f6db7b53eaa2d3ca84c77eea1e096b11a5d227c679f5994699973b6274e3d6afe0
-  data.tar.gz: a8a61db572a00d9d0ca6d41f23d2c76490196c14a6fdd8202b88d1f924fd5f9a32ebbea0e7811a4fba2a49a6fcec4ac40242cd79dfbaec8f37deaff046650436
+  metadata.gz: 052c93d1c09d569479d4bec70ffa9413e3ab0404f19482955b89552891c7058ab76ec11191456aa79016e1e2c98403eb355c89982ffe31f31d37f650517970b2
+  data.tar.gz: 530542deae99ffbd089cfc294b614d9560e2298ed4f7a20161970185935155a620b578bd99066a219e7d0410305d525e354c9b811162adbefdf3dda35c904427

data/.idea/util.iml

@@ -8,9 +8,9 @@
     <content url="file://$MODULE_DIR$" />
     <orderEntry type="jdk" jdkName="RVM: ruby-2.7.4" jdkType="RUBY_SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="library" scope="PROVIDED" name="actionpack (v7.0.2.4, RVM: ruby-2.7.4) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="actionview (v7.0.2.4, RVM: ruby-2.7.4) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="activesupport (v7.0.2.4, RVM: ruby-2.7.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="actionpack (v7.0.3, RVM: ruby-2.7.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="actionview (v7.0.3, RVM: ruby-2.7.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="activesupport (v7.0.3, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="addressable (v2.8.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="amazing_print (v1.4.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="ast (v2.4.2, RVM: ruby-2.7.4) [gem]" level="application" />
@@ -35,13 +35,13 @@
     <orderEntry type="library" scope="PROVIDED" name="http-cookie (v1.0.4, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="i18n (v1.10.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="lograge (v0.12.0, RVM: ruby-2.7.4) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="loofah (v2.17.0, RVM: ruby-2.7.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="loofah (v2.18.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="method_source (v1.0.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="mime-types (v3.4.1, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="mime-types-data (v3.2022.0105, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="minitest (v5.15.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="netrc (v0.11.0, RVM: ruby-2.7.4) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.13.5, RVM: ruby-2.7.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.13.6, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="oj (v3.13.11, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="ougai (v1.9.1, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="parallel (v1.22.1, RVM: ruby-2.7.4) [gem]" level="application" />
@@ -52,7 +52,7 @@
     <orderEntry type="library" scope="PROVIDED" name="rack-test (v1.1.0, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rails-dom-testing (v2.0.3, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rails-html-sanitizer (v1.4.2, RVM: ruby-2.7.4) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="railties (v7.0.2.4, RVM: ruby-2.7.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="railties (v7.0.3, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rainbow (v3.1.1, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rake (v13.0.6, RVM: ruby-2.7.4) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="regexp_parser (v2.3.1, RVM: ruby-2.7.4) [gem]" level="application" />
@@ -98,7 +98,7 @@
           </RakeTaskImpl>
           <RakeTaskImpl description="Run all specs in spec directory, with coverage" fullCommand="coverage" id="coverage" />
           <RakeTaskImpl description="Run tests, check test coverage, check code style, check for vulnerabilities, build gem" fullCommand="default" id="default" />
-          <RakeTaskImpl description="Build berkeley_library-util.gemspec as berkeley_library-util-0.1.1.gem" fullCommand="gem" id="gem" />
+          <RakeTaskImpl description="Build berkeley_library-util.gemspec as berkeley_library-util-0.1.4.gem" fullCommand="gem" id="gem" />
           <RakeTaskImpl description="Run RuboCop with auto-correct, and output results to console" fullCommand="ra" id="ra" />
           <RakeTaskImpl description="Run rubocop with HTML output" fullCommand="rubocop" id="rubocop" />
           <RakeTaskImpl id="rubocop">

data/.rubocop.yml

@@ -55,6 +55,10 @@ Style/CommentedKeyword:
 Style/Documentation:
   Enabled: false
 
+# You say "cryptic", I say "compact"
+Style/FormatString:
+  EnforcedStyle: percent
+
 # Adding more line noise to format strings will not improve them
 Style/FormatStringToken:
   Enabled: false

data/CHANGES.md

@@ -1,3 +1,10 @@
+# 0.1.5 (2022-09-16)
+
+- Adds `URIs#path_escape` to escape URL path segments
+- Adds `URIs#head` and `URIs#head_response`, to make a HEAD request and return the HTTP status code 
+  and raw response object, respectively
+- Clarifies documentation for `URIs#get` and `URIs#get_response` regarding unsuccessful requests
+
 # 0.1.4 (2022-07-20)
 
 - Adds `URIs#safe_parse_uri`, which returns `nil` for invalid URLs (unlike `URIs#uri_or_nil`, which

data/lib/berkeley_library/util/module_info.rb

@@ -7,7 +7,7 @@ module BerkeleyLibrary
       SUMMARY = 'Miscellaneous Ruby utilities for the UC Berkeley Library'.freeze
       DESCRIPTION = 'A collection of miscellaneous Ruby routines for the UC Berkeley Library.'.freeze
       LICENSE = 'MIT'.freeze
-      VERSION = '0.1.4'.freeze
+      VERSION = '0.1.5'.freeze
       HOMEPAGE = 'https://github.com/BerkeleyLibrary/util'.freeze
     end
   end

data/lib/berkeley_library/util/uris/requester.rb

@@ -16,29 +16,56 @@ module BerkeleyLibrary
           # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
           # @param headers [Hash] the request headers.
           # @return [String] the body as a string.
-          # @raise [RestClient::Exception] in the event of an error.
+          # @raise [RestClient::Exception] in the event of an unsuccessful request.
           def get(uri, params: {}, headers: {})
-            resp = make_get_request(uri, params, headers)
+            resp = make_request(:get, uri, params, headers)
             resp.body
           end
 
-          # Performs a GET request and returns the response.
+          # Performs a HEAD request and returns the response status as an integer.
+          # Note that unlike {Requester#get}, this does not raise an error in the
+          # event of an unsuccessful request.
+          #
+          # @param uri [URI, String] the URI to HEAD
+          # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+          # @param headers [Hash] the request headers.
+          # @return [Integer] the response code as an integer.
+          def head(uri, params: {}, headers: {})
+            head_response(uri, params: params, headers: headers).code
+          end
+
+          # Performs a GET request and returns the response, even in the event of
+          # a failed request.
           #
           # @param uri [URI, String] the URI to GET
           # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
           # @param headers [Hash] the request headers.
           # @return [RestClient::Response] the body as a string.
           def get_response(uri, params: {}, headers: {})
-            make_get_request(uri, params, headers)
+            make_request(:get, uri, params, headers)
+          rescue RestClient::Exception => e
+            e.response
+          end
+
+          # Performs a HEAD request and returns the response, even in the event of
+          # a failed request.
+          #
+          # @param uri [URI, String] the URI to HEAD
+          # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+          # @param headers [Hash] the request headers.
+          # @return [RestClient::Response] the body as a string.
+          def head_response(uri, params: {}, headers: {})
+            make_request(:head, uri, params, headers)
           rescue RestClient::Exception => e
             e.response
           end
 
           private
 
-          def make_get_request(uri, params, headers)
+          # @return [RestClient::Response]
+          def make_request(method, uri, params, headers)
             url_str = url_str_with_params(uri, params)
-            get_or_raise(url_str, headers)
+            req_resp_or_raise(method, url_str, headers)
           end
 
           def url_str_with_params(uri, params)
@@ -57,15 +84,15 @@ module BerkeleyLibrary
           end
 
           # @return [RestClient::Response]
-          def get_or_raise(url_str, headers)
-            resp = RestClient.get(url_str, headers)
+          def req_resp_or_raise(method, url_str, headers)
+            resp = RestClient::Request.execute(method: method, url: url_str, headers: headers)
             begin
               return resp if (status = resp.code) == 200
 
               raise(exception_for(resp, status))
             ensure
               # noinspection RubyMismatchedReturnType
-              logger.info("GET #{url_str} returned #{status}")
+              logger.info("#{method.to_s.upcase} #{url_str} returned #{status}")
             end
           end
 

data/lib/berkeley_library/util/uris.rb

@@ -8,6 +8,8 @@ module BerkeleyLibrary
     module URIs
       include BerkeleyLibrary::Logging
 
+      UTF_8 = Encoding::UTF_8
+
       class << self
         include URIs
       end
@@ -24,18 +26,31 @@ module BerkeleyLibrary
         Appender.new(uri, *elements).to_uri
       end
 
-      # Performs a GET request.
+      # Performs a GET request and returns the response body as a string.
       #
       # @param uri [URI, String] the URI to GET
       # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
       # @param headers [Hash] the request headers.
       # @return [String] the body as a string.
-      # @raise [RestClient::Exception] in the event of an error.
+      # @raise [RestClient::Exception] in the event of an unsuccessful request.
       def get(uri, params: {}, headers: {})
         Requester.get(uri, params: params, headers: headers)
       end
 
-      # Performs a GET request and returns the response.
+      # Performs a HEAD request and returns the response status as an integer.
+      # Note that unlike {Requester#get}, this does not raise an error in the
+      # event of an unsuccessful request.
+      #
+      # @param uri [URI, String] the URI to HEAD
+      # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+      # @param headers [Hash] the request headers.
+      # @return [Integer] the response code as an integer.
+      def head(uri, params: {}, headers: {})
+        Requester.head(uri, params: params, headers: headers)
+      end
+
+      # Performs a GET request and returns the response, even in the event of
+      # a failed request.
       #
       # @param uri [URI, String] the URI to GET
       # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
@@ -45,6 +60,17 @@ module BerkeleyLibrary
         Requester.get_response(uri, params: params, headers: headers)
       end
 
+      # Performs a HEAD request and returns the response, even in the event of
+      # a failed request.
+      #
+      # @param uri [URI, String] the URI to HEAD
+      # @param params [Hash] the query parameters to add to the URI. (Note that the URI may already include query parameters.)
+      # @param headers [Hash] the request headers.
+      # @return [RestClient::Response] the body as a string.
+      def head_response(uri, params: {}, headers: {})
+        Requester.head_response(uri, params: params, headers: headers)
+      end
+
       # Returns the specified URL as a URI, or `nil` if the URL is `nil`.
       # @param url [String, URI, nil] the URL.
       # @return [URI] the URI, or `nil`.
@@ -54,6 +80,19 @@ module BerkeleyLibrary
         Validator.uri_or_nil(url)
       end
 
+      # Escapes the specified string so that it can be used as a URL path segment,
+      # replacing disallowed characters (including /) with percent-encodings as needed.
+      def path_escape(s)
+        raise ArgumentError, "Can't escape #{s.inspect}: not a string" unless s.respond_to?(:encoding)
+        raise ArgumentError, "Can't escape #{s.inspect}: expected #{UTF_8}, was #{s.encoding}" unless s.encoding == UTF_8
+
+        ''.tap do |escaped|
+          s.bytes.each do |b|
+            escaped << (should_escape?(b, :path_segment) ? '%%%02X' % b : b.chr)
+          end
+        end
+      end
+
       # Returns the specified URL as a URI, or `nil` if the URL cannot
       # be parsed.
       # @param url [Object, nil] the URL.
@@ -65,6 +104,32 @@ module BerkeleyLibrary
         logger.warn("Error parsing URL #{url.inspect}", e)
         nil
       end
+
+      private
+
+      # TODO: extend to cover other modes - host, zone, path, password, query, fragment
+      #       cf. https://github.com/golang/go/blob/master/src/net/url/url.go
+      ALLOWED_BYTES_BY_MODE = {
+        path_segment: [0x24, 0x26, 0x2b, 0x3a, 0x3d, 0x40] # @ & = + $
+      }.freeze
+
+      def should_escape?(b, mode)
+        return false if unreserved?(b)
+        return false if ALLOWED_BYTES_BY_MODE[mode]&.include?(b)
+
+        true
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      def unreserved?(byte)
+        return true if byte >= 0x41 && byte <= 0x5a # A-Z
+        return true if byte >= 0x61 && byte <= 0x7a # a-z
+        return true if byte >= 0x30 && byte <= 0x39 # 0-9
+        return true if [0x2d, 0x2e, 0x5f, 0x7e].include?(byte) # - . _ ~
+
+        false
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity
     end
   end
 end

data/spec/berkeley_library/util/uris/requester_spec.rb

@@ -66,6 +66,70 @@ module BerkeleyLibrary
             expect(result).to eq(expected_body)
           end
         end
+
+        describe :head do
+          it 'returns an HTTP status code for a URL string' do
+            url = 'https://example.org/'
+            expected_status = 203
+            stub_request(:head, url).to_return(status: expected_status)
+
+            result = Requester.head(url)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'returns an HTTP response body for a URI' do
+            uri = URI.parse('https://example.org/')
+            expected_status = 203
+            stub_request(:head, uri).to_return(status: expected_status)
+
+            result = Requester.head(uri)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'appends query parameters' do
+            url = 'https://example.org/'
+            params = { p1: 1, p2: 2 }
+            url_with_query = "#{url}?#{URI.encode_www_form(params)}"
+            expected_status = 203
+            stub_request(:head, url_with_query).to_return(status: expected_status)
+
+            result = Requester.head(url, params: params)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'sends request headers' do
+            url = 'https://example.org/'
+            headers = { 'X-help' => 'I am trapped in a unit test' }
+            expected_status = 203
+            stub_request(:head, url).with(headers: headers).to_return(status: expected_status)
+
+            result = Requester.head(url, headers: headers)
+            expect(result).to eq(expected_status)
+          end
+
+          it 'returns the status even for unsuccessful requests' do
+            aggregate_failures 'responses' do
+              [207, 400, 401, 403, 404, 405, 418, 451, 500, 503].each do |expected_status|
+                url = "http://example.edu/#{expected_status}"
+                stub_request(:head, url).to_return(status: expected_status)
+
+                result = Requester.head(url)
+                expect(result).to eq(expected_status)
+              end
+            end
+          end
+
+          it 'handles redirects' do
+            url1 = 'https://example.org/'
+            url2 = 'https://example.edu/'
+            stub_request(:head, url1).to_return(status: 302, headers: { 'Location' => url2 })
+            expected_status = 203
+            stub_request(:head, url2).to_return(status: expected_status)
+
+            result = Requester.head(url1)
+            expect(result).to eq(expected_status)
+          end
+        end
       end
     end
   end

data/spec/berkeley_library/util/uris_spec.rb

@@ -16,16 +16,14 @@ module BerkeleyLibrary::Util
         expect(new_uri).to eq(expected_uri)
       end
 
-      # TODO: make this work
-      xit "doesn't append to a bare URI when there's nothing to append" do
+      it 'returns a bare URI when there\'s nothing to append' do
         original_url = 'https://example.org'
         new_uri = URIs.append(original_url)
         expected_uri = URI(original_url)
         expect(new_uri).to eq(expected_uri)
       end
 
-      # TODO: make this work
-      xit "doesn't append to a bare URI when there's only a query string" do
+      it 'appends to a bare URI even when there\'s only a query string' do
         original_url = 'https://example.org'
         new_uri = URIs.append(original_url, '?foo=bar')
         expected_uri = URI("#{original_url}?foo=bar")
@@ -136,6 +134,19 @@ module BerkeleyLibrary::Util
         new_uri = URIs.append(original_uri, '?qux=corge', '&grault=plugh#xyzzy')
         expect(new_uri).to eq(URI('https://example.org/foo/bar?qux=corge&grault=plugh#xyzzy'))
       end
+
+      it 'rejects invalid characters' do
+        original_uri = URI('https://example.org/')
+        expect { URIs.append(original_uri, '精力善用') }.to raise_error(URI::InvalidComponentError)
+      end
+
+      it 'accepts percent-encoded path segments' do
+        original_uri = URI('https://example.org/')
+        encoded_segment = URIs.path_escape('精力善用')
+        new_uri = URIs.append(original_uri, encoded_segment, 'foo.html')
+        expected_url = "https://example.org/#{encoded_segment}/foo.html"
+        expect(new_uri).to eq(URI(expected_url))
+      end
     end
 
     describe 'requests' do
@@ -177,6 +188,42 @@ module BerkeleyLibrary::Util
           expect(response.code).to eq(404)
         end
       end
+
+      describe :head do
+        it 'makes a HEAD request' do
+          expected_status = 200
+          stub_request(:head, url_with_query).with(headers: headers).to_return(status: expected_status)
+
+          result = URIs.head(url, params: params, headers: headers)
+          expect(result).to eq(expected_status)
+        end
+
+        it 'returns the status even for unsuccessful requests' do
+          expected_status = 404
+          stub_request(:head, url_with_query).with(headers: headers).to_return(status: expected_status)
+
+          result = URIs.head(url, params: params, headers: headers)
+          expect(result).to eq(expected_status)
+        end
+      end
+
+      describe :head_response do
+        it 'makes a HEAD request' do
+          stub_request(:head, url_with_query).with(headers: headers).to_return(body: expected_body)
+
+          response = URIs.head_response(url, params: params, headers: headers)
+          expect(response.body).to eq(expected_body)
+          expect(response.code).to eq(200)
+        end
+
+        it 'returns the response even for errors' do
+          stub_request(:head, url_with_query).with(headers: headers).to_return(status: 404, body: expected_body)
+
+          response = URIs.head_response(url, params: params, headers: headers)
+          expect(response.body).to eq(expected_body)
+          expect(response.code).to eq(404)
+        end
+      end
     end
 
     describe :safe_parse_uri do
@@ -210,5 +257,49 @@ module BerkeleyLibrary::Util
         end
       end
     end
+
+    describe :path_escape do
+      let(:in_out) do
+        {
+          '' => '',
+          'corge' => 'corge',
+          'foo+bar' => 'foo+bar',
+          'qux/quux' => 'qux%2Fquux',
+          'foo bar baz' => 'foo%20bar%20baz',
+          '25%' => '25%25',
+          "\t !\"#$%&'()*+,/:;<=>?@[\\]^`{|}☺" => '%09%20%21%22%23$%25&%27%28%29%2A+%2C%2F:%3B%3C=%3E%3F@%5B%5C%5D%5E%60%7B%7C%7D%E2%98%BA',
+          '精力善用' => '%E7%B2%BE%E5%8A%9B%E5%96%84%E7%94%A8'
+        }
+      end
+
+      it 'escapes a path segment' do
+        aggregate_failures do
+          in_out.each do |in_str, out_str|
+            expect(URIs.path_escape(in_str)).to eq(out_str)
+          end
+        end
+      end
+
+      it 'rejects non-strings' do
+        str = in_out.keys.last
+        expect { URIs.path_escape(str.bytes) }.to raise_error(ArgumentError)
+      end
+
+      it 'rejects non-UTF-8 strings' do
+        str = in_out.keys.last
+        expect { URIs.path_escape(str.encode(Encoding::Shift_JIS)) }.to raise_error(ArgumentError)
+      end
+
+      it 'accepts non-UTF-8 strings converted to UTF-8' do
+        in_str = in_out.keys.last
+        out_str = in_out[in_str]
+
+        # OK, we're really just testing String#encode here, but
+        # it's useful for documentation
+        in_str_sjis = in_str.encode(Encoding::Shift_JIS)
+        in_str_utf8 = in_str_sjis.encode(Encoding::UTF_8)
+        expect(URIs.path_escape(in_str_utf8)).to eq(out_str)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: berkeley_library-util
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - David Moles
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-20 00:00:00.000000000 Z
+date: 2022-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: berkeley_library-logging