burly 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ee29f60f553f2beda4f2904c9451620e3f51629b3ca3a0f32889ac07e9623be
-  data.tar.gz: 1711b71044722dfc61876645ba51c6e66fe65817e59927ce02a3d26e9f1a5ecc
+  metadata.gz: 0f2cd8eabc523f6e281042e03a815b8e71738e8befa93ec0e33805803cc3f90a
+  data.tar.gz: 68a5db717b7d48c863aaaeccf89885ecf87fb7ec34d43682d3926e4015f45a02
 SHA512:
-  metadata.gz: f4d6527105e193e904fcfb027b0399971a968db52b7ef974f273d1056b8f06a2a343eb6cb0ed10ce3cf577c124f1fdb35975c1e6b9f607c86286470052606800
-  data.tar.gz: a6acfe754967fcd61635c7fc8d7d10950d5d4a951d092ca556156fe3d3e21e8477cf2431f3f76d7bbf1226c3c50b2217a96a67a3a22e8b01ec15fc9b15221278
+  metadata.gz: 32401d0e85a4c205ef96bf54e21cda6f1c3ce09353bd8c18d983b618e0dfef33869cf97e2386b934ebf730188e4bb088033d0f3abf9ab79645198652c8acdc57
+  data.tar.gz: e07cd49d7f9439af7334def57a8b49c5c0023ef10f742c3879e25de8c06f96f251ae5889cab06ac0a58db1cf8e27c5ffe6c81335e9b2cd2b5ea30a1e49b5d3d5

data/README.md

@@ -8,9 +8,9 @@ A Ruby gem for extracting URLs from HTML, JSON, and plaintext documents.
 
 ## Getting Started
 
-Before installing and using Burly, you'll want to have [Ruby](https://www.ruby-lang.org) 2.6 (or newer) installed. Using a Ruby version managment tool like [rbenv](https://github.com/rbenv/rbenv), [chruby](https://github.com/postmodern/chruby), or [rvm](https://github.com/rvm/rvm) is recommended.
+Before installing and using Burly, you'll want to have [Ruby](https://www.ruby-lang.org) 2.6 (or newer) installed. Using a Ruby version managment tool like [rv](https://github.com/spinel-coop/rv) or [rbenv](https://github.com/rbenv/rbenv) is recommended.
 
-Burly is developed using Ruby 3.4 and is tested against additional Ruby versions using [Forgejo Actions](https://codeberg.org/jgarber/burly/actions).
+Burly is developed using Ruby 4.0 and is tested against additional Ruby versions using [Forgejo Actions](https://codeberg.org/jgarber/burly/actions).
 
 ## Installation
 
@@ -72,6 +72,28 @@ In all cases, Burly will search for nodes matching the provided selector(s) and
 > Burly.parse(response, context: [".h-entry .e-content", ".h-entry", "body"], mime_type: "text/html")
 > ```
 
+## Extending Burly
+
+New parsers may be added to Burly by subclassing `Burly::Parsers::Base` and implementing an instance method named `parse` that returns an Array of Strings.
+
+```ruby
+require "burly"
+require "csv"
+
+class CsvParser < Burly::Parsers::Base
+  @mime_types = ["text/csv"]
+
+  def parse
+    doc = CSV.parse(@document, headers: true)
+
+    doc.flat_map(&:fields).grep(uri_regexp)
+  end
+end
+```
+
+> [!TIP]
+> The private `uri_regexp` method available to `Burly::Parsers::Base` subclasses memoizes the regular expression provided by [`URI::RFC2396_Parser#make_regexp`](https://docs.ruby-lang.org/en/master/URI/RFC2396_Parser.html#method-i-make_regexp).
+
 ## License
 
 This project is freely available under the [MIT License](https://opensource.org/license/MIT).

data/burly.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 2.6"
 
   spec.name = "burly"
-  spec.version = "0.3.1"
+  spec.version = "0.4.0"
   spec.authors = ["Jason Garber"]
   spec.email = ["jason@sixtwothree.org"]
 

data/lib/burly/parsers/base.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Burly
+  module Parsers
+    # @api private
+    class Base
+      class << self
+        # @return [Array<String>]
+        attr_reader :mime_types
+      end
+
+      # @return [Array<Class>]
+      def self.descendants
+        ObjectSpace.each_object(Class).select { |klass| klass < self }
+      end
+
+      # @param document (see Burly.parse)
+      def initialize(document, **_kwargs)
+        @document = document.to_s
+      end
+
+      # @raise [NotImplementedError]
+      def parse
+        raise NotImplementedError
+      end
+
+      private
+
+      def uri_parser
+        @uri_parser ||= URI::RFC2396_Parser.new
+      end
+
+      def uri_regexp
+        @uri_regexp ||= uri_parser.make_regexp(["http", "https"])
+      end
+    end
+  end
+end

data/lib/burly/parsers/html.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Burly
+  module Parsers
+    # @api private
+    class Html < Base
+      @mime_types = ["text/html"]
+
+      # An array of HTML +srcset+ attributes.
+      #
+      # @see https://html.spec.whatwg.org/#srcset-attributes HTML Standard § 4.8.4.2.1 Srcset attributes
+      SRCSET_ATTRIBUTES = ["imagesrcset", "srcset"].freeze
+
+      # A map of HTML URL attributes and their associated element names.
+      #
+      # @see https://html.spec.whatwg.org/#attributes-3 HTML Standard § Attributes
+      #
+      # rubocop:disable-next Layout/HashAlignment
+      URL_ATTRIBUTES_MAP = {
+        "action"      => ["form"],
+        "cite"        => ["blockquote", "del", "ins", "q"],
+        "data"        => ["object"],
+        "formaction"  => ["button", "input"],
+        "href"        => ["a", "area", "base", "link"],
+        "imagesrcset" => ["link"],
+        "ping"        => ["a", "area"],
+        "poster"      => ["video"],
+        "src"         => ["audio", "embed", "iframe", "img", "input", "script", "source", "track", "video"],
+        "srcset"      => ["img", "source"],
+      }.freeze
+
+      # @param document (see Burly.parse)
+      # @param context [String, Array<String>]
+      def initialize(document, context: nil)
+        @context = context
+
+        super
+      end
+
+      # Parse an HTML document for absolute or relative URLs.
+      #
+      # @return [Array<String>]
+      def parse
+        context_node
+          .xpath(*xpaths_from_url_attributes_map)
+          .lazy
+          .flat_map { |node| parse_attribute_node(node) }
+          .to_a
+      end
+
+      private
+
+      # @return [Nokogiri::HTML5::Document, Nokogiri::XML::Element]
+      def context_node
+        @context_node ||= doc.search(*Array(@context)).first || doc
+      end
+
+      # @return [Nokogiri::HTML5::Document]
+      def doc
+        @doc ||= Nokogiri.HTML5(@document)
+      end
+
+      # @return [String, Array<String>]
+      def parse_attribute_node(node)
+        return node.value.strip unless SRCSET_ATTRIBUTES.include?(node.name)
+
+        node.value.split(/\s*,\s*/).lazy.flat_map { |value| value.strip.split(/\s|$/)[0] }
+      end
+
+      # @return [Array<String>]
+      def xpaths_from_url_attributes_map
+        URL_ATTRIBUTES_MAP.each_with_object([]) do |(attribute, names), xpaths|
+          xpaths.concat(names.map { |name| ".//#{name}/@#{attribute}" })
+        end
+      end
+    end
+  end
+end

data/lib/burly/parsers/json.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Burly
+  module Parsers
+    # @api private
+    class Json < Base
+      @mime_types = ["application/json"]
+
+      # Parse a JSON document for string values whose content are absolute URLs.
+      #
+      # @return [Array<String>]
+      def parse
+        recursive_parse(JSON.parse(@document)).to_a.compact
+      end
+
+      private
+
+      def recursive_parse(*objs)
+        objs.lazy.flat_map do |obj|
+          case obj
+          when Array  then recursive_parse(*obj)
+          when Hash   then recursive_parse(obj.values)
+          when String then obj if obj.match?(uri_regexp)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/burly/parsers/{plaintext_parser.rb → plaintext.rb}

@@ -3,16 +3,14 @@
 module Burly
   module Parsers
     # @api private
-    class PlaintextParser < Parser
+    class Plaintext < Base
       @mime_types = ["text/plain"]
 
-      Burly.register_parser(self)
-
       # Parse a plaintext document for absolute URLs.
       #
       # @return [Array<String>]
       def parse
-        URI_PARSER.extract(@document, ["http", "https"])
+        uri_parser.extract(@document, ["http", "https"])
       end
     end
   end

data/lib/burly.rb

@@ -5,16 +5,13 @@ require "uri"
 
 require "nokogiri"
 
-require_relative "burly/exceptions"
-require_relative "burly/parser"
+require_relative "burly/parsers/base"
+require_relative "burly/parsers/html"
+require_relative "burly/parsers/json"
+require_relative "burly/parsers/plaintext"
 
 module Burly
-  @registered_parsers = {}
-
-  class << self
-    # @api private
-    attr_reader :registered_parsers
-  end
+  class UnsupportedMimeType < StandardError; end
 
   # Parse a document for URLs.
   #
@@ -34,19 +31,10 @@ module Burly
   #
   # @return [Array<String>]
   def self.parse(document, mime_type: "text/plain", **options)
-    parser = registered_parsers[mime_type]
+    parser = Parsers::Base.descendants.find { |parser| parser.mime_types.include?(mime_type) }
 
     raise UnsupportedMimeType unless parser
 
     parser.new(document, **options).parse
   end
-
-  # @api private
-  def self.register_parser(klass)
-    klass.mime_types.each { |mime_type| @registered_parsers[mime_type] = klass }
-  end
 end
-
-require_relative "burly/parsers/html_parser"
-require_relative "burly/parsers/json_parser"
-require_relative "burly/parsers/plaintext_parser"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: burly
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Jason Garber
@@ -34,21 +34,20 @@ files:
 - README.md
 - burly.gemspec
 - lib/burly.rb
-- lib/burly/exceptions.rb
-- lib/burly/parser.rb
-- lib/burly/parsers/html_parser.rb
-- lib/burly/parsers/json_parser.rb
-- lib/burly/parsers/plaintext_parser.rb
+- lib/burly/parsers/base.rb
+- lib/burly/parsers/html.rb
+- lib/burly/parsers/json.rb
+- lib/burly/parsers/plaintext.rb
 homepage: https://codeberg.org/jgarber/burly
 licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://codeberg.org/jgarber/burly/issues
-  changelog_uri: https://codeberg.org/jgarber/burly/releases/tag/v0.3.1
-  documentation_uri: https://rubydoc.info/gems/burly/0.3.1
+  changelog_uri: https://codeberg.org/jgarber/burly/releases/tag/v0.4.0
+  documentation_uri: https://rubydoc.info/gems/burly/0.4.0
   homepage_uri: https://codeberg.org/jgarber/burly
   rubygems_mfa_required: 'true'
-  source_code_uri: https://codeberg.org/jgarber/burly/src/tag/v0.3.1
+  source_code_uri: https://codeberg.org/jgarber/burly/src/tag/v0.4.0
 rdoc_options: []
 require_paths:
 - lib

data/lib/burly/exceptions.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-module Burly
-  class UnsupportedMimeType < StandardError; end
-end

data/lib/burly/parser.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Burly
-  # @api private
-  class Parser
-    URI_PARSER = defined?(URI::RFC2396_PARSER) ? URI::RFC2396_PARSER : URI::Generic::DEFAULT_PARSER
-
-    URI_REGEXP = URI_PARSER.make_regexp(["http", "https"])
-
-    class << self
-      # @return [Array<String>]
-      attr_reader :mime_types
-    end
-
-    # @param document (see Burly.parse)
-    def initialize(document, **_kwargs)
-      @document = document.to_s
-    end
-
-    # @raise [NotImplementedError]
-    def parse
-      raise NotImplementedError
-    end
-  end
-end

data/lib/burly/parsers/html_parser.rb

@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module Burly
-  module Parsers
-    # @api private
-    class HtmlParser < Parser
-      @mime_types = ["text/html"]
-
-      Burly.register_parser(self)
-
-      # A map of HTML +srcset+ attributes and their associated element names.
-      #
-      # @see https://html.spec.whatwg.org/#srcset-attributes
-      # @see https://html.spec.whatwg.org/#attributes-3
-      SRCSET_ATTRIBUTES_MAP = {
-        "imagesrcset" => ["link"],
-        "srcset"      => ["img", "source"],
-      }.freeze
-
-      # A map of HTML URL attributes and their associated element names.
-      #
-      # @see https://html.spec.whatwg.org/#attributes-3
-      URL_ATTRIBUTES_MAP = {
-        "action"     => ["form"],
-        "cite"       => ["blockquote", "del", "ins", "q"],
-        "data"       => ["object"],
-        "formaction" => ["button", "input"],
-        "href"       => ["a", "area", "base", "link"],
-        "ping"       => ["a", "area"],
-        "poster"     => ["video"],
-        "src"        => ["audio", "embed", "iframe", "img", "input", "script", "source", "track", "video"],
-      }.freeze
-
-      ATTRIBUTES_XPATHS =
-        URL_ATTRIBUTES_MAP.merge(SRCSET_ATTRIBUTES_MAP).flat_map do |attribute, names|
-          names.map { |name| ".//#{name} / @#{attribute}" }
-        end
-
-      # @param document (see Burly.parse)
-      # @param context [String, Array<String>]
-      def initialize(document, context: nil)
-        @context = context
-
-        super
-      end
-
-      # Parse an HTML document for absolute or relative URLs.
-      #
-      # @return [Array<String>]
-      def parse
-        attr_nodes.flat_map do |attr_node|
-          if SRCSET_ATTRIBUTES_MAP.key?(attr_node.name)
-            urls_from_candidate_strings(attr_node.value.split(/\s*,\s*/))
-          else
-            attr_node.value.strip
-          end
-        end
-      end
-
-      private
-
-      # @return [Nokogiri::XML::NodeSet]
-      def attr_nodes
-        @attr_nodes ||= context_node.xpath(*ATTRIBUTES_XPATHS)
-      end
-
-      # @return [Nokogiri::HTML5::Document, Nokogiri::XML::Element]
-      def context_node
-        @context_node ||= doc.search(*Array(@context)).first || doc
-      end
-
-      # @return [Nokogiri::HTML5::Document]
-      def doc
-        @doc ||= Nokogiri.HTML5(@document)
-      end
-
-      def urls_from_candidate_strings(candidate_strings)
-        candidate_strings.flat_map { |candidate_string| candidate_string.strip.split(/\s|$/)[0] }
-      end
-    end
-  end
-end

data/lib/burly/parsers/json_parser.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Burly
-  module Parsers
-    # @api private
-    class JsonParser < Parser
-      @mime_types = ["application/json"]
-
-      Burly.register_parser(self)
-
-      # Parse a JSON document for string values whose contents are absolute
-      # URLs.
-      #
-      # @return [Array<String>]
-      def parse
-        recursive_parse(JSON.parse(@document))
-      end
-
-      private
-
-      def recursive_parse(*objs)
-        objs.flat_map do |obj|
-          if obj.is_a?(Array)
-            obj.map! { |value| recursive_parse(value) }
-            obj.flatten!
-            obj.compact!
-
-            return obj
-          end
-
-          return recursive_parse(obj.values) if obj.is_a?(Hash)
-
-          obj if obj.is_a?(String) && obj.match?(URI_REGEXP)
-        end
-      end
-    end
-  end
-end