burly 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5012d619b0f133bb3b826065339370fb825ad444bf832987bdb01485daa5c232
+  data.tar.gz: 74921dc5b75e56b3e38aec36b175176a790cbb81c5a8fc67e73f40f6f61ba505
+SHA512:
+  metadata.gz: 1d5c5f24ab94231093ebc53a9e2c47d595d1ff3a1a4810b8dc300467ae6475263e0f05a14f6f9051216d97dd77009018bb379d6519414603e254192bb09947f9
+  data.tar.gz: b0ad732e42aa2d84693eb727c4cca9e61fd85be214323fed5a79cc12663c953d5145a934fa3920fecd8b708a626e00322d1c60e84237fa60e335510a291039f9

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Jason Garber
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# Burly
+
+**A Ruby gem for extracting URLs from HTML, JSON, and plaintext documents.**
+
+[![Gem](https://img.shields.io/gem/v/burly.svg?logo=rubygems&style=for-the-badge)](https://rubygems.org/gems/burly)
+[![Downloads](https://img.shields.io/gem/dt/burly.svg?logo=rubygems&style=for-the-badge)](https://rubygems.org/gems/burly)
+[![Source](https://img.shields.io/badge/get_it_on_codeberg-2185d0?labelColor=555&logo=codeberg&logoColor=fff&style=for-the-badge)](https://codeberg.org/jgarber/burly)
+
+## Getting Started
+
+Before installing and using Burly, you'll want to have [Ruby](https://www.ruby-lang.org) 2.6 (or newer) installed. Using a Ruby version managment tool like [rbenv](https://github.com/rbenv/rbenv), [chruby](https://github.com/postmodern/chruby), or [rvm](https://github.com/rvm/rvm) is recommended.
+
+Burly is developed using Ruby 3.4 and is tested against additional Ruby versions using [Forgejo Actions](https://codeberg.org/jgarber/burly/actions).
+
+## Installation
+
+Add Burly to your project's `Gemfile` and run `bundle install`:
+
+```ruby
+source "https://rubygems.org"
+
+gem "burly"
+```
+
+## Usage
+
+Using Burly to parse plaintext documents is as straightforward as:
+
+```ruby
+Burly.parse(File.read("example.txt"))
+```
+
+Parsing JSON or HTML documents is only slightly more complicated:
+
+```ruby
+Burly.parse(File.read("example.json"), mime_type: "application/json")
+
+Burly.parse(File.read("example.html", mime_type: "text/html"))
+```
+
+Burly uses _slightly_ different parsing rules for each supported MIME type:
+
+- In plaintext documents, Burly extracts absolute URLs (e.g. `https://website.example`) from the document.
+- In JSON documents, Burly extracts string values that _only_ contain absolute URLs (e.g. `{ "url": "https://website.example" }` and `{ "urls": ["https://website.example", "https://another-website.example] }`)
+- In HTML documents, Burly extracts absolute and relative URLs from [URL attributes](https://html.spec.whatwg.org/#attributes-3) and [srcset attributes](https://html.spec.whatwg.org/#srcset-attributes).
+
+In all cases, neither order nor uniqueness is guaranteed. You may also consider converting relative URLs extract from HTML documents to absolute URLs using the document's source URL and/or the `<base>` element's `href` attribute value (Ruby's [`URI.join` class method](https://docs.ruby-lang.org/en/master/URI.html#method-c-join) is good for this!).
+
+## License
+
+Burly is freely available under the [MIT License](https://opensource.org/license/MIT).

data/burly.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.required_ruby_version = ">= 2.6"
+
+  spec.name = "burly"
+  spec.version = "0.1.0"
+  spec.authors = ["Jason Garber"]
+  spec.email = ["jason@sixtwothree.org"]
+
+  spec.summary = "Extract URLs from HTML, JSON, and plaintext documents."
+  spec.description = spec.summary
+  spec.homepage = "https://codeberg.org/jgarber/burly"
+  spec.license = "MIT"
+
+  spec.files = Dir["lib/**/*"].reject { |f| File.directory?(f) }
+  spec.files += ["LICENSE", "README.md"]
+  spec.files += ["burly.gemspec"]
+
+  spec.require_paths = ["lib"]
+
+  spec.metadata = {
+    "bug_tracker_uri" => "#{spec.homepage}/issues",
+    "changelog_uri" => "#{spec.homepage}/releases/tag/v#{spec.version}",
+    "documentation_uri" => "https://rubydoc.info/gems/#{spec.name}/#{spec.version}",
+    "homepage_uri" => spec.homepage,
+    "rubygems_mfa_required" => "true",
+    "source_code_uri" => "#{spec.homepage}/tree/v#{spec.version}",
+  }
+
+  spec.add_dependency "nokogiri", ">= 1.13"
+end

data/lib/burly/exceptions.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Burly
+  class UnsupportedMimeType < StandardError; end
+end

data/lib/burly/parser.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Burly
+  # @api private
+  class Parser
+    URI_PARSER = defined?(URI::RFC2396_PARSER) ? URI::RFC2396_PARSER : URI::Generic::DEFAULT_PARSER
+
+    URI_REGEXP = URI_PARSER.make_regexp(["http", "https"])
+
+    class << self
+      # @return [Array<String>]
+      attr_reader :mime_types
+    end
+
+    # @param document [String]
+    def initialize(document)
+      @document = document
+    end
+
+    # @raise [NotImplementedError]
+    def parse
+      raise NotImplementedError
+    end
+  end
+end

data/lib/burly/parsers/html_parser.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Burly
+  module Parsers
+    # @api private
+    class HtmlParser < Parser
+      @mime_types = ["text/html"]
+
+      Burly.register_parser(self)
+
+      # A map of HTML +srcset+ attributes and their associated element names.
+      #
+      # @see https://html.spec.whatwg.org/#srcset-attributes
+      # @see https://html.spec.whatwg.org/#attributes-3
+      SRCSET_ATTRIBUTES_MAP = {
+        "imagesrcset" => ["link"],
+        "srcset"      => ["img", "source"],
+      }.freeze
+
+      # A map of HTML URL attributes and their associated element names.
+      #
+      # @see https://html.spec.whatwg.org/#attributes-3
+      URL_ATTRIBUTES_MAP = {
+        "action"     => ["form"],
+        "cite"       => ["blockquote", "del", "ins", "q"],
+        "data"       => ["object"],
+        "formaction" => ["button", "input"],
+        "href"       => ["a", "area", "base", "link"],
+        "ping"       => ["a", "area"],
+        "poster"     => ["video"],
+        "src"        => ["audio", "embed", "iframe", "img", "input", "script", "source", "track", "video"],
+      }.freeze
+
+      ATTRIBUTES_XPATHS =
+        URL_ATTRIBUTES_MAP.merge(SRCSET_ATTRIBUTES_MAP).flat_map do |attribute, names|
+          names.map { |name| "//#{name} / @#{attribute}" }
+        end
+
+      # Parse an HTML document for absolute or relative URLs.
+      #
+      # @return [Array<String>]
+      def parse
+        attr_nodes.flat_map do |attr_node|
+          if SRCSET_ATTRIBUTES_MAP.key?(attr_node.name)
+            urls_from_candidate_strings(attr_node.value.split(/\s*,\s*/))
+          else
+            attr_node.value.strip
+          end
+        end
+      end
+
+      private
+
+      # @return [Nokogiri::XML::NodeSet]
+      def attr_nodes
+        @attr_nodes ||= doc.xpath(*ATTRIBUTES_XPATHS)
+      end
+
+      # @return [Nokogiri::HTML5::Document]
+      def doc
+        @doc ||= Nokogiri.HTML5(@document)
+      end
+
+      def urls_from_candidate_strings(candidate_strings)
+        candidate_strings.flat_map { |candidate_string| candidate_string.strip.split(/\s|$/)[0] }
+      end
+    end
+  end
+end

data/lib/burly/parsers/json_parser.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Burly
+  module Parsers
+    # @api private
+    class JsonParser < Parser
+      @mime_types = ["application/json"]
+
+      Burly.register_parser(self)
+
+      # Parse a JSON document for string values whose contents are absolute
+      # URLs.
+      #
+      # @return [Array<String>]
+      def parse
+        recursive_parse(JSON.parse(@document))
+      end
+
+      private
+
+      def recursive_parse(*objs)
+        objs.flat_map do |obj|
+          return obj.flat_map { |value| recursive_parse(value) }.compact if obj.is_a?(Array)
+          return recursive_parse(obj.values) if obj.is_a?(Hash)
+
+          obj if obj.is_a?(String) && obj.match?(URI_REGEXP)
+        end
+      end
+    end
+  end
+end

data/lib/burly/parsers/plaintext_parser.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Burly
+  module Parsers
+    # @api private
+    class PlaintextParser < Parser
+      @mime_types = ["text/plain"]
+
+      Burly.register_parser(self)
+
+      # Parse a plaintext document for absolute URLs.
+      #
+      # @return [Array<String>]
+      def parse
+        URI_PARSER.extract(@document, ["http", "https"])
+      end
+    end
+  end
+end

data/lib/burly.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+require "nokogiri"
+
+require_relative "burly/exceptions"
+require_relative "burly/parser"
+
+module Burly
+  @registered_parsers = {}
+
+  class << self
+    # @api private
+    attr_reader :registered_parsers
+  end
+
+  # @param document [String]
+  #
+  # @return [Array<String>]
+  def self.parse(document, mime_type: "text/plain")
+    parser = registered_parsers[mime_type]
+
+    raise UnsupportedMimeType unless parser
+
+    parser.new(document).parse
+  end
+
+  # @api private
+  def self.register_parser(klass)
+    klass.mime_types.each { |mime_type| @registered_parsers[mime_type] = klass }
+  end
+end
+
+require_relative "burly/parsers/html_parser"
+require_relative "burly/parsers/json_parser"
+require_relative "burly/parsers/plaintext_parser"

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: burly
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jason Garber
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.13'
+description: Extract URLs from HTML, JSON, and plaintext documents.
+email:
+- jason@sixtwothree.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- burly.gemspec
+- lib/burly.rb
+- lib/burly/exceptions.rb
+- lib/burly/parser.rb
+- lib/burly/parsers/html_parser.rb
+- lib/burly/parsers/json_parser.rb
+- lib/burly/parsers/plaintext_parser.rb
+homepage: https://codeberg.org/jgarber/burly
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://codeberg.org/jgarber/burly/issues
+  changelog_uri: https://codeberg.org/jgarber/burly/releases/tag/v0.1.0
+  documentation_uri: https://rubydoc.info/gems/burly/0.1.0
+  homepage_uri: https://codeberg.org/jgarber/burly
+  rubygems_mfa_required: 'true'
+  source_code_uri: https://codeberg.org/jgarber/burly/tree/v0.1.0
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.8
+specification_version: 4
+summary: Extract URLs from HTML, JSON, and plaintext documents.
+test_files: []