async-feedbag 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e899338c5afa426f2847a62a0e2ec083de5e3dac00eec1ccdad3488f29417864
+  data.tar.gz: f74563f3b9b6bd9188f9ed0325ed3745c0a5d7d77bb36935c8bc9776f573fe65
+SHA512:
+  metadata.gz: e403c42cb90ba8cc4e27648a376b54073642d3ac2cba3227961cedfddb8324c15b938e4f1b245c16d9afbf5e371cf0476e0cf6c826692feaa51e7c052b765d92
+  data.tar.gz: f7c5c5b019d2e110edef47d94127ce7b91e519f4dc0798f0bd5023b9526086c4f96356b9a076e85f85f8fb7d9882522bb6717fc23e34a6e580c87bc66ddaed26

data/COPYING

@@ -0,0 +1,20 @@
+Copyright (C) 2008-2022 David Moreno <damog@damog.net> et al.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,77 @@
+Async-Feedbag
+=======
+
+Async-Feedbag is a fork of Ruby's favorite auto-discovery tool/library, using [async-http](https://github.com/socketry/async-http) for the requests.
+
+### Quick synopsis
+
+```ruby
+>> require "async-feedbag"
+=> true
+>> AsyncFeedbag.find "damog.net/blog"
+=> ["http://damog.net/blog/atom.xml"]
+>> AsyncFeedbag.feed? "perl.org"
+=> false
+>> AsyncFeedbag.feed?("https://m.signalvnoise.com/feed")
+=> true
+```
+
+### Installation
+
+    $ gem install async-feedbag
+
+Or just grab async-feedbag.rb and use it on your own project:
+
+    $ wget https://raw.githubusercontent.com/renatolond/feedbag/main/lib/async-feedbag.rb
+
+You can also use the command line tool for quick queries, if you install the gem:
+
+    » feedbag https://www.ruby-lang.org/en/
+    == https://www.ruby-lang.org/en/:
+    - https://www.ruby-lang.org/en/feeds/news.rss
+
+### Usage
+Feedbag will find all RSS feed types.  Here's an example of finding ATOM and JSON Feed
+
+```ruby
+> AsyncFeedbag.find('https://daringfireball.net')
+=> ["https://daringfireball.net/feeds/main", "https://daringfireball.net/feeds/json", "https://daringfireball.net/linked/2021/02/17/bookfeed"]
+```
+
+Feedbag defaults to a User-Agent string of **AsyncFeedbag/1.10.2**, however you can override this
+
+```ruby
+0> AsyncFeedbag.find('https://kottke.org', 'User-Agent' => "My Personal Agent/1.0.1")
+=> ["http://feeds.kottke.org/main", "http://feeds.kottke.org/json"]
+````
+
+The other options passed to find, will be passed to OpenURI. For example:
+
+```ruby
+AsyncFeedbag.find("https://kottke.org", 'User-Agent' => "My Personal Agent/1.0.1", open_timeout: 1000)
+```
+
+You can find the other options to OpenURI [here](https://rubyapi.org/o/openuri/openread#method-i-open).
+
+
+### Why should you use it?
+
+- Because it only uses [Nokogiri](http://nokogiri.org/) as dependency.
+- Because it follows modern feed filename conventions (like those ones used by WordPress blogs, or Blogger, etc).
+- Because it's a single file you can embed easily in your application.
+- Because it's faster than anything else.
+
+### Author
+
+[David Moreno](http://damog.net/) <[damog@damog.net](mailto:damog@damog.net)> is the original author of [feedbag](https://github.com/damog/feedbag).
+[Renato "Lond" Cerqueira](https://lond.com.br) is the author of the async fork.
+
+### Donations
+
+![Superfeedr](https://raw.githubusercontent.com/damog/feedbag/master/img/superfeedr_150.png)
+
+[Superfeedr](http://superfeedr.com) has kindly financially [supported](https://github.com/damog/feedbag/issues/9) the development of Feedbag.
+
+### Copyright
+
+This is and will always be free software. See [COPYING](https://raw.githubusercontent.com/renatolond/feedbag/master/COPYING) for more information.

data/bin/feedbag

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rubygems"
+require "feedbag-async"
+
+def usage
+  %(
+    #{$PROGRAM_NAME} <url 1> [<url 2> <url 3> ... <url n>]
+  )
+end
+
+if ARGV.empty?
+  puts usage
+  exit 1
+end
+
+ARGV.each do |url|
+  puts "== #{url}:"
+  feeds = FeedbagAsync.find url
+  if feeds.empty?
+    puts "   no feeds found!"
+  else
+    feeds.each do |f|
+      puts " - #{f}"
+    end
+  end
+end

data/lib/async-feedbag.rb

@@ -0,0 +1,195 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+# See COPYING before using this software.
+
+require "nokogiri"
+require "async/http/internet/instance"
+require "async/http/middleware/location_redirector"
+
+class AsyncInternetWithRedirect < Async::HTTP::Internet
+  protected
+
+    def make_client(endpoint)
+      ::Protocol::HTTP::AcceptEncoding.new(
+        Async::HTTP::Middleware::LocationRedirector.new(Async::HTTP::Client.new(endpoint, **@options))
+      )
+    end
+end
+
+class AsyncFeedbag
+  VERSION = "2.0.0"
+  CONTENT_TYPES = [
+    "application/x.atom+xml",
+    "application/atom+xml",
+    "application/xml",
+    "text/xml",
+    "application/rss+xml",
+    "application/rdf+xml",
+    "application/json",
+    "application/feed+json"
+  ].freeze
+
+  class << self
+    # (see #feed?)
+    def feed?(url)
+      new.feed?(url)
+    end
+
+    # @param url [String]
+    # @param options [Hash]
+    # @return (see #find)
+    def find(url, options = {})
+      new(options: options).find(url, **options)
+    end
+  end
+
+  def initialize(options: nil)
+    @feeds = []
+    @options = options || {}
+    @options["User-Agent"] ||= "AsyncFeedbag/#{VERSION}"
+  end
+
+  FEED_SCHEME_RE = %r{^feed://}
+  def feed?(url)
+    # use LWR::Simple.normalize some time
+    url_uri = URI.parse(url)
+    url = "#{url_uri.scheme or "http"}://#{url_uri.host}#{url_uri.path}"
+    url << "?#{url_uri.query}" if url_uri.query
+
+    # hack:
+    url.sub!(FEED_SCHEME_RE, "http://")
+
+    res = AsyncFeedbag.find(url)
+    (res.size == 1) && (res.first == url)
+  end
+
+  RedirectionError = Class.new(StandardError)
+
+  XML_RE = /.xml$/
+  SERVICE_FEED_XPATH = "//link[@rel='alternate' or @rel='service.feed'][@href][@type]"
+  JSON_FEED_XPATH = "//link[@rel='alternate' and @type='application/json'][@href]"
+  def find(url, _options = {})
+    url_uri = URI.parse(url)
+    url = nil
+    if url_uri.scheme.nil?
+      url = "http://#{url_uri}"
+    elsif url_uri.scheme == "feed"
+      return add_feed(url_uri.to_s.sub(FEED_SCHEME_RE, "http://"), nil)
+    else
+      url = url_uri.to_s
+    end
+
+    # check if feed_valid is avail
+    begin
+      require "feed_validator"
+      v = W3C::FeedValidator.new
+      v.validate_url(url)
+      return add_feed(url, nil) if v.valid?
+    rescue LoadError
+      # scoo
+    rescue REXML::ParseException
+      # usually indicates timeout
+      # TODO: actually find out timeout. use Terminator?
+      # $stderr.puts "Feed looked like feed but might not have passed validation or timed out"
+    rescue => e
+      warn "#{e.class} error occurred with: `#{url}': #{e.message}"
+    end
+
+    retries = 2
+    begin
+      headers = @options.slice("User-Agent")
+      Sync do
+        response = AsyncInternetWithRedirect.get(url, headers)
+        if response.redirection?
+          original_uri = URI.parse(url)
+          uri = URI.parse(response.headers["location"])
+          if uri.host == original_uri.host
+            url = response.headers["location"]
+            raise RedirectionError
+          end
+        end
+
+        content_type = response.headers["content-type"].gsub(/;.*$/, "").downcase
+        next add_feed(url, nil) if CONTENT_TYPES.include?(content_type)
+
+        doc = Nokogiri::HTML(response.read)
+
+        @base_uri = (doc.at("base")["href"] if doc.at("base") && doc.at("base")["href"])
+
+        # first with links
+        (doc / "atom:link").each do |l|
+          next unless l["rel"] && l["href"].present?
+
+          add_feed(l["href"], url, @base_uri) if l["type"] && CONTENT_TYPES.include?(l["type"].downcase.strip) && (l["rel"].downcase == "self")
+        end
+
+        doc.xpath(SERVICE_FEED_XPATH).each do |l|
+          add_feed(l["href"], url, @base_uri) if CONTENT_TYPES.include?(l["type"].downcase.strip)
+        end
+
+        doc.xpath(JSON_FEED_XPATH).each do |e|
+          add_feed(e["href"], url, @base_uri) if looks_like_feed?(e["href"])
+        end
+
+        (doc / "a").each do |a|
+          next unless a["href"]
+
+          add_feed(a["href"], url, @base_uri) if looks_like_feed?(a["href"]) && (a["href"].include?("/") || a["href"] =~ /#{url_uri.host}/)
+
+          next unless a["href"]
+
+          add_feed(a["href"], url, @base_uri) if looks_like_feed?(a["href"])
+        end
+
+        # Added support for feeds like http://tabtimes.com/tbfeed/mashable/full.xml
+        add_feed(url, nil) if url.match(XML_RE) && doc.root && doc.root["xml:base"] && (doc.root["xml:base"].strip == url.strip)
+      ensure
+        response&.close
+      end
+    rescue RedirectionError
+      retries -= 1
+      retry if retries >= 0
+    rescue Timeout::Error => e
+      warn "Timeout error occurred with `#{url}: #{e}'"
+    rescue => e
+      warn "#{e.class} error occurred with: `#{url}': #{e.message}"
+    end
+    return @feeds
+  end
+
+  FEED_RE = %r{(\.(rdf|xml|rss)(\?([\w'\-%]?(=[\w'\-%.]*)?(&|#|\+|;)?)+)?(:[\w'\-%]+)?$|feed=(rss|atom)|(atom|feed)/?$)}i
+  def looks_like_feed?(url)
+    FEED_RE.match?(url)
+  end
+
+  def add_feed(feed_url, orig_url, base_uri = nil)
+    url = feed_url.sub(/^feed:/, "").strip
+
+    if base_uri
+      url = URI.parse(base_uri).merge(feed_url).to_s
+    end
+
+    begin
+      uri = URI.parse(url)
+    rescue
+      puts "Error with `#{url}'"
+      exit 1
+    end
+    unless uri.absolute?
+      orig = URI.parse(orig_url)
+      url = orig.merge(url).to_s
+    end
+
+    # verify url is really valid
+    @feeds.push(url) unless @feeds.include?(url)
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  if ARGV.empty?
+    puts "usage: feedbag url"
+  else
+    puts AsyncFeedbag.find ARGV.first
+  end
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: async-feedbag
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+platform: ruby
+authors:
+- Renato "Lond" Cerqueira
+- David Moreno
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.89'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.89'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.18
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.18
+description: Ruby's favorite feed auto-discovery tool
+email: renato@lond.com.br
+executables:
+- feedbag
+extensions: []
+extra_rdoc_files:
+- COPYING
+- README.markdown
+files:
+- COPYING
+- README.markdown
+- bin/feedbag
+- lib/async-feedbag.rb
+homepage: http://github.com/renatolond/async-feedbag
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+rdoc_options:
+- "--main"
+- README.markdown
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: RSS/Atom feed auto-discovery tool
+test_files: []