automatic 14.12.2 → 26.09

data/lib/automatic/feed_maker.rb

@@ -1,22 +1,23 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::FeedMaker
-# Author::    774 <http://id774.net>
-# Created::   Feb 21, 2014
-# Updated::   Feb 26, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::FeedMaker
+# Author:       id774 (More info: https://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 21, 2014
+# Updated::     Sep  7, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic
   module FeedMaker
     require 'rss'
     require 'uri'
-    require 'nokogiri'
 
     class FeedObject
       attr_accessor :title, :link, :description, :author, :comments
       def initialize
-        @link        = 'http://dummy'
-        @title       = 'dummy'
+        @link        = nil
+        @title       = nil
         @description = ''
         @author      = ''
         @comments    = ''
@@ -33,30 +34,36 @@ module Automatic
       feed_object
     end
 
+    # Plain-value standard fields, copied onto the rebuilt item as-is when the
+    # item being rebuilt carries them. See doc/REQUIREMENTS.md and
+    # doc/PLUGINS.md for the standard item field contract this preserves.
+    REBUILD_SIMPLE_FIELDS = %i[title link description author comments content_encoded].freeze
+
+    # source and enclosure are RSS child elements. RSS::Maker exposes each of
+    # them on a new item as a builder with its own sub-attributes rather than
+    # accepting the finished element directly, so each is rebuilt attribute by
+    # attribute instead of by one assignment.
+    REBUILD_STRUCTURED_FIELDS = {
+      source:    %i[url content],
+      enclosure: %i[url length type]
+    }.freeze
+
     def self.create_pipeline(feeds = [])
       RSS::Maker.make("2.0") {|maker|
         xss = maker.xml_stylesheets.new_xml_stylesheet
         maker.channel.title = "Automatic Ruby"
         maker.channel.description = "Automatic::FeedMaker"
-        maker.channel.link = "https://github.com/automaticruby/automaticruby"
+        maker.channel.link = "https://github.com/id774/automaticruby"
         maker.items.do_sort = true
 
         unless feeds.nil?
           feeds.each {|feed|
-            unless feed.link.nil?
-              Automatic::Log.puts("info", "Create Pipeline: #{feed.link}")
-              item = maker.items.new_item
-              item.title = feed.title
-              item.link = feed.link
-              begin
-                item.description = feed.description
-                item.author = feed.author
-                item.comments = feed.comments
-                item.date = feed.pubDate || Time.now
-              rescue NoMethodError
-                Automatic::Log.puts("warn", "Undefined field detected in feed.")
-              end
-            end
+            Automatic::Log.puts("info", "Create Pipeline: #{feed.link}")
+            item = maker.items.new_item
+            item.date = (feed.pubDate if feed.respond_to?(:pubDate)) || Time.now
+
+            REBUILD_SIMPLE_FIELDS.each { |field| copy_rebuild_field(feed, item, field) }
+            REBUILD_STRUCTURED_FIELDS.each_key { |field| copy_rebuild_structured_field(feed, item, field) }
           }
         end
       }
@@ -67,7 +74,7 @@ module Automatic
         xss = maker.xml_stylesheets.new_xml_stylesheet
         maker.channel.title = "Automatic Ruby"
         maker.channel.description = "Automatic::FeedMaker"
-        maker.channel.link = "https://github.com/automaticruby/automaticruby"
+        maker.channel.link = "https://github.com/id774/automaticruby"
         maker.items.do_sort = true
         item = maker.items.new_item
         item.title = "Automatic Ruby"
@@ -76,5 +83,38 @@ module Automatic
         item.date = Time.now
       }
     end
+
+    # A field the item being rebuilt does not carry, or carries as nil, is
+    # left off the new item: that is the normal shape of an optional field,
+    # not a reason to skip copying anything else. See doc/POLICY.md on
+    # distinguishing a missing optional value from a programming error.
+    def self.copy_rebuild_field(feed, item, field)
+      return unless feed.respond_to?(field)
+
+      value = feed.public_send(field)
+      return if value.nil?
+
+      item.public_send("#{field}=", value)
+    end
+    private_class_method :copy_rebuild_field
+
+    # source and enclosure are rebuilt sub-attribute by sub-attribute, on the
+    # same missing-is-normal basis as copy_rebuild_field, so that (for
+    # example) an enclosure with no type recorded still keeps its url.
+    def self.copy_rebuild_structured_field(feed, item, field)
+      return unless feed.respond_to?(field)
+
+      value = feed.public_send(field)
+      return if value.nil?
+
+      target = item.public_send(field)
+      REBUILD_STRUCTURED_FIELDS.fetch(field).each do |sub_field|
+        next unless value.respond_to?(sub_field)
+
+        sub_value = value.public_send(sub_field)
+        target.public_send("#{sub_field}=", sub_value) unless sub_value.nil?
+      end
+    end
+    private_class_method :copy_rebuild_structured_field
   end
 end

data/lib/automatic/feed_parser.rb

@@ -1,51 +1,67 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::FeedParser
-# Author::    774 <http://id774.net>
-# Created::   Feb 19, 2012
-# Updated::   Feb 26, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::FeedParser
+# Author:       id774 (More info: https://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 19, 2012
+# Updated::     Aug 14, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# Fetching and parsing, on the way into the pipeline. What leaves here has the
+# shape described in doc/PLUGINS.md section 3.4.
 
 module Automatic
   module FeedParser
     require 'rss'
-    require 'uri'
-    require 'nokogiri'
+    require 'automatic/http'
 
+    # Fetch a URL and parse it as a feed. Validation is off, because feeds in
+    # the wild frequently are not valid and are still readable.
+    #
+    # Fetching goes through Automatic::Http, which is where the scheme
+    # allowlist, the timeouts and the redirect limit live.
     def self.get_url(url)
-      begin
-        unless url.nil?
-          Automatic::Log.puts("info", "Parsing Feed: #{url}")
-          feed = URI.parse(url).normalize
-          open(feed) {|http|
-            response = http.read
-            RSS::Parser.parse(response, false)
-          }
-        end
-      rescue => e
-        raise e
-      end
+      return if url.nil?
+
+      Automatic::Log.puts('info', "Parsing Feed: #{url}")
+      RSS::Parser.parse(Automatic::Http.read(url), false)
     end
 
+    # Build a feed whose items are the links of an HTML document. This is how
+    # a page that publishes no feed enters the pipeline.
+    #
+    # nokogiri is required here rather than at the top of the file: it is the
+    # only thing in the framework that wants an HTML parser, it is an optional
+    # dependency rather than a runtime one, and requiring `automatic` must
+    # neither load one nor need one installed. Only the plugins that call this
+    # method -- SubscriptionLink and SubscriptionTumblr -- do.
+    # See doc/POLICY.md sections 2.5 and 9.1.
     def self.parse_html(html)
-      RSS::Maker.make("2.0") {|maker|
-        xss = maker.xml_stylesheets.new_xml_stylesheet
-        maker.channel.title = "Automatic Ruby"
-        maker.channel.description = "Automatic::FeedParser"
-        maker.channel.link = "https://github.com/automaticruby/automaticruby"
+      Automatic.require_optional(
+        'nokogiri',
+        needed_by: 'Automatic::FeedParser.parse_html, used by SubscriptionLink ' \
+                   'and SubscriptionTumblr'
+      )
+
+      RSS::Maker.make('2.0') do |maker|
+        maker.xml_stylesheets.new_xml_stylesheet
+        maker.channel.title = 'Automatic Ruby'
+        maker.channel.description = 'Automatic::FeedParser'
+        maker.channel.link = 'https://github.com/id774/automaticruby'
         maker.items.do_sort = true
 
         doc = Nokogiri::HTML(html)
-        (doc/:a).each {|link|
-          unless link[:href].nil?
-            item = maker.items.new_item
-            item.title = "Automatic Ruby"
-            item.link = link[:href]
-            item.date = Time.now
-            item.description = ""
-          end
-        }
-      }
+        (doc / :a).each do |link|
+          next if link[:href].nil?
+
+          item = maker.items.new_item
+          item.title = 'Automatic Ruby'
+          item.link = link[:href]
+          item.date = Time.now
+          item.description = ''
+        end
+      end
     end
   end
 end

data/lib/automatic/http.rb

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# Name::        Automatic::Http
+# Author:       id774 (More info: https://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Aug 15, 2026
+# Updated::     Aug 21, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# One way in for everything the plugins fetch over HTTP.
+#
+# This exists because seven plugins were each calling `URI.open` on a string,
+# and the four decisions that call implies -- which schemes are allowed, how
+# long to wait, how many redirects to follow, what to send as a User-Agent --
+# were being made seven times, mostly by omission. They are made once here.
+# It is a helper, not a client framework: one method fetches, one builds a URI,
+# and a plugin that wants something else still calls Ruby directly.
+# See doc/POLICY.md section 9.2 and doc/PLUGINS.md section 3.8.
+
+require 'open-uri'
+require 'uri'
+require 'automatic/version'
+
+module Automatic
+  module Http
+    # A link in a pipeline item comes from a feed, which is to say from
+    # outside. `URI.open` on such a string will happily read `file:///etc/passwd`
+    # or run an FTP session; a plugin fetching an article body wants neither.
+    SCHEMES = %w[http https].freeze
+
+    OPEN_TIMEOUT = 10
+    READ_TIMEOUT = 30
+
+    # open-uri follows redirects itself and refuses an HTTPS-to-HTTP downgrade.
+    # This bounds the chain so that a redirect loop ends as an error rather
+    # than as an unattended run that never returns.
+    REDIRECT_LIMIT = 5
+
+    USER_AGENT = "Automatic Ruby/#{Automatic::VERSION} " \
+                 '(+https://github.com/id774/automaticruby)'
+
+    # The RFC 2396 parser is named directly: `URI.escape` was removed in Ruby
+    # 3.0, and `URI::Parser` became the RFC 3986 parser in 3.4, which reports
+    # #escape as obsolete. This spelling means the same thing on every
+    # supported Ruby.
+    ESCAPER = URI::RFC2396_Parser.new
+
+    class << self
+      # Fetch a URL and return its body as a string. Raises rather than
+      # returning nil: a plugin's retry handling is built around an exception,
+      # and a body that could not be fetched is not an empty body.
+      def read(url)
+        open(url, &:read)
+      end
+
+      # Fetch a URL and yield the IO, for a caller that would rather stream
+      # than hold the whole body.
+      def open(url, &block)
+        uri(url).open(
+          'User-Agent' => USER_AGENT,
+          open_timeout: OPEN_TIMEOUT,
+          read_timeout: READ_TIMEOUT,
+          redirect: true,
+          max_redirects: REDIRECT_LIMIT,
+          &block
+        )
+      end
+
+      # Parse a URL into a URI this framework will fetch, or raise. A string
+      # carrying characters a URI may not (a space, a Japanese query term) is
+      # escaped and parsed again, which is what the plugins used to do for
+      # themselves before every call.
+      def uri(url)
+        string = url.to_s.strip
+        raise ArgumentError, 'no URL to fetch' if string.empty?
+
+        parsed = parse(string)
+        unless SCHEMES.include?(parsed.scheme)
+          raise ArgumentError, "not an HTTP or HTTPS URL: #{string}"
+        end
+        unless parsed.host
+          raise ArgumentError, "HTTP or HTTPS URL has no host: #{string}"
+        end
+
+        parsed.normalize
+      end
+
+      # Whether a string is a URL this framework will fetch. For a plugin that
+      # skips an item rather than failing the run on one.
+      def fetchable?(url)
+        uri(url)
+        true
+      rescue ArgumentError, URI::InvalidURIError
+        false
+      end
+
+      private
+
+      def parse(string)
+        URI.parse(string)
+      rescue URI::InvalidURIError
+        URI.parse(ESCAPER.escape(string))
+      end
+    end
+  end
+end

data/lib/automatic/log.rb

@@ -1,31 +1,62 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Log
-# Author::    774 <http://id774.net>
-# Created::   Feb 20, 2012
-# Updated::   Oct 09, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Log
+# Author:       id774 (More info: https://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 20, 2012
+# Updated::     Aug 14, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# One log, to standard output, behind a level filter set per Recipe by
+# global.log.level. See doc/BASIC_DESIGN.md section 4.7.
 
 require 'logger'
 
 module Automatic
   module Log
-    LOG_LEVELS = ['info', 'warn', 'error', 'none']
+    # In increasing order of severity. 'none' is the threshold that admits
+    # nothing; it is never used as the level of a message.
+    LOG_LEVELS = %w[info warn error none].freeze
 
-    def self.level(level)
-      @level = level
-    end
+    DEFAULT_LEVEL = 'info'
 
-    def self.logger
-      @logger ||= Logger.new(STDOUT)
-    end
+    class << self
+      # Set the threshold. An unrecognised name, including nil, means the
+      # default: an unattended run should not be silenced by a typo in a
+      # Recipe, nor should it raise in the middle of a job.
+      def level(level = nil)
+        @level = normalize(level, DEFAULT_LEVEL)
+      end
 
-    def self.puts(level = :info, message)
-      if LOG_LEVELS.index(@level).to_i > LOG_LEVELS.index(level).to_i
-        return
+      def logger
+        @logger ||= Logger.new($stdout)
       end
-      logger.send(level, message)
-    end
 
+      # Replace the logger. Used by the specs; a plugin has no business
+      # calling this.
+      attr_writer :logger
+
+      # Emit a message at the given level, if the threshold admits it.
+      # Both 'info' and :info are accepted, because plugins pass both.
+      def puts(level, message)
+        name = normalize(level, DEFAULT_LEVEL)
+        return if name == 'none'
+        return if LOG_LEVELS.index(name) < LOG_LEVELS.index(current_level)
+
+        logger.public_send(name, message)
+      end
+
+      private
+
+      def current_level
+        @level ||= DEFAULT_LEVEL
+      end
+
+      def normalize(level, fallback)
+        name = level.to_s
+        LOG_LEVELS.include?(name) ? name : fallback
+      end
+    end
   end
 end

data/lib/automatic/opml.rb

@@ -220,7 +220,9 @@ module Automatic
       end # each_outline
 
       def read_text
-        text = ""
+        # +'' rather than "": this string is appended to below, and a literal
+        # is on its way to being frozen.
+        text = +''
         while event = @p.pull
           case event.event_type
           when :end_element

data/lib/automatic/pipeline.rb

@@ -1,43 +1,84 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Pipeline
-# Author::    774 <http://id774.net>
-# Created::   Feb 22, 2012
-# Updated::   Mar 16, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Pipeline
+# Author:       id774 (More info: https://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 22, 2012
+# Updated::     Sep  6, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# The core: resolve a plugin's class name to a file, then run the Recipe's
+# plugins in order, each receiving the previous one's output. See
+# doc/BASIC_DESIGN.md section 4.6 and doc/PLUGINS.md section 3.
+#
+# Every module the Recipe names is discovered before any plugin runs, so that
+# an unknown module later in the Recipe is refused before an earlier plugin's
+# side effect, not after it. Discovery only registers an autoload; it does not
+# read a plugin's own source, which stays lazy until that plugin's turn to
+# run. See doc/PLUGINS.md on unknown plugin being refused before any plugin
+# runs.
 
-require 'active_support'
-require 'active_support/core_ext'
-require 'active_support/deprecation'
+require 'active_support/core_ext/string/inflections'
 
 module Automatic
-  module Plugin end
-  class NoPluginError < StandardError; end
-  class NoRecipeError < StandardError; end
+  # The namespace every plugin class lives in. Plugin files are registered
+  # here with autoload, so a file is read when its constant is first used.
+  module Plugin; end
 
   module Pipeline
-    def self.load_plugin(module_name)
-      Dir[Automatic.user_plugins_dir + "/*",
-          Automatic.plugins_dir + "/*"].each {|dir|
-        subdir = File.basename dir
-        if /#{subdir}_(.*)$/ =~ module_name.underscore
-          path = dir + "/#{$1}.rb"
-          return Automatic::Plugin.autoload module_name.to_sym,
-            path.to_s if File.exists? path
+    class << self
+      # Resolve a module name such as SubscriptionFeed to
+      # <search root>/subscription/feed.rb and register it for autoloading.
+      #
+      # The user directory is searched before the installation, so a plugin in
+      # ~/.automatic/plugins shadows a shipped plugin of the same name.
+      def load_plugin(module_name)
+        underscored = module_name.underscore
+
+        search_roots.each do |dir|
+          category = File.basename(dir)
+          next unless /\A#{Regexp.escape(category)}_(.*)\z/ =~ underscored
+
+          path = File.join(dir, "#{Regexp.last_match(1)}.rb")
+          next unless File.exist?(path)
+
+          return Automatic::Plugin.autoload(module_name.to_sym, path)
         end
-      }
-      raise NoPluginError, "unknown plugin named #{module_name}"
-    end
 
-    def self.run(recipe)
-      raise NoRecipeError if recipe.nil?
-      pipeline = []
-      recipe.each_plugin {|plugin|
-        mod = plugin.module
-        load_plugin(mod)
-        klass = Automatic::Plugin.const_get(mod)
-        pipeline = klass.new(plugin.config, pipeline).run
-      }
+        raise NoPluginError, "unknown plugin named #{module_name}"
+      end
+
+      # Run a Recipe: each plugin's return value is the next plugin's input.
+      #
+      # Nothing here rescues a plugin's exception. A Recipe is a sequence in
+      # which each step consumes the previous one's output, so continuing past
+      # a failed step would run the rest on a value their author never
+      # intended. See doc/REQUIREMENTS.md section 12.
+      def run(recipe)
+        raise NoRecipeError, 'no recipe given' if recipe.nil?
+
+        entries = []
+        recipe.each_plugin { |plugin| entries << [plugin, plugin.module] }
+
+        entries.each { |_plugin, mod| load_plugin(mod) }
+
+        pipeline = []
+        entries.each do |plugin, mod|
+          klass = Automatic::Plugin.const_get(mod)
+          pipeline = klass.new(plugin.config, pipeline).run
+        end
+        pipeline
+      end
+
+      private
+
+      def search_roots
+        Dir[File.join(Automatic.user_plugins_dir, '*'),
+            File.join(Automatic.plugins_dir, '*')].select { |path|
+          File.directory?(path)
+        }
+      end
     end
   end
 end