automatic 14.12.2 → 26.08

data/plugins/filter/gemini.rb

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+# Name::        Automatic::Plugin::Filter::Gemini
+# Description:: Replace each item's description with what the Google Gemini API answers.
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Aug 17, 2026
+# Updated::     Aug 17, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# One transformation: the item's description goes to the Gemini API under the
+# Recipe's prompt, and the answer becomes the item's description. What that
+# transformation is -- a summary, a translation, an extraction, a
+# classification -- is the prompt's business, not this plugin's.
+#
+# This plugin knows Gemini and nothing else. Its model is named in the URL
+# rather than in the body, its API key is a header of its own, and its request
+# and answer are built of `contents` and `parts`. None of that is bent into
+# another service's shape, and no other service's request is built here.
+#
+# It speaks `generateContent`, which Google states remains fully supported and
+# is the single-turn text interface: a request that carries an instruction and
+# a text and answers with a text, which is exactly what this plugin does.
+# @see https://ai.google.dev/api/generate-content
+
+module Automatic::Plugin
+  class FilterGemini
+    require 'json'
+    require 'net/http'
+    require 'openssl'
+    require 'uri'
+
+    # The model is part of the path here, unlike every other service in this
+    # directory, so the endpoint is built per Recipe rather than being one
+    # constant.
+    ENDPOINT_FORMAT = 'https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent'.freeze
+
+    OPEN_TIMEOUT = 10
+
+    # Generous, and bounded. A model given several articles thinks for a while;
+    # an unattended run that waits forever is the failure this exists against.
+    READ_TIMEOUT = 300
+
+    # A failure that another attempt will not get past: a setting that is
+    # wrong, a request the service refuses, an answer this plugin cannot read.
+    class Error < StandardError; end
+
+    # A failure that another attempt may get past: the network, a rate limit, a
+    # server error.
+    class TemporaryError < StandardError; end
+
+    def initialize(config, pipeline = [])
+      @config   = config || {}
+      @pipeline = pipeline
+    end
+
+    # Replaces each item's description with the answer. Nothing else about an
+    # item is touched, and the feeds and their items arrive and leave in the
+    # same order and number.
+    def run
+      validate_settings
+
+      @pipeline.each { |feeds|
+        next if feeds.nil?
+
+        feeds.items.each { |item| transform(item) }
+      }
+      @pipeline
+    end
+
+    private
+
+    # Checked before the first request, because a Recipe this plugin cannot
+    # carry out is the operator's mistake and will be the same mistake on every
+    # item. The token is never named in a message.
+    def validate_settings
+      raise ArgumentError, 'FilterGemini needs a token' if token.empty?
+      raise ArgumentError, 'FilterGemini needs a model' if model.empty?
+      raise ArgumentError, 'FilterGemini needs a prompt' if prompt.empty?
+
+      endpoint
+    end
+
+    def token
+      @config['token'].to_s
+    end
+
+    def model
+      @config['model'].to_s.strip
+    end
+
+    def prompt
+      @config['prompt'].to_s.strip
+    end
+
+    # A model name that cannot go in a URL is the Recipe's mistake, and is
+    # reported as one rather than as a failed request.
+    def endpoint
+      @endpoint ||= URI(format(ENDPOINT_FORMAT, model))
+    rescue URI::InvalidURIError
+      raise ArgumentError, "FilterGemini cannot build a request for the model #{model}"
+    end
+
+    def transform(item)
+      text = item.description.to_s
+      if text.strip.empty?
+        Automatic::Log.puts('warn', "FilterGemini: nothing to send for #{item.link}")
+        return
+      end
+
+      Automatic::Log.puts('info', "FilterGemini: asking #{model} about #{item.link}")
+      item.description = answer(text)
+    end
+
+    # The retry shape of doc/PLUGINS.md section 3.6, applied only to what
+    # retrying can help. A missing setting, a refused request or an answer in a
+    # shape this plugin cannot read is raised at once: trying again would fail
+    # the same way, more slowly.
+    def answer(text)
+      retries   = 0
+      retry_max = @config['retry'].to_i
+      begin
+        generated(text)
+      rescue TemporaryError => e
+        retries += 1
+        Automatic::Log.puts('error', "ErrorCount: #{retries}, FilterGemini: #{e.message}")
+        if retries <= retry_max
+          sleep(@config['interval'].to_i)
+          retry
+        end
+        raise Error, "FilterGemini gave up after #{retries} attempts: #{e.message}"
+      end
+    end
+
+    def generated(text)
+      # The prompt is the system instruction and the description is the content
+      # it is applied to. They are separate fields, so that what an article
+      # says is never read as an instruction to this plugin or to the model.
+      body = {
+        'system_instruction' => { 'parts' => [{ 'text' => prompt }] },
+        'contents' => [{ 'role' => 'user', 'parts' => [{ 'text' => text }] }]
+      }
+      content(post(JSON.generate(body)))
+    end
+
+    def post(body)
+      request = Net::HTTP::Post.new(endpoint)
+      # The key goes in a header rather than in the query string, which is the
+      # way Google documents and the way that keeps a credential out of a URL.
+      request['x-goog-api-key'] = token
+      request['Content-Type']   = 'application/json'
+      request.body = body
+
+      # TLS with the certificate verified, which is Net::HTTP's own default and
+      # is named here because it is not a thing to be turned off.
+      Net::HTTP.start(endpoint.host, endpoint.port,
+                      use_ssl: true,
+                      verify_mode: OpenSSL::SSL::VERIFY_PEER,
+                      open_timeout: OPEN_TIMEOUT,
+                      read_timeout: READ_TIMEOUT) { |http| http.request(request) }
+    rescue Timeout::Error, SystemCallError, SocketError, IOError,
+           OpenSSL::SSL::SSLError, Net::HTTPBadResponse => e
+      raise TemporaryError, "the request to Gemini failed: #{e.message}"
+    end
+
+    def content(response)
+      case response
+      when Net::HTTPSuccess
+        answer_text(parse(response.body))
+      when Net::HTTPTooManyRequests, Net::HTTPServerError
+        raise TemporaryError, "Gemini answered #{response.code}: #{reason(response)}"
+      else
+        raise Error, "Gemini answered #{response.code}: #{reason(response)}"
+      end
+    end
+
+    def parse(body)
+      JSON.parse(body.to_s)
+    rescue JSON::ParserError => e
+      raise Error, "Gemini answered with something that is not JSON: #{e.message}"
+    end
+
+    # The text of the first candidate. A response with no candidate is what a
+    # request stopped by a safety filter looks like, and it is an error rather
+    # than an empty description: a Recipe that published the empty string here
+    # would have thrown the article away and reported success.
+    def answer_text(body)
+      candidates = body['candidates']
+      unless candidates.is_a?(Array) && candidates.first.is_a?(Hash)
+        raise Error, 'Gemini answered without a candidate'
+      end
+
+      parts = candidates.first.dig('content', 'parts')
+      raise Error, 'Gemini answered without content parts' unless parts.is_a?(Array)
+
+      text = parts.select { |part| part.is_a?(Hash) }.map { |part| part['text'].to_s }.join.strip
+      raise Error, 'Gemini answered with no text' if text.empty?
+
+      text
+    end
+
+    # The service's own explanation where it gave one, the status line
+    # otherwise. Neither carries the token, and the settings are never logged
+    # or raised wholesale.
+    def reason(response)
+      body = JSON.parse(response.body.to_s)
+      error = body['error']
+      return error['message'].to_s if error.is_a?(Hash) && !error['message'].to_s.empty?
+
+      response.message.to_s
+    rescue JSON::ParserError
+      response.message.to_s
+    end
+  end
+end

data/plugins/filter/github_feed.rb

@@ -1,37 +1,49 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::GithubFeed
-# Author::    Kohei Hasegawa <http://github.com/banyan>
-#             774 <http://id774.net>
-# Created::   Jun  6, 2013
-# Updated::   Feb 21, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::GithubFeed
+# Author:       Kohei Hasegawa (More info: http://github.com/banyan)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Jun  6, 2013
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class FilterGithubFeed
-
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
+    # Converts Atom entries -- where title, id and content are elements with a
+    # #content -- into the flat items the rest of the pipeline expects. Needed
+    # because GitHub publishes Atom, not RSS.
     def run
-      @return_feeds = []
-      @pipeline.each {|feeds|
-        new_feeds = []
-        unless feeds.nil?
-          feeds.items.each {|feed|
-            Automatic::Log.puts("info", "Invoked: FilterGithubFeed")
-            hashie = Hashie::Mash.new
-            hashie.title       = feed.title.content
-            hashie.link        = feed.id.content
-            hashie.description = feed.content.content
-            new_feeds << hashie
-          }
-        end
-        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
-      }
-      @return_feeds
+      @pipeline.each_with_object([]) do |feeds, returned|
+        items = feeds.nil? ? [] : feeds.items.map { |item| flatten(item) }
+        returned << Automatic::FeedMaker.create_pipeline(items)
+      end
+    end
+
+    private
+
+    def flatten(item)
+      Automatic::Log.puts('info', 'Invoked: FilterGithubFeed')
+      entry             = Hashie::Mash.new
+      entry.title       = value(item, :title)
+      entry.link        = value(item, :id)
+      entry.description = value(item, :content)
+      entry
+    end
+
+    # An Atom element carries its text in #content; a field that is already a
+    # string is used as it stands, so that a pipeline which has been through
+    # another filter first is not a NoMethodError.
+    def value(item, name)
+      return nil unless item.respond_to?(name)
+
+      field = item.send(name)
+      field.respond_to?(:content) ? field.content : field
     end
   end
 end

data/plugins/filter/ignore.rb

@@ -1,63 +1,52 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::Ignore
-# Author::    774 <http://id774.net>
-# Created::   Feb 22, 2012
-# Updated::   Feb 21, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::Ignore
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 22, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class FilterIgnore
-    def initialize(config, pipeline=[])
-      @config = config
+    FIELDS = %i[title link description].freeze
+
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
+    # Drops items containing any listed keyword. Matching is a substring test,
+    # so an empty keyword drops everything.
     def run
-      @return_feeds = []
-      @pipeline.each {|feeds|
-        new_feeds = []
-        unless feeds.nil?
-          feeds.items.each {|items|
-            new_feeds << items if exclude(items) == false
-          }
-        end
-        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0
-      }
-      @return_feeds
+      @pipeline.each_with_object([]) do |feeds, returned|
+        kept = feeds.nil? ? [] : feeds.items.reject { |item| exclude?(item) }
+        returned << Automatic::FeedMaker.create_pipeline(kept) unless kept.empty?
+      end
     end
 
     private
-    def detect_exclude(item, evaluation, reason)
-      begin
-        if item.include?(evaluation)
-          Automatic::Log.puts("info", "Excluded by #{reason}: #{item}")
-          return true
+
+    def exclude?(item)
+      FIELDS.any? do |field|
+        Array(@config[field.to_s]).any? do |keyword|
+          excluded?(item.send(field), keyword.to_s.chomp, field)
         end
-      rescue NoMethodError
-        Automatic::Log.puts("warn", "Invalid feed detected in ignore process with #{item}")
-        return false
       end
     end
 
-    def exclude(items)
-      detection = false
-      unless @config['title'].nil?
-        @config['title'].each {|e|
-          detection = true if detect_exclude(items.title, e.chomp, 'title')
-        }
-      end
-      unless @config['link'].nil?
-        @config['link'].each {|e|
-          detection = true if detect_exclude(items.link, e.chomp, 'link')
-        }
-      end
-      unless @config['description'].nil?
-        @config['description'].each {|e|
-          detection = true if detect_exclude(items.description, e.chomp, 'description')
-        }
+    # An item whose field is missing is kept, with a warning.
+    def excluded?(value, keyword, field)
+      unless value.respond_to?(:include?)
+        Automatic::Log.puts('warn', "Invalid feed detected in ignore process with #{value}")
+        return false
       end
-      detection
+
+      return false unless value.include?(keyword)
+
+      Automatic::Log.puts('info', "Excluded by #{field}: #{value}")
+      true
     end
   end
 end

data/plugins/filter/image.rb

@@ -1,42 +1,53 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::Image
-# Author::    774 <http://id774.net>
-# Created::   Sep 18, 2012
-# Updated::   Apr  5, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::Image
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Sep 18, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class FilterImage
+    require 'uri'
 
-    def initialize(config, pipeline=[])
-      @config = config
+    # webp and avif are here because they are what an image link on the
+    # current web frequently is; tif joins tiff for the same reason.
+    EXTENSIONS = /\.(jpe?g|gif|png|tiff?|webp|avif)\z/i
+
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
+    # Sets link to nil unless it names an image. Note that the items are kept:
+    # their links are blanked, and the plugins after this one skip an item
+    # whose link is nil.
     def run
-      @return_feeds = []
-      @pipeline.each {|feeds|
-        return_feed_items = []
-        unless feeds.nil?
-          feeds.items.each {|feed|
-            feed.link = image?(feed.link) unless feed.link.nil?
-          }
-          @return_feeds << feeds
+      @pipeline.each_with_object([]) do |feeds, returned|
+        next if feeds.nil?
+
+        feeds.items.each do |item|
+          item.link = nil unless item.link.nil? || image?(item.link)
         end
-      }
-      @return_feeds
+        returned << feeds
+      end
     end
 
     private
+
+    # The test is on the path, so that a link carrying a query string -- which
+    # is how an image is served by most of what serves images now -- is still
+    # recognised. A string that will not parse is tested whole, as before.
     def image?(link)
-      case link
-        when /\.jpe?g\Z/i then link
-        when /\.gif\Z/i then link
-        when /\.png\Z/i then link
-        when /\.tiff\Z/i then link
-        else nil
-      end
+      EXTENSIONS.match?(path(link))
+    end
+
+    def path(link)
+      URI.parse(link).path.to_s
+    rescue URI::InvalidURIError
+      link
     end
   end
 end

data/plugins/filter/image_source.rb

@@ -1,71 +1,77 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::ImageSource
-# Author::    774 <http://id774.net>
-# Created::   Feb 28, 2012
-# Updated::   Feb 26, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::ImageSource
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 28, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class FilterImageSource
-    require 'net/http'
-    require 'kconv'
+    Automatic.require_optional('nokogiri', needed_by: 'FilterImageSource')
+    require 'uri'
 
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
+    # Replaces each item with one item per image found: the images in the
+    # description, or, where it has none, the images on the page the link
+    # points at. The second case reaches the network.
     def run
-      @return_feeds = []
-      @pipeline.each {|feeds|
-        new_feeds = Array.new
-        unless feeds.nil?
-          feeds.items.each {|feed|
-            arr = rewrite_link(feed)
-            if arr.length > 0
-              arr.each {|link|
-                Automatic::Log.puts("info", "Extract Image: #{link}")
-                hashie = Hashie::Mash.new
-                hashie.title = 'FilterImageSource'
-                hashie.link = link
-                new_feeds << hashie
-              }
-            end
-          }
-        end
-        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
-      }
-      @return_feeds
+      @pipeline.each_with_object([]) do |feeds, returned|
+        items = feeds.nil? ? [] : feeds.items.flat_map { |item| extract(item) }
+        returned << Automatic::FeedMaker.create_pipeline(items)
+      end
     end
 
     private
 
-    def rewrite_link(feed)
-      array = Array.new
-      feed.description.scan(/<img src="(.*?)"/) {|matched|
-        array = array | matched
-      }
-      if array.length === 0 && feed.link != nil
-        array = imgs(feed.link)
+    def extract(item)
+      images(item).map do |link|
+        Automatic::Log.puts('info', "Extract Image: #{link}")
+        image = Hashie::Mash.new
+        image.title = 'FilterImageSource'
+        image.link  = link
+        image
       end
-      array
     end
 
-    def imgs(link)
-      images = Array.new
-      html = open(link).read
-      unless html.nil?
-        doc = Nokogiri::HTML(html)
-        (doc/:img).each {|img|
-          image = img[:src]
-          unless /^http/ =~ image
-            image = link.sub(/\/([^\/]+)$/, image.sub(/^\./,''))
-          end
-          images << image
-        }
-      end
-      images
+    def images(item)
+      found = sources(item.description.to_s, item.link)
+      return found unless found.empty?
+      return [] if item.link.nil?
+
+      page_images(item.link)
+    end
+
+    def page_images(link)
+      sources(Automatic::Http.read(link), link)
+    rescue StandardError => e
+      Automatic::Log.puts('warn', "Failed to read images from #{link}: #{e.message}")
+      []
+    end
+
+    # The images of an HTML fragment, as absolute URLs. This was a scan for
+    # `<img src="` before, which found nothing in a document quoting its
+    # attributes with apostrophes or writing src after another attribute; the
+    # parser this plugin already needs answers the question properly.
+    def sources(html, base)
+      Nokogiri::HTML.fragment(html).css('img').filter_map do |image|
+        source = image['src'].to_s.strip
+        absolute(source, base) unless source.empty?
+      end.uniq
+    end
+
+    def absolute(source, base)
+      return source if base.nil?
+
+      URI.join(base, source).to_s
+    rescue StandardError
+      source
     end
   end
 end