automatic 14.12.2 → 26.08

data/plugins/filter/claude.rb

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+# Name::        Automatic::Plugin::Filter::Claude
+# Description:: Replace each item's description with what the Anthropic Claude API answers.
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Aug 17, 2026
+# Updated::     Aug 17, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# One transformation: the item's description goes to the Anthropic Messages API
+# under the Recipe's prompt, and the answer becomes the item's description.
+# What that transformation is -- a summary, a translation, an extraction, a
+# classification -- is the prompt's business, not this plugin's.
+#
+# This plugin knows Anthropic and nothing else. Its authentication is an
+# `x-api-key` header rather than a bearer token, it requires an API version
+# header and a `max_tokens`, and its answer is a list of content blocks. None
+# of that is bent into another service's shape, and no other service's request
+# is built here.
+#
+# @see https://docs.anthropic.com/en/api/messages
+
+module Automatic::Plugin
+  class FilterClaude
+    require 'json'
+    require 'net/http'
+    require 'openssl'
+    require 'uri'
+
+    ENDPOINT = URI('https://api.anthropic.com/v1/messages')
+
+    # The API version header the Messages API requires on every request. It is
+    # the version of the HTTP interface, not of a model, which is why it is a
+    # constant here and the model is a setting.
+    API_VERSION = '2023-06-01'.freeze
+
+    # `max_tokens` is required by this API and by no other one this repository
+    # speaks to, so it is a setting of this plugin alone. The default is a
+    # length a digest or a translation fits in; a Recipe that wants a longer
+    # answer says so.
+    DEFAULT_MAX_TOKENS = 4096
+
+    OPEN_TIMEOUT = 10
+
+    # Generous, and bounded. A model given several articles thinks for a while;
+    # an unattended run that waits forever is the failure this exists against.
+    READ_TIMEOUT = 300
+
+    # A failure that another attempt will not get past: a setting that is
+    # wrong, a request the service refuses, an answer this plugin cannot read.
+    class Error < StandardError; end
+
+    # A failure that another attempt may get past: the network, a rate limit, a
+    # server error.
+    class TemporaryError < StandardError; end
+
+    def initialize(config, pipeline = [])
+      @config   = config || {}
+      @pipeline = pipeline
+    end
+
+    # Replaces each item's description with the answer. Nothing else about an
+    # item is touched, and the feeds and their items arrive and leave in the
+    # same order and number.
+    def run
+      validate_settings
+
+      @pipeline.each { |feeds|
+        next if feeds.nil?
+
+        feeds.items.each { |item| transform(item) }
+      }
+      @pipeline
+    end
+
+    private
+
+    # Checked before the first request, because a Recipe this plugin cannot
+    # carry out is the operator's mistake and will be the same mistake on every
+    # item. The token is never named in a message.
+    def validate_settings
+      raise ArgumentError, 'FilterClaude needs a token' if token.empty?
+      raise ArgumentError, 'FilterClaude needs a model' if model.empty?
+      raise ArgumentError, 'FilterClaude needs a prompt' if prompt.empty?
+      raise ArgumentError, 'FilterClaude needs a positive max_tokens' unless max_tokens.positive?
+    end
+
+    def token
+      @config['token'].to_s
+    end
+
+    def model
+      @config['model'].to_s.strip
+    end
+
+    def prompt
+      @config['prompt'].to_s.strip
+    end
+
+    def max_tokens
+      given = @config['max_tokens']
+      given.nil? ? DEFAULT_MAX_TOKENS : given.to_i
+    end
+
+    def transform(item)
+      text = item.description.to_s
+      if text.strip.empty?
+        Automatic::Log.puts('warn', "FilterClaude: nothing to send for #{item.link}")
+        return
+      end
+
+      Automatic::Log.puts('info', "FilterClaude: asking #{model} about #{item.link}")
+      item.description = answer(text)
+    end
+
+    # The retry shape of doc/PLUGINS.md section 3.6, applied only to what
+    # retrying can help. A missing setting, a refused request or an answer in a
+    # shape this plugin cannot read is raised at once: trying again would fail
+    # the same way, more slowly.
+    def answer(text)
+      retries   = 0
+      retry_max = @config['retry'].to_i
+      begin
+        message(text)
+      rescue TemporaryError => e
+        retries += 1
+        Automatic::Log.puts('error', "ErrorCount: #{retries}, FilterClaude: #{e.message}")
+        if retries <= retry_max
+          sleep(@config['interval'].to_i)
+          retry
+        end
+        raise Error, "FilterClaude gave up after #{retries} attempts: #{e.message}"
+      end
+    end
+
+    def message(text)
+      # The prompt is the system instruction and the description is the user
+      # turn it is applied to. They are separate fields, so that what an
+      # article says is never read as an instruction to this plugin or to the
+      # model.
+      body = {
+        'model' => model,
+        'max_tokens' => max_tokens,
+        'system' => prompt,
+        'messages' => [{ 'role' => 'user', 'content' => text }]
+      }
+      content(post(JSON.generate(body)))
+    end
+
+    def post(body)
+      request = Net::HTTP::Post.new(ENDPOINT)
+      request['x-api-key']         = token
+      request['anthropic-version'] = API_VERSION
+      request['Content-Type']      = 'application/json'
+      request.body = body
+
+      # TLS with the certificate verified, which is Net::HTTP's own default and
+      # is named here because it is not a thing to be turned off.
+      Net::HTTP.start(ENDPOINT.host, ENDPOINT.port,
+                      use_ssl: true,
+                      verify_mode: OpenSSL::SSL::VERIFY_PEER,
+                      open_timeout: OPEN_TIMEOUT,
+                      read_timeout: READ_TIMEOUT) { |http| http.request(request) }
+    rescue Timeout::Error, SystemCallError, SocketError, IOError,
+           OpenSSL::SSL::SSLError, Net::HTTPBadResponse => e
+      raise TemporaryError, "the request to Claude failed: #{e.message}"
+    end
+
+    def content(response)
+      case response
+      when Net::HTTPSuccess
+        answer_text(parse(response.body))
+      when Net::HTTPTooManyRequests, Net::HTTPServerError
+        raise TemporaryError, "Claude answered #{response.code}: #{reason(response)}"
+      else
+        raise Error, "Claude answered #{response.code}: #{reason(response)}"
+      end
+    end
+
+    def parse(body)
+      JSON.parse(body.to_s)
+    rescue JSON::ParserError => e
+      raise Error, "Claude answered with something that is not JSON: #{e.message}"
+    end
+
+    # The text of the answer, out of the content blocks the Messages API
+    # returns. A block of another type -- this API has several -- is not text
+    # and is passed over. An answer this plugin cannot find is an error and not
+    # an empty description: a Recipe that published the empty string here would
+    # have thrown the article away and reported success.
+    def answer_text(body)
+      blocks = body['content']
+      raise Error, 'Claude answered without a content array' unless blocks.is_a?(Array)
+
+      text = blocks.select { |block| block.is_a?(Hash) && block['type'] == 'text' }.
+             map { |block| block['text'].to_s }.join.strip
+      raise Error, 'Claude answered with no text content' if text.empty?
+
+      text
+    end
+
+    # The service's own explanation where it gave one, the status line
+    # otherwise. Neither carries the token, and the settings are never logged
+    # or raised wholesale.
+    def reason(response)
+      body = JSON.parse(response.body.to_s)
+      error = body['error']
+      return error['message'].to_s if error.is_a?(Hash) && !error['message'].to_s.empty?
+
+      response.message.to_s
+    rescue JSON::ParserError
+      response.message.to_s
+    end
+  end
+end

data/plugins/filter/clear.rb

@@ -1,18 +1,22 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::Clear
-# Author::    774 <http://id774.net>
-# Created::   Oct 20, 2014
-# Updated::   Oct 20, 2014
-# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::Clear
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Oct 20, 2014
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class FilterClear
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
+    # Returns an empty pipeline, to end a Recipe after a store plugin has done
+    # the work so that later plugins publish nothing.
     def run
       []
     end

data/plugins/filter/description_link.rb

@@ -1,74 +1,72 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::DescriptionLink
-# Author::    774 <http://id774.net>
-# Created::   Oct 03, 2014
-# Updated::   Oct 16, 2014
-# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::DescriptionLink
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Oct 03, 2014
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class FilterDescriptionLink
+    Automatic.require_optional('nokogiri', needed_by: 'FilterDescriptionLink')
     require 'uri'
-    require 'nkf'
 
-    def initialize(config, pipeline=[])
-      @config = config
+    # URI.extract and URI::PATTERN answer through the RFC 3986 parser that
+    # URI::Parser became in Ruby 3.4, which reports both as obsolete. The RFC
+    # 2396 parser is what they were always reaching, it is spelled the same way
+    # on every supported Ruby, and it is named here directly.
+    PARSER = URI::RFC2396_Parser.new
+
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
+    # Takes the last HTTP or HTTPS URL out of the description and makes it the
+    # link, for feeds that carry the real destination in the body.
     def run
-      @return_feeds = []
-      @pipeline.each {|feeds|
-        new_feeds = []
-        unless feeds.nil?
-          feeds.items.each {|feed|
-            new_feeds << rewrite_link(feed)
-          }
-        end
-        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
-      }
-      @return_feeds
+      @pipeline.each_with_object([]) do |feeds, returned|
+        items = feeds.nil? ? [] : feeds.items.map { |item| rewrite(item) }
+        returned << Automatic::FeedMaker.create_pipeline(items)
+      end
     end
 
     private
 
-    def get_title(url)
-      new_title = nil
-      if url.class == String
-        url.gsub!(Regexp.new("[^#{URI::PATTERN::ALNUM}\/\:\?\=&~,\.\(\)#]")) {|match| ERB::Util.url_encode(match)}
-        begin
-          read_data = NKF.nkf("--utf8", open(url).read)
-          get_text = Nokogiri::HTML.parse(read_data, nil, 'utf8').xpath('//title').text
-          new_title = get_text if get_text.class == String
-        rescue
-          Automatic::Log.puts("warn", "Failed in get title for: #{url}")
-        end
-      end
+    def rewrite(item)
+      link = PARSER.extract(item.description.to_s, %w[http https]).uniq.last
+      item.link = link unless link.nil?
+
+      item.description = '' if setting?('clear_description')
+      retitle(item) if setting?('get_title')
 
-      new_title
+      item
     end
 
-    def rewrite_link(feed)
-      new_link = URI.extract(feed.description, %w{http https}).uniq.last
-      feed.link = new_link unless new_link.nil?
+    # Settings are read whatever the mapping is. This tested `@config.class ==
+    # Hash`, which a Recipe never satisfies: the framework hands a plugin a
+    # Hashie::Mash, so both settings below were silently ignored in every real
+    # run and read only by a spec passing a plain Hash.
+    def setting?(name)
+      @config[name].to_s == '1'
+    end
 
-      if @config.class == Hash
-        if @config['clear_description'] == 1
-          feed.description = ""
-        end
+    def retitle(item)
+      title = fetch_title(item.link)
+      item.title = title unless title.nil? || title.empty?
+    end
 
-        if @config['get_title'] == 1
-          begin
-            new_title = get_title(feed.link)
-            feed.title = new_title unless new_title.nil?
-          rescue OpenURI::HTTPError
-            Automatic::Log.puts("warn", "404 Not Found in get title process.")
-          end
-        end
-      end
+    # One request per item; use FilterOne or a store plugin before this on a
+    # large feed. A page that cannot be read leaves the item's own title.
+    def fetch_title(url)
+      return nil unless Automatic::Http.fetchable?(url)
 
-      feed
+      Nokogiri::HTML.parse(Automatic::Http.read(url)).xpath('//title').text
+    rescue StandardError => e
+      Automatic::Log.puts('warn', "Failed in get title for: #{url}, #{e.message}")
+      nil
     end
   end
 end
-

data/plugins/filter/full_feed.rb

@@ -1,77 +1,183 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Filter::FullFeed
-# Author::    progd <http://d.hatena.ne.jp/progd/20120429/automatic_ruby_filter_full_feed>
-#             774 <http://id774.net>
-# Created::   Apr 29, 2012
-# Updated::   Jun 23, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Filter::FullFeed
+# Author:       progd (More info: http://d.hatena.ne.jp/progd/20120429/automatic_ruby_filter_full_feed)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Apr 29, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
-
   class FilterFullFeed
-    require 'nokogiri'
+    Automatic.require_optional('nokogiri', needed_by: 'FilterFullFeed')
+    require 'json'
+    require 'stringio'
+
+    SITEINFO_TYPES = %w[SBM INDIVIDUAL IND SUBGENERAL SUB GENERAL GEN].freeze
 
-    SITEINFO_TYPES = %w[SBM INDIVIDUAL IND SUBGENERAL SUB GENERAL GEN]
+    # One siteinfo record, reduced to the four things a match needs and with
+    # its URL pattern compiled once. The database ships with 3,504 usable
+    # records, so compiling them per item -- which is what matching against
+    # the raw JSON did -- was several thousand `Regexp.new` calls for every
+    # link in every feed.
+    Entry = Struct.new(:url, :pattern, :xpath, :encoding)
 
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
-      @siteinfo = get_siteinfo
+      @siteinfo = siteinfo
     end
 
+    # Replaces a summary with the article body, by matching the link against a
+    # siteinfo database of URL patterns and XPaths and fetching the page.
     def run
-      @return_feeds = []
-      @pipeline.each {|feeds|
+      @pipeline.each_with_object([]) do |feeds, returned|
         unless feeds.nil?
-          feeds.items.each {|feed|
-            feed = fulltext(feed)
-          }
+          feeds.items.each { |item| fulltext(item) }
         end
-        @return_feeds << feeds
-      }
-      @return_feeds
+        returned << feeds
+      end
     end
 
     private
 
-    def get_siteinfo
-      Automatic::Log.puts(:info, "Loading siteinfo from #{@config['siteinfo']}")
-      siteinfo = JSON.load(open(File.join(assets_dir, @config['siteinfo'])).read.force_encoding("UTF-8"))
-      siteinfo.select! { |info| SITEINFO_TYPES.include? (info['data']['type']) }
-      siteinfo.sort! { |a, b|
-        atype, btype = a['data']['type'], b['data']['type']
-        SITEINFO_TYPES.index(atype) <=> SITEINFO_TYPES.index(btype)
-      }
-      return siteinfo
+    def siteinfo
+      name = @config['siteinfo'].to_s
+      raise ArgumentError, 'FilterFullFeed needs a siteinfo file name' if name.empty?
+
+      Automatic::Log.puts('info', "Loading siteinfo from #{name}")
+      records = JSON.parse(File.read(File.join(assets_dir, name), encoding: 'UTF-8'))
+      entries = records.select { |info| SITEINFO_TYPES.include?(info['data']['type']) }
+                       .sort_by { |info| SITEINFO_TYPES.index(info['data']['type']) }
+                       .filter_map { |info| entry(info['data']) }
+      Automatic::Log.puts('info', "Loaded #{entries.size} siteinfo entries")
+      entries
+    end
+
+    # A record is dropped here rather than failing a match later. A record
+    # without a URL pattern would match every link -- an empty pattern matches
+    # everything -- and one without an XPath has nothing to select with.
+    def entry(data)
+      url   = data['url'].to_s
+      xpath = data['xpath'].to_s.strip
+      return nil if url.empty? || xpath.empty?
+
+      Entry.new(url, Regexp.new(url), xpath, charset(data['enc']))
+    rescue RegexpError
+      nil
     end
 
     def assets_dir
-      dir = (File.expand_path('~/.automatic/assets/siteinfo'))
-      if File.directory?(dir)
-        dir
-      else
-        File.join(File.dirname(__FILE__), '..', '..', 'assets', 'siteinfo')
+      dir = File.expand_path('~/.automatic/assets/siteinfo')
+      return dir if File.directory?(dir)
+
+      File.expand_path('../../assets/siteinfo', __dir__)
+    end
+
+    def fulltext(item)
+      return if item.link.nil?
+
+      record = match(item.link)
+      if record.nil?
+        Automatic::Log.puts('info', "Fulltext SITEINFO not found: #{item.link}")
+        return
       end
+
+      Automatic::Log.puts('info', "Siteinfo matched: #{record.url}")
+      html = body(item.link, record)
+      if html.nil?
+        # The page was read but the XPath selected nothing, which is what a
+        # site that has been redesigned since its record was written looks
+        # like. Assigning the empty result here is what used to replace a
+        # perfectly good summary with an empty description.
+        Automatic::Log.puts('warn', "Fulltext XPath selected nothing on #{item.link}: #{record.xpath}")
+        return
+      end
+
+      item.description = html
+    rescue StandardError => e
+      # An unreadable page leaves the item's own summary in place, which is
+      # what this filter is an improvement on rather than a replacement for.
+      Automatic::Log.puts('warn', "Failed to read fulltext for #{item.link}: #{e.message}")
     end
 
-    def fulltext(feed)
-      return feed unless feed.link
-      @siteinfo.each {|info|
-        begin
-          if feed.link.match(info['data']['url'])
-            Automatic::Log.puts(:info, "Siteinfo matched: #{info['data']['url']}")
-            html = Nokogiri::HTML.parse(open(feed.link))
-            body = html.xpath(info['data']['xpath'])
-            feed.description = body.to_html.encode('UTF-8', :undef => :replace)
-            return feed
-          end
-        rescue
-          return feed
-        end
-      }
-      Automatic::Log.puts(:info, "Fulltext SITEINFO not found: #{feed.link}")
-      return feed
+    def match(link)
+      links = schemes(link)
+      @siteinfo.find { |record| links.any? { |candidate| record.pattern.match?(candidate) } }
+    end
+
+    # The database was last updated in 2013 and 3,448 of its 3,504 records
+    # anchor on a scheme, nearly all of them `^http://`. The sites they name
+    # have since moved to HTTPS, so an https link out of a feed matches none of
+    # them and the filter silently does nothing. Matching the link under either
+    # scheme is what keeps those records reachable; a record is about a site's
+    # layout, not about how it is transported. Only the match is rewritten --
+    # the page is fetched from the link the feed gave.
+    def schemes(link)
+      case link
+      when %r{\Ahttps://} then [link, link.sub(%r{\Ahttps://}, 'http://')]
+      when %r{\Ahttp://}  then [link, link.sub(%r{\Ahttp://}, 'https://')]
+      else [link]
+      end
+    end
+
+    # Returns the selected body as UTF-8, or nil when the XPath selects
+    # nothing, so that the caller can tell an article apart from an empty
+    # result and keep the summary it already had.
+    def body(link, entry)
+      nodes = document(link, entry).xpath(entry.xpath)
+      return nil if nodes.empty?
+
+      # Normalised to UTF-8, and scrubbed rather than raised on. The parser
+      # returns a string in whatever encoding it settled on for the page, and
+      # a page whose declared charset is not the one it is written in is
+      # common enough in a database this old; what leaves here goes on to a
+      # publish plugin, which has no way to recover from a string it cannot
+      # encode. `invalid:` as well as `undef:`, because converting UTF-8 to
+      # UTF-8 leaves invalid bytes alone unless they are named.
+      html = nodes.to_html.encode('UTF-8', invalid: :replace, undef: :replace)
+      html.strip.empty? ? nil : html
+    end
+
+    # Read the page and parse it under the encoding it is actually written in.
+    #
+    # The page is handed to the parser as a stream rather than as a decoded
+    # string. open-uri applies an encoding to what it returns whether or not
+    # the response declared one -- a page served as `text/html` with no charset
+    # comes back tagged UTF-8 -- and a decoded string tells the parser to
+    # believe that tag and never look at the meta tag underneath it. The
+    # database is full of sites that declare their charset only in a meta tag,
+    # and for those the difference is the whole article in mojibake.
+    #
+    # A record's own `enc` is the last resort, for a page that declares nothing
+    # anywhere: it was recorded in 2013, and trusting it ahead of what the page
+    # says would break every site that has changed encoding since.
+    def document(link, entry)
+      page, declared = Automatic::Http.open(link) { |io| [io.read, declared_charset?(io)] }
+      parsed = Nokogiri::HTML.parse(StringIO.new(page))
+      return parsed if declared || parsed.meta_encoding || entry.encoding.nil?
+
+      Nokogiri::HTML.parse(StringIO.new(page), nil, entry.encoding)
+    end
+
+    # Whether the response itself named a charset, as opposed to open-uri
+    # having settled on one in the absence of an answer.
+    def declared_charset?(io)
+      return false unless io.respond_to?(:meta)
+
+      io.meta['content-type'].to_s.match?(/;\s*charset\s*=/i)
+    end
+
+    # An encoding named by a siteinfo record. An unknown name is ignored rather
+    # than raised, and the page is left to speak for itself.
+    def charset(name)
+      string = name.to_s.strip
+      return nil if string.empty?
+
+      Encoding.find(string).name
+    rescue ArgumentError
+      nil
     end
   end
 end