automatic 14.12.2 → 26.08

data/plugins/store/digest.rb

@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+# Name::        Automatic::Plugin::Store::Digest
+# Description:: Drop items whose content has been seen before, by SHA-256 digest.
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Aug 17, 2026
+# Updated::     Aug 17, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+#
+# Content identity, where StorePermalink is URL identity. The fields the Recipe
+# names are normalized, joined into one canonical string and hashed, and the
+# digest is what the database holds: an item whose digest is already there has
+# been seen, whatever its link says, and an item whose link has been seen before
+# under different content has not.
+#
+# It is exact matching, of the content the Recipe selected. Whitespace runs and
+# Unicode composition differences are absorbed; nothing else is. Two articles
+# that say the same thing in different words are two articles here, and making
+# them one is not this plugin's work -- see doc/PLUGINS.md section 6.4.
+
+require 'digest'
+require_relative 'database'
+
+module Automatic::Plugin
+  # Named for what it holds rather than as `Digest`, which is Ruby's own
+  # hashing module and is what computes the value stored here.
+  class DigestRecord < ActiveRecord::Base
+  end
+
+  class StoreDigest
+    include Automatic::Plugin::Database
+
+    # The item fields a digest may be taken over. `date` is not one: an item
+    # republished unchanged carries a new date often enough that including it
+    # would defeat the purpose. `enclosure` is not one either, being a
+    # structure rather than a value.
+    FIELDS = %w[title link description author comments source content_encoded].freeze
+
+    # What an item is, absent a Recipe saying otherwise: what it says, not
+    # where it is. A feed that reissues an article under a new URL is the case
+    # this plugin exists for.
+    DEFAULT_FIELDS = %w[title description].freeze
+
+    # The field name is part of what is hashed, and this separates it from its
+    # value and one field from the next. A byte no title, body or URL contains,
+    # so that no arrangement of two fields collides with another.
+    SEPARATOR = "\0"
+
+    def initialize(config, pipeline = [])
+      @config   = config || {}
+      @pipeline = pipeline
+    end
+
+    def column_definition
+      { digest: :string, created_at: :string }
+    end
+
+    def model_class
+      Automatic::Plugin::DigestRecord
+    end
+
+    # Records the digest of each item's selected fields and passes on only the
+    # digests not already recorded.
+    #
+    # Nothing is rescued around the database. A store plugin that failed to
+    # write and passed the item on anyway would publish it again on the next
+    # run, which is the one thing this plugin is for.
+    def run
+      validate_settings
+      prepare_database
+
+      @pipeline.each_with_object([]) do |feeds, returned|
+        next if feeds.nil?
+
+        new_items = feeds.items.select { |item| new_item?(item) }
+        returned << Automatic::FeedMaker.create_pipeline(new_items) unless new_items.empty?
+      end
+    end
+
+    private
+
+    # The table the mixin builds, plus the constraint. Two runs of one Recipe
+    # overlapping -- a `cron` entry that takes longer than its interval -- would
+    # otherwise both read a digest as absent and both store it. The unique
+    # index makes the second write fail instead, which #store below reads as
+    # what it is.
+    def create_table
+      super
+      ActiveRecord::Base.connection.add_index(model_class.table_name, :digest, unique: true)
+    end
+
+    # Everything the Recipe has to get right, checked before the database file
+    # is opened. A Recipe this plugin cannot carry out is the operator's
+    # mistake and will be the same mistake next hour.
+    def validate_settings
+      raise ArgumentError, 'StoreDigest needs a db file name' if @config['db'].to_s.strip.empty?
+
+      fields
+    end
+
+    # The fields, in the order the Recipe wrote them. That order is part of the
+    # fingerprint and is not sorted here: a Recipe that changes it has said
+    # something different, and its digests are different digests.
+    #
+    # Only an absent `fields` takes the default. Every other way of getting it
+    # wrong is refused rather than corrected, because a Recipe's own statement
+    # of what makes two items the same is what this plugin has to obey.
+    def fields
+      @fields ||= validated_fields
+    end
+
+    def validated_fields
+      given = @config['fields']
+      return DEFAULT_FIELDS if given.nil?
+
+      unless given.is_a?(Array)
+        raise ArgumentError, "StoreDigest takes a list of fields, not #{given.inspect}"
+      end
+
+      names = given.map(&:to_s)
+      raise ArgumentError, 'StoreDigest was given an empty fields list' if names.empty?
+
+      unknown = names - FIELDS
+      unless unknown.empty?
+        raise ArgumentError,
+              "StoreDigest cannot take a digest over #{unknown.join(', ')}; " \
+              "the fields are #{FIELDS.join(', ')}"
+      end
+
+      duplicated = names.tally.select { |_name, count| count > 1 }.keys
+      unless duplicated.empty?
+        raise ArgumentError, "StoreDigest was given #{duplicated.join(', ')} twice"
+      end
+
+      names
+    end
+
+    # Whether the item goes downstream. An item whose digest was stored here is
+    # new, one whose digest was already stored is not, and one there is nothing
+    # to hash goes on unjudged.
+    def new_item?(item)
+      digest = digest_for(item)
+
+      if digest.nil?
+        Automatic::Log.puts('warn',
+                            "StoreDigest: no digestable content for #{item.link}; " \
+                            'passing item unchanged')
+        return true
+      end
+
+      return false if stored_digest?(digest)
+
+      store(digest, item)
+    end
+
+    def stored_digest?(digest)
+      model_class.exists?(digest: digest)
+    end
+
+    # True when this run is the one that stored the digest. The read above
+    # answers this on its own for a single run; the rescue is for the run
+    # overlapping another, where the row appeared between the two statements.
+    def store(digest, item)
+      model_class.create!(digest: digest, created_at: Time.now.strftime('%Y/%m/%d %X'))
+      Automatic::Log.puts('info', "Saving Digest: #{digest} (#{item.link})")
+      true
+    rescue ActiveRecord::RecordNotUnique
+      false
+    end
+
+    # The digest of the item's selected fields, or nil where the Recipe
+    # selected nothing the item has.
+    #
+    # An item with nothing to hash is not stored under the digest of the empty
+    # string: every item with no description would then be the same item as
+    # every other, and the first of them would silence the rest for good.
+    def digest_for(item)
+      values = fields.map { |field| [field, normalize(value_of(item, field))] }
+      return nil if values.all? { |_field, value| value.empty? }
+
+      canonical = values.map { |field, value| "#{field}#{SEPARATOR}#{value}" }.join(SEPARATOR)
+      ::Digest::SHA256.hexdigest(canonical)
+    end
+
+    # A field an item does not carry reads as absent rather than raising.
+    # `content_encoded` is the case that matters: an item built by a filter has
+    # it, one straight from an RSS 2.0 feed may not.
+    def value_of(item, field)
+      item.respond_to?(field) ? item.public_send(field) : nil
+    end
+
+    # What two spellings of one string have to survive to hash alike: the
+    # encoding, the composition of accented characters, and how much whitespace
+    # a feed happens to have put between words. Nothing beyond that -- case,
+    # punctuation and markup are content here, and an item that differs in them
+    # is a different item.
+    #
+    # `scrub` is not redundant after `encode`: a conversion whose source and
+    # destination encodings are the same is skipped, invalid bytes and all, and
+    # `unicode_normalize` raises on what is left.
+    def normalize(value)
+      value.to_s.
+        encode('UTF-8', invalid: :replace, undef: :replace).
+        scrub.
+        unicode_normalize(:nfc).
+        gsub(/\s+/, ' ').
+        strip
+    end
+  end
+end

data/plugins/store/file.rb

@@ -1,95 +1,126 @@
-#!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Store::File
-# Author::    774 <http://id774.net>
-# Created::   Feb 28, 2012
-# Updated::   Feb 25, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
-
-require 'open-uri'
+# Name::        Automatic::Plugin::Store::File
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 28, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
+
+require 'fileutils'
 require 'uri'
-require 'aws-sdk'
 
 module Automatic::Plugin
   class StoreFile
+    # A link with either scheme is fetched from S3 rather than over HTTP.
+    # `s3n` is what Recipes written for this plugin use; `s3` is the spelling
+    # everything else uses and is accepted as well.
+    S3_SCHEMES = %w[s3 s3n].freeze
 
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
-      unless @config['bucket_name'].nil?
-        s3 = AWS::S3.new(
-          :access_key_id => @config['access_key'],
-          :secret_access_key => @config['secret_key']
-        )
-        @bucket = s3.buckets[@config['bucket_name']]
-      end
-      @return_feeds = []
     end
 
+    # Downloads what each link points at and rewrites the link to a file URI,
+    # which is how PublishAmazonS3 later knows it has a local file.
     def run
-      @pipeline.each {|feeds|
-        unless feeds.nil?
-          feeds.items.each {|feed|
-            unless feed.link.nil?
-              Automatic::Log.puts("info", "Downloading File: #{feed.link}")
-              FileUtils.mkdir_p(@config['path']) unless FileTest.exist?(@config['path'])
-              retries = 0
-              retry_max = @config['retry'].to_i || 0
-              begin
-                retries += 1
-                feed.link = get_file(feed.link)
-                sleep ||= @config['interval'].to_i
-                @return_feeds << feed
-              rescue
-                Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault during file download.")
-                sleep ||= @config['interval'].to_i
-                retry if retries <= retry_max
-              end
-            end
-          }
+      stored = []
+      @pipeline.each do |feeds|
+        next if feeds.nil?
+
+        feeds.items.each do |feed|
+          next if feed.link.nil?
+
+          stored << feed if store(feed)
         end
-      }
+      end
+
       @pipeline = []
-      @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0
+      @pipeline << Automatic::FeedMaker.create_pipeline(stored) unless stored.empty?
       @pipeline
     end
 
     private
 
+    def store(feed)
+      Automatic::Log.puts('info', "Downloading File: #{feed.link}")
+      FileUtils.mkdir_p(@config['path'].to_s)
+
+      retries   = 0
+      retry_max = @config['retry'].to_i
+      begin
+        feed.link = get_file(feed.link)
+        sleep(@config['interval'].to_i)
+        true
+      rescue StandardError => e
+        retries += 1
+        Automatic::Log.puts('error',
+                            "ErrorCount: #{retries}, Fault during file download: #{e.message}")
+        return false if retries > retry_max
+
+        sleep(@config['interval'].to_i)
+        retry
+      end
+    end
+
     def get_file(url)
       uri = URI.parse(url)
-      case uri.scheme
-      when "s3n"
-        return_path = get_aws(uri)
-      else
-        return_path = wget(uri, url)
-      end
-      Automatic::Log.puts("info", "Saved File: #{return_path}")
-      "file://" + return_path
+      path = S3_SCHEMES.include?(uri.scheme) ? from_s3(uri) : download(url)
+      Automatic::Log.puts('info', "Saved File: #{path}")
+      "file://#{path}"
     end
 
-    def wget(uri, url)
-      filename = File.basename(uri.path)
-      filepath = File.join(@config['path'], filename)
-      open(url) {|source|
-        open(filepath, "w+b") { |o|
-          o.print(source.read)
-        }
-      }
-      filepath
+    # Only HTTP and HTTPS are fetched: a link arrives from a feed, which is to
+    # say from outside, and `file://` is not something a store plugin should
+    # be talked into reading.
+    def download(url)
+      path = local_path(Automatic::Http.uri(url).path)
+      File.binwrite(path, Automatic::Http.read(url))
+      path
     end
 
-    def get_aws(uri)
-      filename = File.basename(uri.path)
-      filepath = File.join(@config['path'], filename)
-      object = @bucket.objects[uri.path]
-      File.open(filepath, 'wb') do |file|
-        object.read do |chunk|
-           file.write(chunk)
-        end
+    # AWS SDK for Ruby v3, one gem for one service. The bucket is the Recipe's
+    # `bucket_name` where it has one, so that an existing Recipe keeps its
+    # meaning, and the link's own host otherwise. Credentials are the Recipe's
+    # where it carries them and the SDK's default chain -- environment,
+    # profile, instance role -- where it does not, which is the way to run
+    # this without a secret in a file.
+    def from_s3(uri)
+      path = local_path(uri.path)
+      s3.get_object(bucket: bucket(uri), key: uri.path.sub(%r{\A/}, ''),
+                    response_target: path)
+      path
+    end
+
+    def s3
+      @s3 ||= begin
+        Automatic.require_optional('aws-sdk-s3', needed_by: 'the S3 path of StoreFile')
+        Aws::S3::Client.new(**client_options)
+      end
+    end
+
+    def client_options
+      options = {}
+      options[:region] = @config['region'].to_s unless @config['region'].nil?
+      unless @config['access_key'].nil?
+        options[:access_key_id]     = @config['access_key'].to_s
+        options[:secret_access_key] = @config['secret_key'].to_s
       end
-      filepath
+      options
+    end
+
+    def bucket(uri)
+      name = @config['bucket_name'].to_s
+      name.empty? ? uri.host.to_s : name
+    end
+
+    def local_path(remote_path)
+      name = File.basename(remote_path.to_s)
+      raise ArgumentError, "no file name in #{remote_path}" if name.empty? || name == '/'
+
+      File.join(@config['path'].to_s, name)
     end
   end
 end

data/plugins/store/full_text.rb

@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Store::FullText
-# Author::    774 <http://id774.net>
-# Created::   Feb 26, 2012
-# Updated::   Oct 09, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Store::FullText
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 26, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
-require 'plugins/store/database'
+require_relative 'database'
 
 module Automatic::Plugin
   class Blog < ActiveRecord::Base
@@ -15,43 +17,48 @@ module Automatic::Plugin
   class StoreFullText
     include Automatic::Plugin::Database
 
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
     def column_definition
       {
-        :title => :string,
-        :link => :string,
-        :description => :string,
-        :content => :string,
-        :created_at => :string,
+        title: :string,
+        link: :string,
+        description: :string,
+        content: :string,
+        created_at: :string
       }
     end
 
+    # Link or title, so that a republished article with a new URL is not
+    # stored twice.
     def unique_keys
-      [:link, :title]
+      %i[link title]
     end
 
     def model_class
       Automatic::Plugin::Blog
     end
 
+    # Records title, link, description and content_encoded, and passes on only
+    # what is new.
     def run
-      for_each_new_feed {|feed|
-        Automatic::Log.puts("info", "Saving FullText: #{feed.link}")
+      for_each_new_feed do |feed|
+        Automatic::Log.puts('info', "Saving FullText: #{feed.link}")
         begin
           Blog.create(
-            :title => feed.title,
-            :link => feed.link,
-            :description => feed.description,
-            :content => feed.content_encoded,
-            :created_at => Time.now.strftime("%Y/%m/%d %X"))
-        rescue
-          Automatic::Log.puts("warn", "Skip feed due to fault in save.")
+            title: feed.title,
+            link: feed.link,
+            description: feed.description,
+            content: feed.content_encoded,
+            created_at: Time.now.strftime('%Y/%m/%d %X')
+          )
+        rescue StandardError => e
+          Automatic::Log.puts('warn', "Skip feed due to fault in save: #{e.message}")
         end
-      }
+      end
     end
   end
 end

data/plugins/store/permalink.rb

@@ -1,13 +1,14 @@
-#!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::Store::Permalink
-# Author::    774 <http://id774.net>
-# Created::   Feb 22, 2012
-# Updated::   Oct 09, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::Store::Permalink
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 22, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
-require 'plugins/store/database'
+require_relative 'database'
 
 module Automatic::Plugin
   class Permalink < ActiveRecord::Base
@@ -16,16 +17,13 @@ module Automatic::Plugin
   class StorePermalink
     include Automatic::Plugin::Database
 
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
     def column_definition
-      {
-        :url => :string,
-        :created_at => :string
-      }
+      { url: :string, created_at: :string }
     end
 
     def unique_keys
@@ -36,15 +34,13 @@ module Automatic::Plugin
       Automatic::Plugin::Permalink
     end
 
+    # Records each item's link and passes on only the links not already
+    # recorded. The usual guard against publishing the same item twice.
     def run
-      for_each_new_feed {|feed|
-        unless feed.link.nil?
-          Permalink.create(
-            :url => feed.link,
-            :created_at => Time.now.strftime("%Y/%m/%d %X"))
-          Automatic::Log.puts("info", "Saving Permalink: #{feed.link}")
-        end
-      }
+      for_each_new_feed do |feed|
+        Permalink.create(url: feed.link, created_at: Time.now.strftime('%Y/%m/%d %X'))
+        Automatic::Log.puts('info', "Saving Permalink: #{feed.link}")
+      end
     end
   end
 end

data/plugins/subscription/feed.rb

@@ -1,33 +1,46 @@
 # -*- coding: utf-8 -*-
-# Name::      Automatic::Plugin::SubscriptionFeed
-# Author::    774 <http://id774.net>
-# Created::   Feb 22, 2012
-# Updated::   Feb 21, 2014
-# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
-# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+# Name::        Automatic::Plugin::SubscriptionFeed
+# Author:       id774 (More info: http://id774.net)
+# Source Code:: https://github.com/id774/automaticruby
+# License::     The GPL version 3, or LGPL version 3 (Dual License).
+# Contact::     idnanashi@gmail.com
+# Created::     Feb 22, 2012
+# Updated::     Aug 15, 2026
+# Copyright::   Copyright (c) 2012-2026 Automatic Ruby Developers.
 
 module Automatic::Plugin
   class SubscriptionFeed
-    def initialize(config, pipeline=[])
-      @config = config
+    def initialize(config, pipeline = [])
+      @config   = config || {}
       @pipeline = pipeline
     end
 
     def run
-      @config['feeds'].each {|feed|
-        retries = 0
-        retry_max = @config['retry'].to_i || 0
-        begin
-          rss = Automatic::FeedParser.get_url(feed)
-          @pipeline << rss
-        rescue
-          retries += 1
-          Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{feed}")
-          sleep ||= @config['interval'].to_i
-          retry if retries <= retry_max
-        end
-      }
+      Array(@config['feeds']).each do |url|
+        feed = fetch(url)
+        @pipeline << feed unless feed.nil?
+      end
       @pipeline
     end
+
+    private
+
+    # A feed that fails after its retries is logged and skipped; the others
+    # still run.
+    def fetch(url)
+      retries   = 0
+      retry_max = @config['retry'].to_i
+      begin
+        Automatic::FeedParser.get_url(url)
+      rescue StandardError => e
+        retries += 1
+        Automatic::Log.puts('error',
+                            "ErrorCount: #{retries}, Fault in parsing: #{url}, #{e.message}")
+        return nil if retries > retry_max
+
+        sleep(@config['interval'].to_i)
+        retry
+      end
+    end
   end
 end