flnews_post_proc 1.7

data/lib/flnews_post_proc.conf

@@ -0,0 +1,173 @@
+#/***************************************************************************
+# *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>           *
+# *                                                                         *
+# *   This program is free software; you can redistribute it and/or modify  *
+# *   it under the terms of the DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE *
+# *                                                                         *
+# *   This program is distributed in the hope that it will be useful,       *
+# *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+# *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+# *                                                                         *
+# ***************************************************************************/
+# This is a YAML file. Keep intact these three dashes.
+---
+# CONFIG_VERSION – DO NOT MODIFY
+# If this version is smaller than that of the current program, this file
+# may be updated automatically. Your settings should not be altered in 
+# the process, if they are still valid.
+# ATTN! COMMENTS WILL BE REMOVED but a copy of your previous configuration
+# will be saved in a file with a version suffix.
+CONFIG_VERSION: 1.35
+
+# FUP_NAME 
+# A Regular Expression, describing the string which contains the name of
+# previous poster who is the author of a quoted post.  This string is
+# recognized in the original article and may be used with the fitting element
+# from GROUP_INTRO, below. The Regexp-format is that of the Regexp class in
+# Ruby, noted as a String. Beware to mask a backslash '\' by another one, 
+# like in the example. A capture-group '()' serves to extract the name from the
+# match result. 
+# Leave this field empty to keep the default from the FLNews configuration
+# intact.
+# CONTENT: A String equivalent of a regular expression.
+# DEFAULT: EMPTY
+# EXAMPLE1: "Am \\d+.\\d+.\\d{2,4} um \\d+:\\d+ schrieb (.*):"
+# EXAMPLE2: "(.*) wrote:"
+FUP_NAME: "(.*) wrote in"
+
+#FUP_GROUP
+# A Regular Expression, describing the string which contains the newsgroup
+# where the previous post, that you are referring to in the followup, had been
+# published.  
+# Leave this field empty to ignore the precise group.
+# CONTENT: A String equivalent of a regular expression.
+# DEFAULT: EMPTY
+# EXAMPLE: "wrote in (.*):"
+FUP_GROUP: 'wrote in (.*)'
+
+# GROUP_INTROS:
+# Introductory strings, referring to the previous poster who is the author of a
+# quoted post. If you match the newsgroup of the post (see FUP_GROUP), you can 
+# use these variables in the result. 
+# Currently only %fup_name% and %fup_group% are reproduced in the resulting 
+# introductory string.
+# CONTENT: A newsgroup or regexp, followed by a colon, a space and a String.
+# DEFAULT: As configured in FLNews
+# EXAMPLE: alt.test: "Thus spoke #{fup_name} on that baleful #{fup_date}:"
+GROUP_INTROS:
+  .*fr.test: "(%fup_name%) a écris :"
+  .*de.test: "%fup_name% hat in %fup_group% getestet:"
+  de\.*: "%fup_name% hat geschrieben:"
+  uk\.*: "%fup_name% wrote:"
+  fr\.*: "%fup_name% a écrit:"
+
+# GROUP_SIGS
+# A signature line per Newsgroup. 
+# ATTN! In multi line signatures, use \r\n for line breaks!
+# CONTENT: A newsgroup or regexp, followed by a colon, a space and a String.
+# DEFAULT: As configured in flnews
+# EXAMPLE: alt.test: "Signature for alt.test\r\nsecond line"
+GROUP_SIGS:
+  .*de.test: "newsgroup_hook .*de.test\r\nMit Zeilenumbruch"
+  .*fr.test: "newsgroup_hook .*fr.test\r\n2ème ligne, « guillemets »"
+  de.*: "„Erst betonieren, dann reinwerfen.”\r\n– „Erst Schulden machen, dann klugscheißen?”"
+  fr.*: "« Ton ordinateur de rêve ? » – « Le bleue. »\r\n "
+
+# CUSTOM_HEADERS
+# Additional headers for the outgoing article.
+# If you wish to insert an "X-Post-Processor" header, the current version
+# of the program  will be appended.
+# CONTENT: A dash and space, then a String, comprising the name of the header, ending in a 
+#          colon and the value of the header
+# DEFAULT: undefined
+# EXAMPLE (two headers):
+# - 'X-My-Header: nothing fancy'
+# - 'X-Post-Processor: flnews_post_proc'
+CUSTOM_HEADERS:
+- "X-Post-Processor: flnews_post_proc"
+
+# NO_ARCHIVE_GROUPS:
+# The newsgroups, where a header X-No-Archive: YES shall be set.
+# CONTENT: a dash and space, then a String, containing the name of the group
+#          or a regexp.
+# DEFAULT: empty
+# EXAMPLE: - "alt.test"
+NO_ARCHIVE_GROUPS:
+- ".*.test"
+
+# DEBUG_LOG:
+# The name of a file, where debug messages are written. Setting this
+# variable will enable the log. Leave empty to disable logging.
+# CONTENT: The name of a writable file, which will be overwritten.
+# DEFAULT: empty
+# EXAMPLE: '/tmp/a_log-file.txt'
+DEBUG_LOG: '/tmp/flnews_post_proc.log'
+
+# LOG LEVEL
+# One of debug, fatal, error, info, warn
+LOG_LEVEL: 'info'
+
+# REFERENCES_SEPARATOR 
+# A symbol or sequence of symbols marking the end of the message-body and the
+# beginning of a list of “references” or “footnotes”. It will only appear, if
+# the original message-body contains text marked for use as such a footnote. See
+# REFERENCES_DELIMITER.
+# If the option is not defined or empty, the list of footnotes will appear
+# below the last line of the message body and no separator will be inserted.
+# CONTENT: A quoted symbol or sequence of symbols.
+# DEFAULT: empty
+# EXAMPLE: '---------'
+# REFERENCES_SEPARATOR: "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
+REFERENCES_SEPARATOR: "──────────────────"
+
+# REFERENCE_DELIMITER
+# A symbol or sequence of symbols marking the beginning of a text which will
+# serve as footnote (or reference). The reversed sequence musst be used to mark
+# the end of the text. The presence of this sequence or symbol in the origimal
+# message body will cause the enclosed text to be moved below the message body.
+# The REFERENCES_SEPARATOR, if defined, will separate the message from the list
+# of footnotes.
+# If this option is not defined or empty, footnotes are not created.
+# CONTENT a quoted symbol or sequence of symbols.
+# DEFAULT: none/empty
+# EXAMPLE: '%?'
+REFERENCES_DELIMITER: "%="
+
+# REFERENCE_FORMAT
+# A format-string, using %s for a number, replacing the reference-
+# text in the message body. 
+# DEFAULT: " %s]"  -> becomes 1) ... 2)  ... 3) 
+# EXAMPLE: "(%s)"  -> becomes (1) ... (2)  ... (3) 
+REFERENCE_FORMAT: " ➤%s"
+
+# VFY_URLS
+# A Boolean constant. It determines if the program shall verify and possibly
+# try to correct URLs. Even if URLs are identified as such, only a few
+# manipulations are attempted :
+# * Angular brackets '<' and '>' are added, if missing
+# -----> handling <news:/> is disabled
+# * Article-references are prepended with "news:", if missing 
+# * Slashes are added, if they are found missing after "http(s):" 
+# ATTN! The program is unable to discern "mailto:" and "news:" references.  If
+# neither is given, but '@' is present, "news:" is automatically prepended.
+# <-----
+# If the variable is not set, a value 'yes' is assumed.
+# CONTENT: One of YES, yes, NO, no, and other variations of case.
+# DEFAULT: No
+# Example: ... I let you guess.
+VFY_URLS: No
+
+# OVERRIDE_CONFIG
+# A Boolean constant. You can choose to override the following
+# configuration options before an article is posted:
+# GROUP_SIGS, XNAY_GROUPS, CUSTOM_HEADERS and DEBUG_LOG.
+# A dialog may be displayed which allows you to disable any of these 
+# four options, so that the defaults from flnews prevail.
+# A fifth option (PP) can be used to disable post-processing completely.
+#
+# Set this option to no, NO or similar to disable the dialog.
+# DEFAULT: yes
+OVERRIDE_CONFIG: YES
+
+# EOF
+

data/lib/flnews_post_proc.rb

@@ -0,0 +1,72 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+require 'yaml'
+require 'ostruct'
+
+# --needs the Diffy gem
+require 'diffy'
+
+require_relative 'configuration'
+require_relative 'basic_logging'
+require_relative 'headers'
+require_relative 'body'
+
+# line-break in the final article
+$LN = "\r\n"
+
+# The main application class.
+#  Does it.
+class PostProcessor
+  include BasicLogging  
+
+  def initialize(article_text)
+    @config = Configuration.instance
+    # for simplicity.
+    # separate the headers and the body.
+    debug ' initializing headers'
+    headers = Headers.new(article_text)
+    debug('headers is ' << headers.inspect)
+    body = Body.new(article_text)
+
+    debug('calling headers.update')
+    headers.update()
+
+    newsgroups = headers.newsgroups
+
+    # Order matters. These actions work on a 
+    # preliminary version of the article, each 
+    # one on the result of the previous !
+    body.set_intro(newsgroups.intro)
+
+    # if need be, extract references and footnotes.
+    body.handle_references
+    body.set_signature(newsgroups.signature)
+
+    # verify and eventually correct URIs.
+    # Will only handle http(s) for the time.
+    body.handle_uris
+
+    # get the headers and the body as a string.
+    # Assemble.
+    @article = headers.join << $LN << body.join
+
+    diff = Diffy::Diff.new(article_text, @article, :context => 2).to_s
+    info("\n" << "–" * 20 << "\nDiffs\n" << "–" * 20 << "\n" << diff)  
+  end
+
+  attr_reader :article
+end
+# EOF

data/lib/headers.rb

@@ -0,0 +1,172 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+require_relative 'basic_logging'
+require_relative 'configuration'
+require_relative 'newsgroups'
+
+# an object of this class represents the headers of a news-article
+
+class Headers
+  include BasicLogging
+
+  # read the headers from the article
+  def initialize(article_text)
+    
+    @config = Configuration.instance
+    line = nil
+    # transform the article to an array.
+    debug('before split, article_text is : ' << article_text)
+    line_array = article_text.split($LN)
+
+    # Emsure that all three headers are present.
+    missing_header = ['From:', 'Newsgroups:', 'Message-ID:'].detect{|h| ! line_array.any?{|l| l.match(h) } } 
+    if(missing_header)
+      msg = "Input does not look like a news-article, no #{missing_header.delete(':')}; aborting."
+      STDERR.puts msg
+      error(msg)
+      exit false
+    end
+
+    debug('after split, line_array is : ' << line_array.inspect)
+    # find the first empty line
+    end_index = line_array.index {|ele| ele.strip == ''}
+    # keep the preceding lines.
+    begin
+      @lines = line_array.slice!(0, end_index)
+    rescue Exception => ex
+      msg = 'ERROR: cannot split the input into lines: ' << self.class.name << ': ' << ex.message
+      # console
+      STDERR.puts msg
+      # log
+      error(msg)
+      exit false
+    end
+
+    debug('headers: ' << @lines.to_s)
+
+    # headername: headervalue
+    @headers = {}
+
+    # fill the headers Hash from the header-lines.
+    # headers may have been line-wrapped. 
+
+    cur_header = nil
+    @lines.each do |l| 
+      # has the header been wrapped?
+      if !l.start_with?(/\s+/)
+        # header is all before the first colon
+        begin  
+          cur_header = l.match(/^(.*?):/)[1].to_sym
+        rescue Exception => ex
+          error ("Cannot match a header in line " << l << "(" << ex.message << ")")
+          exit false;
+        end
+        # Consider the two following fixes as preliminary until proven. 
+        # Getting older, this kind of problem occupies me a lot more than
+        # it should.
+        #
+        # value is all after the first colon
+        #  BUGGY: val = l.match(/:(.*)/)[1].strip
+        #  BUGFIX 3/2024, use lstrip
+        val = l.match(/:(.*)/)[1].lstrip
+      else # start_with?('  ')
+        # a wrapped value is not devided
+        # BUGGY: val = l.strip
+        # BUGFIX 3/2024
+        val = l
+      end
+      # add value to the existing 
+      if cur_header && @headers[cur_header]  
+        @headers[cur_header] += val
+      else
+        # or add a new value
+        @headers[cur_header] = val
+      end
+      #@headers[l.match(/^(.*?):/)[1].to_sym] = l.match(/:(.*)/)[1].strip
+      # h = l.split(':')
+      # @headers[h[0].strip.to_sym] = h[1...h.size].join(':').strip
+    end
+    debug('headers are ' << @headers.to_s)
+    @newsgroups = Newsgroups.new(header(:Newsgroups))
+    debug('Newsgroups is ' << @newsgroups.inspect)
+
+  end
+
+  # returns the value of header 'name'
+  def header(name)
+    # name must be a symbol.
+    if name.respond_to?(:to_sym)
+      @headers[name]     
+    else 
+      error(name.to_s << ' is not a symbol!')
+      nil
+    end
+  end
+
+  # Modify headers, if need be.
+  def update()
+    no_archive = @newsgroups.no_archive
+    debug('no_archive should be set now : ' << no_archive.to_s)
+    if no_archive 
+      @headers["X-No-Archive".to_sym] = no_archive
+      @headers["Archive".to_sym] = 'no'
+    end
+    if @config.CUSTOM_HEADERS    
+      ch = @config.CUSTOM_HEADERS
+      debug('setting custom headers : ' << ch.inspect)
+      @config.CUSTOM_HEADERS.each do |pair| 
+        ch = pair.split(':')
+        hn = ch[0].strip
+        hv = ch[1].strip
+        # Ensure header is ascii only
+        if hv.ascii_only? && hn.ascii_only?
+          # <---------- special treatment Post-Processor ---------->
+          hv << ' ' << PROGVERSION.to_s if hn == 'X-Post-Processor' && hv == 'flnews_post_proc'
+          #  >----------<
+          @headers[hn.to_sym] = hv
+        else
+          warn "Custom header [#{hn}:#{hv}] should be ASCII only! Header is ignored!"
+        end
+      end
+      @headers.compact!
+    end
+    debug('updated headers are ' << @headers.inspect)
+  end
+
+  # remove a header
+  def remove(name)
+    @headers.delete(name) if @headers[name]
+  end
+
+  # basically a replacement for header(name), above.
+  # But you can call self.Newsgroups or self.From etc.
+  def method_missing(method, args = nil)
+    return @headers[method] if @headers[method]
+    error("unknown symbol '#{method}'")
+  end
+
+  # return the headers as a String.
+  def join
+    htext = ''
+    @headers.each_pair {|h, t| htext << h.to_s << ': ' << t << $LN }
+    debug('joined headers: ' << htext)
+    htext
+  end
+
+  attr_reader :lines, :newsgroups
+
+end
+# EOF

data/lib/newsgroups.rb

@@ -0,0 +1,152 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+# An object of this class concentrates the specificities of chosen
+# newsgroups, as defined in the configuration.
+
+require_relative 'configuration'
+require_relative 'basic_logging'
+
+class Newsgroups
+
+  include BasicLogging  
+  def initialize(groups)
+    @config = Configuration.instance
+    @groups = groups.split(',')
+    debug('set signature, intro, no_archive')
+    # set details for this post
+    if @groups.size == 1
+      set_signature
+      set_intro 
+      set_no_archive 
+    end
+  end
+
+  def no_archive
+    debug('returning ' <<( @no_archive ? @no_archive : ' nil ') )
+    return @no_archive ? @no_archive : nil 
+  end
+  attr_reader :signature, :intro, :groups
+
+  private
+
+  # defines the intro-line as per group.
+  def set_intro
+
+    @intro = nil 
+    # only one group.
+    group = @groups[0]
+    # all configured intro-lines.
+    gintros = @config.GROUP_INTROS
+
+    if gintros && gintros.respond_to?(:to_hash)
+      # find the intro for the group.
+      # either by name
+      if gintros.keys.include?(group)
+        @intro = gintros[group] 
+      else
+        # or by a regular expression.
+        gintros.each do |gr, intro|
+          unless @intro
+            @intro = intro if group.match(gr)
+            debug "matched group against " << gr if @intro
+          end
+        end
+      end
+      debug('group_intro is ' << @intro.to_s) 
+    else
+      msg = 'Cannot set the introduction line from the configuration!'
+      msg << "\nPlease verify that GROUP_INTROS is set"
+      warn(msg)
+    end
+  end
+
+  # replace all \n by \r\n
+  def correct_linebreaks(text)
+    warned = false
+    # find all lonely \n
+    while text.match(/([^\r])\n/) do
+      warn("ATTN! Line-breaks should be \\r\\n! Verify signatures!" ) if !warned
+      warned ||= true
+      # ... and silently marry them to \r
+      text.gsub!($~[0],$~[1] + "\r\n")
+      text.gsub!("\n ", "\n")
+      # Luxury you can afford.
+    end # \n
+    text
+  end
+
+  # define the signature 
+  def set_signature
+    @signature = nil
+    # 1 group
+    group = @groups[0]
+    gsigs = @config.GROUP_SIGS
+
+    if gsigs && gsigs.respond_to?(:to_hash)
+      # find the signature for the group
+      # either by name
+      if gsigs.keys.include?(group)
+        @signature = gsigs[group] 
+        debug('signature is ' << @signature ) if @signature
+        # .., or by applying a regexp.
+      else
+        gsigs.each do |g, s|
+          unless @signature
+            rg = Regexp.new(g)
+            sm = group.match(rg)
+            debug('signature for group(s) ' << g << ': ' << s) if sm
+            if sm
+              @signature = s
+            end # if sm
+          end # if no signature
+        end # gsigs.each
+      end # gsigs for group?
+      if(@signature && @signature.strip.start_with?('/') )
+        debug('picking signature from file ' << @signature)
+        @signature = pick_sig(@signature)
+      end
+      @signature = correct_linebreaks(@signature) if @signature
+    else # gsigs and is hash?
+      msg = "Cannot read the signatures from the configuration."
+      msg << "\nPlease verify that GROUP_SIGS is set."
+      warn(msg)
+    end
+  end
+
+  # pick a signature from a file
+  def pick_sig(file)
+    if file && File.exist?(file) && File.readable?(file)
+      allSigs = File::read(file).force_encoding('utf-8').split("\n\n")
+      numSigs = allSigs.length
+
+      srand(Time.now.nsec)
+      return allSigs[rand(numSigs)]
+    else
+      error 'cannot read signature from file ' << file
+    end
+  end
+
+  # define the no_archive header.
+  def set_no_archive
+    @no_archive = nil
+    xgs = @config.NO_ARCHIVE_GROUPS
+    if xgs && !xgs.empty? && xgs.detect {|g| @groups[0].match(g) }
+      debug("setting no_archive")
+      @no_archive = 'yes' 
+    end
+  end
+end
+# EOF