flnews_post_proc 1.40

data/lib/flnews_post_proc.rb

@@ -0,0 +1,71 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+require 'yaml'
+require 'ostruct'
+# --needs the Diffy gem
+require 'diffy'
+
+require_relative 'configuration'
+require_relative 'basic_logging'
+require_relative 'headers'
+require_relative 'body'
+
+# line-break in the final article
+$LN = "\r\n"
+
+# The main application class.
+#  Does it.
+class PostProcessor
+  # class-level configuration object
+  @@config = Configuration.instance
+  include BasicLogging  
+
+  def initialize(article_text)
+    # for simplicity.
+    # separate the headers and the body.
+    debug ' initializing headers'
+    headers = Headers.new(article_text)
+    debug('headers is ' << headers.inspect)
+    body = Body.new(article_text)
+
+    debug('calling headers.update')
+    headers.update()
+
+    newsgroups = headers.newsgroups
+
+    # Order matters. These actions work on a 
+    # preliminary version of the article, each 
+    # one on the result of the previous !
+    body.set_intro(newsgroups.intro)
+
+    # if need be, extract references and footnotes.
+    body.handle_references
+    body.set_signature(newsgroups.signature)
+
+    # verify and eventually correct URLs
+    body.handle_urls
+
+    # get the headers and the body as a string.
+    # Assemble.
+    @article = headers.join << $LN << body.join
+
+    diff = Diffy::Diff.new(article_text, @article, :context => 2).to_s
+    info("\n" << "–" * 20 << "\nDiffs\n" << "–" * 20 << "\n" << diff)  
+  end
+
+  attr_reader :article
+end
+# EOF

data/lib/headers.rb

@@ -0,0 +1,148 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+require_relative 'basic_logging'
+require_relative 'configuration'
+require_relative 'newsgroups'
+
+# an object of this class represents the headers of a news-article
+
+class Headers
+  # class-level configuration object
+  @@config = Configuration.instance
+  include BasicLogging
+
+  # read the headers from the article
+  def initialize(article_text)
+    
+    line = nil
+    # transform the article to an array.
+    debug('before split, article_text is : ' << article_text)
+    line_array = article_text.split($LN)
+    debug('after split, line_array is : ' << line_array.inspect)
+    # find the first empty line
+    end_index = line_array.index {|ele| ele.strip == ''}
+    # keep the preceding lines.
+    @lines = line_array.slice!(0, end_index)
+    debug('headers: ' << @lines.to_s)
+
+    # headername: headervalue
+    @headers = {}
+
+    # fill the headers Hash from the header-lines.
+    # headers may have been line-wrapped. 
+
+    cur_header = nil
+    @lines.each do |l| 
+      # has the header been wrapped?
+      if !l.start_with?(/\s+/)
+        # header is all before the first colon
+        begin  
+          cur_header = l.match(/^(.*?):/)[1].to_sym
+        rescue Exception => ex
+          error ("Cannot match a header in line " << l << "(" << ex.message << ")")
+          exit false;
+        end
+        # value is all after the first colon
+        #  BUGGY: val = l.match(/:(.*)/)[1].strip
+        #  BUGFIX 3/2024
+        val = l.match(/:(.*)/)[1].lstrip
+      else # start_with?('  ')
+        # a wrapped value is not devided
+        # BUGGY: val = l.strip
+        # BUGFIX 3/2024
+        val = l
+      end
+      # add value to the existing 
+      if cur_header && @headers[cur_header]  
+        @headers[cur_header] += val
+      else
+        # or add a new value
+        @headers[cur_header] = val
+      end
+      #@headers[l.match(/^(.*?):/)[1].to_sym] = l.match(/:(.*)/)[1].strip
+      # h = l.split(':')
+      # @headers[h[0].strip.to_sym] = h[1...h.size].join(':').strip
+    end
+    debug('headers are ' << @headers.to_s)
+    @newsgroups = Newsgroups.new(header(:Newsgroups))
+    debug('Newsgroups is ' << @newsgroups.inspect)
+
+  end
+
+  # returns the value of header 'name'
+  def header(name)
+    # name must be a symbol.
+    if name.respond_to?(:to_sym)
+      @headers[name]     
+    else 
+      error(name.to_s << ' is not a symbol!')
+      nil
+    end
+  end
+
+  # Modify headers, if need be.
+  def update()
+    xnay = @newsgroups.xnay
+    debug('xnay should be set now : ' << xnay.to_s)
+    if xnay 
+      @headers["X-No-Archive".to_sym] = xnay
+    end
+    ch = @@config.CUSTOM_HEADERS
+    debug('setting custom headers : ' << ch.inspect)
+    if @@config.CUSTOM_HEADERS    
+      @@config.CUSTOM_HEADERS.each do |pair| 
+        ch = pair.split(':')
+        hn = ch[0].strip
+        hv = ch[1].strip
+        # Ensure header is ascii only
+        if hv.ascii_only? && hn.ascii_only?
+          # <---------- special treatment Post-Processor ---------->
+          hv << ' ' << PROGVERSION.to_s if hn == 'X-Post-Processor' && hv == 'flnews_post_proc'
+          #  >----------<
+          @headers[hn.to_sym] = hv
+        else
+          warn "Custom header [#{hn}:#{hv}] should be ASCII only! Header is ignored!"
+        end
+      end
+      @headers.compact!
+    end
+    debug('updated headers are ' << @headers.inspect)
+  end
+
+  # remove a header
+  def remove(name)
+    @headers.delete(name) if @headers[name]
+  end
+
+  # basically a replacement for header(name), above.
+  # But you can call self.Newsgroups or self.From etc.
+  def method_missing(method, args = nil)
+    return @headers[method] if @headers[method]
+    error("unknown symbol '#{method}'")
+  end
+
+  # return the headers as a String.
+  def join
+    htext = ''
+    @headers.each_pair {|h, t| htext << h.to_s << ': ' << t << $LN }
+    debug('joined headers: ' << htext)
+    htext
+  end
+
+  attr_reader :lines, :newsgroups
+  
+end
+# EOF

data/lib/newsgroups.rb

@@ -0,0 +1,134 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+# An object of this class concentrates the specificities of chosen
+# newsgroups, as defined in the configuration.
+
+require_relative 'configuration'
+require_relative 'basic_logging'
+
+class Newsgroups
+  # class-level configuration object
+  @@config = Configuration.instance
+  include BasicLogging  
+
+  def initialize(groups)
+    @groups = groups.split(',')
+    debug('set signature, intro, xnay')
+    # set details for this post
+    if @groups.size == 1
+      set_signature
+      set_intro 
+      set_xnay 
+    end
+  end
+
+  def xnay
+    debug('returning ' <<( @xnay ? @xnay : ' nil ') )
+    return @xnay ? @xnay : nil 
+  end
+  attr_reader :signature, :intro, :groups
+
+  private
+
+  # defines the intro-line as per group.
+  def set_intro
+
+    @intro = nil 
+    # only one group.
+    group = @groups[0]
+    # all configured intro-lines.
+    gintros = @@config.GROUP_INTROS
+
+    if gintros && gintros.respond_to?(:to_hash)
+      # find the intro for the group.
+      # either by name
+      if gintros.keys.include?(group)
+        @intro = gintros[group] 
+      else
+        # or by a regular expression.
+        gintros.each do |gr, intro|
+          unless @intro
+            @intro = intro if group.match(gr)
+            debug "matched group against " << gr if @intro
+          end
+        end
+      end
+      debug('group_intro is ' << @intro.to_s) 
+    else
+      msg = 'Cannot set the introduction line from the configuration!'
+      msg << "\nPlease verify that GROUP_INTROS is set"
+      warn(msg)
+    end
+  end
+
+  # replace all \n by \r\n
+  def correct_linebreaks(text)
+    warned = false
+    # find all lonely \n
+    while text.match(/([^\r])\n/) do
+      warn("ATTN! Line-breaks should be \\r\\n! Verify signatures!" ) if !warned
+      warned ||= true
+      # ... and silently marry them to \r
+      text.gsub!($~[0],$~[1] + "\r\n")
+      # Luxury you can afford.
+    end # \n
+    text
+  end
+
+  # define the signature 
+  def set_signature
+    @signature = nil
+    # 1 group
+    group = @groups[0]
+    gsigs = @@config.GROUP_SIGS
+
+    if gsigs && gsigs.respond_to?(:to_hash)
+      # find the signature for the group
+      # either by name
+      if gsigs.keys.include?(group)
+        @signature = gsigs[group] 
+        debug('signature is ' << @signature ) if @signature
+        # .., or by applying a regexp.
+      else
+        gsigs.each do |g, s|
+          unless @signature
+            rg = Regexp.new(g)
+            sm = group.match(rg)
+            debug('signature for group(s) ' << g << ': ' << s) if sm
+            if sm
+              @signature = correct_linebreaks(s)
+            end # if sm
+          end # if no signature
+        end # gsigs.each
+      end # gsigs for group?
+    else # gsigs and is hash?
+      msg = "Cannot read the signatures from the configuration."
+      msg << "\nPlease verify that GROUP_SIGS is set."
+      warn(msg)
+    end
+  end
+
+  # define the xnay header.
+  def set_xnay
+    @xnay = nil
+    xgs = @@config.XNAY_GROUPS
+    if xgs && !xgs.empty? && xgs.detect {|g| @groups[0].match(g) }
+      debug("setting XNAY")
+      @xnay = 'YES' 
+    end
+  end
+end
+# EOF

data/lib/override.rb

@@ -0,0 +1,199 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   2023-2024, Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+# This class manages a dialog to alter the behavior of the
+# post-processor on a “per article” basis. It can be displayed
+# before posting, to do some “last minute” changes,
+# notably to deactivate them.
+#
+# Post-processing CANNOT be deactivated completely in this dialog.
+# I do not remember, why a previous comment stated the contrary.
+
+require_relative 'basic_logging'
+
+class OverrideDlg
+
+  include BasicLogging
+
+  XTERM    = 'xterm'
+  # TERMINAL = 'x-terminal-emulator'
+  WHIPTAIL = 'whiptail'
+  YAD      = 'yad'
+  ZENITY   = 'zenity'
+  # The external programs which are supported for the time. They will be run in
+  # a new process (IO.popen() ).  The first program from this array, which is
+  # found in the Path, will be used. Whiptail and a pure ruby dialog
+  # necessitate that xterm be found, too.
+  @@Executables = [YAD, ZENITY, WHIPTAIL, XTERM]
+  @@LIBDIR = File::dirname(__FILE__)
+  # The configuration variables that can be unset.
+  # This class instance variable is exposed via a getter.
+  @cvars = [:GROUP_SIGS, :CUSTOM_HEADERS, :XNAY_GROUPS, :VFY_URLS, :DEBUG_LOG]
+
+  # ... here
+  # For the record: this is rather cool.
+  #      ... believe me! It is!
+  class << self
+    attr_reader :cvars
+  end
+  # Love it.
+
+  # Create the object
+  def initialize
+    @config = Configuration::instance
+    # find the supported programs 
+    @executables = @@Executables.select do |ex_name|
+      program?(ex_name) 
+    end
+    # As we are calling external programs anyway, 
+    # why not continue with that ...
+    @xmsg = ENV['PATH'].split(':').any? {|d| Dir.children(d).include?('xmessage')}
+    # bof.
+  end
+
+  # display a dialog and return the new options.
+  def show
+    if(@executables && !@executables.empty?)
+      debug('found executables ' << @executables.join(', ') )
+      opts = nil
+      begin
+        if has?(YAD)
+          return yad_dlg.split.collect {|o| o.to_s}.join(' ')
+        elsif has?(ZENITY)
+          return zenity_dlg.split.collect {|o| o.to_s}.join(' ')
+        elsif has? XTERM
+          if has?(WHIPTAIL)
+            return whiptail_dlg.split.collect {|o| o.to_s}.join(' ')
+          else
+            debug 'using naked xterm'
+            return ruby_dlg.split.collect {|o| o.to_s}.join(' ')
+          end
+        end
+      rescue SystemExit
+        msg = 'Process is cancelled (SystemExit)'
+        info msg
+        exit true
+      rescue Exception => ex
+        msg = 'Error upon showing dialog ' << ex.message
+        error(msg)
+      end
+    # no program to show the dialog 
+    else 
+      msg = "#{File.basename($0)}: No suitable executable found to display the dialog"
+      warn msg
+      # if xmessage is available, give the user a last chance.
+      if @xmsg
+        io = IO.popen('xmessage -buttons continue:0,abort:1 -default abort -print ' << msg) 
+        Process.wait(io.pid)
+        # ATTN! read io only once!
+        exit if ('continue' != io.read.to_s.strip )
+        # ... but then again, we still have a log.
+        debug('continue with configured settings')
+      end
+    end
+  end
+
+  private
+  # I know what it does and that there is some
+  # redundancy with the following method, but I
+  # do not feel like sorting it out, now.
+  def program?(ex_name)
+    ENV['PATH'].split(':').each do |dir_name| 
+      return true if Dir.entries(dir_name).include?(ex_name) 
+    end
+    return false
+  end
+
+  # returns true if the executable ex_name is to be 
+  # called for the dialog
+  def has?(ex_name)
+    @executables.include? ex_name 
+  end
+
+  ###### The following methods call a script, each, in an external process.
+  # There appears to be some Exception handling missing, but it does not
+  # hurt for the time being. Anyway, I do not care.
+
+  # creates a whiptail dialog in xterm.
+  def whiptail_dlg
+    debug('whiptail dialog')
+    tf = Tempfile.new
+    # dlg = @config.BINDIR << File::Separator << 'whiptail_dlg'
+    dlg = @@LIBDIR << File::Separator << 'whiptail_dlg'
+    dialog = XTERM.dup << ' -e ' << dlg << ' ' << tf.path
+    io = IO::popen(dialog)
+    Process.wait(io.pid )
+    nopts = tf.read
+    exit true if('exit' == nopts.strip)
+    tf.unlink
+    return nopts 
+  end
+
+  # creates a textual dialog in xterm.
+  def ruby_dlg
+    debug('xterm dialog')
+    tf = Tempfile.new
+    # dlg = @config.BINDIR << File::Separator << 'ruby_dlg'
+    dlg = @@LIBDIR << File::Separator << 'ruby_dlg'
+    begin
+      dialog = XTERM.dup  << ' -e ' << dlg << ' ' << tf.path
+      debug('dialog will be ' << dialog)
+    rescue Exception => ex
+      error ex.message
+      wait_for_user
+      exit false
+    end
+    io = IO::popen(dialog)
+    debug('wait for ' << io.pid.to_s)
+    Process.wait(io.pid )
+    nopts = tf.read
+    exit true if('exit' == nopts.strip)
+    tf.unlink
+    return nopts 
+  end
+
+  # creates a Zenity dialog.
+  def zenity_dlg
+    debug('Zenity dialog')
+    # dialog = @config.BINDIR << File::Separator << 'zenity_dlg'
+    dialog = @@LIBDIR << File::Separator << 'zenity_dlg'
+    io = IO.popen(dialog)
+    Process::wait(io.pid)
+    rc = $?.exitstatus 
+
+    exit true if rc != 0
+
+    nopts = io.read
+    return nopts
+  end
+
+  # creates a YAD dialog.
+  def yad_dlg
+    debug('yad dialog')
+    # dialog = @config.BINDIR << File::Separator << 'yad_dlg'
+    dialog = @@LIBDIR << File::Separator << 'yad_dlg'
+    debug('dialog will be ' << dialog)
+    io = IO::popen(dialog)
+    Process::wait(io.pid)
+    exit true if $?.exitstatus != 0
+
+    nopts = io.read
+    return nopts
+  end
+
+end 
+
+# EOF 
+

data/lib/ruby_dlg

@@ -0,0 +1,136 @@
+#!/usr/bin/env ruby
+
+=begin
+/***************************************************************************
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the WTFPL 2.0 or later, see                     * 
+ *   http://www.wtfpl.net/about/                                           *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
+ *                                                                         *
+ ***************************************************************************/
+=end
+
+require 'readline'
+require_relative '../lib/color_output'
+require 'io/wait'
+require 'io/console'
+
+def wait_for_user()
+  char = nil
+  # There is a difference which makes me prefer the below code from STDIN.raw(args)
+  # You can try
+  # char = STDIN.raw(&:getc)	
+  # .., which does *not* work the same way and it won't for me.
+
+  STDIN.raw do 
+    STDIN.noecho do
+      until (STDIN.ready?)
+        sleep(0.1)
+      end
+      char = (STDIN.read_nonblock(1).ord rescue nil)
+    end
+  end
+  return char
+end
+
+
+ofl = nil
+begin
+  ofl = File.open(ARGV[0], 'w+') if ARGV.length > 0
+  puts 'writing to ' << ARGV[0]
+rescue Exception => ex
+  puts red ('cannot open ' << ARGV[1])
+  puts 'hit any key to exit'
+  wait_for_user
+  exit false 
+end
+
+
+
+opt_array = []
+message = ''
+menu ||= %=
+  1 Unset Signature         GROUP_SIGS 
+  2 Unset Custom headers    CUSTOM_HEADERS
+  3 Unset X-No-Archive      XNAY_GROUPS
+  4 Do not correct URLs     VFY_URLS
+  5 Disable log             DEBUG_LOG
+  ––––––––––––––––––––––––––––––––––––––––––––––
+  6 Summary
+  ––––––––––––––––––––––––––––––––––––––––––––––
+  0 Okay, use settings. 
+––––––––––––––––––––––––––––––––––––––––––––––––
+#{bold("Esc, Ctrl+C and 'q'")} terminate the Post-processor 
+and no changes will be applied.
+=
+
+loop do
+  system 'clear'
+  puts yellow(menu)
+  puts cyan(message)
+  option = wait_for_user - 48
+  case option
+    # Esc, Ctrl+C, 'q'
+  when -21, -45, 65
+    puts red('Aborting. Bye.')
+    ofl.write 'exit'
+    ofl.close
+    exit true
+  when 0
+    puts green('Applying changes.') if !opt_array.empty?
+    # write the list of variables to unset (others remain)
+    ofl.write opt_array.join(' ')
+    ofl.close
+    exit true
+
+# toggle options
+# ADD option to the option array, if it should be *removed* 
+    # from the configuration, else remove it from the array.
+  when 1
+    active = !opt_array.include?(:GROUP_SIGS)
+    opt_array << :GROUP_SIGS if active
+    opt_array.delete(:GROUP_SIGS) if !active
+
+    message = "Signature #{active ? 'unset' : 'as configured'}!"
+    menu.sub!("Unset Signature", "Set Signature  ") if active
+    menu.sub!("Set Signature  ", "Unset Signature") if !active
+  when 2
+    active = !opt_array.include?(:CUSTOM_HEADERS)
+    opt_array << :CUSTOM_HEADERS if active
+    opt_array.delete(:CUSTOM_HEADERS) if !active
+
+    message = "Custom headers #{active ? 'removed' : 'are added'}!"
+    menu.sub!("Unset Custom headers", "Add Custom headers  ") if active
+    menu.sub!("Add Custom headers  ", "Unset Custom headers") if !active
+  when 3
+    active = !opt_array.include?(:XNAY_GROUPS)
+    opt_array << :XNAY_GROUPS if active
+    opt_array.delete(:XNAY_GROUPS) if !active
+
+    message = "X-No-Archive #{active ? 'removed' : 'is added'}!"
+    menu.sub!("Unset X-No-Archive", "Add X-No-Archive  ") if active
+    menu.sub!("Add X-No-Archive  ", "Unset X-No-Archive") if !active
+  when 4
+    active = !opt_array.include?(:VFY_URLS)
+    opt_array << :VFY_URLS if active
+    opt_array.delete(:VFY_URLS) if !active
+
+    message = "URLS will #{active ? 'not ' : ''}" <<  "be verified!"
+    menu.sub!("Do not correct URLs", "Correct URLs       ") if active
+    menu.sub!("Correct URLs       ", "Do not correct URLs") if !active
+  when 5
+    active = !opt_array.include?(:DEBUG_LOG)
+    opt_array << :DEBUG_LOG if active
+    opt_array.delete(:DEBUG_LOG) if !active
+
+    message = "Log is #{ active ? 'not ' : ''} " << "written!"
+    menu.sub!("Disable log", "Enable log ") if active
+    menu.sub!("Enable log ", "Disable log") if !active
+  when 6
+    message = "Summary of " << bold('disabled') << " options: " << opt_array.join(' ')
+  end
+end
+