sisu 7.0.0

data/lib/sisu/db_remove.rb

@@ -0,0 +1,182 @@
+# encoding: utf-8
+=begin
+
+* Name: SiSU
+
+** Description: documents, structuring, processing, publishing, search
+*** modules shared by the different db types, dbi, postgresql, sqlite
+
+** Author: Ralph Amissah
+  <ralph@amissah.com>
+  <ralph.amissah@gmail.com>
+
+** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
+  All Rights Reserved.
+
+** License: GPL 3 or later:
+
+  SiSU, a framework for document structuring, publishing and search
+
+  Copyright (C) Ralph Amissah
+
+  This program is free software: you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published by the Free
+  Software Foundation, either version 3 of the License, or (at your option)
+  any later version.
+
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program. If not, see <http://www.gnu.org/licenses/>.
+
+  If you have Internet connection, the latest version of the GPL should be
+  available at these locations:
+  <http://www.fsf.org/licensing/licenses/gpl.html>
+  <http://www.gnu.org/licenses/gpl.html>
+
+  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+** SiSU uses:
+  * Standard SiSU markup syntax,
+  * Standard SiSU meta-markup syntax, and the
+  * Standard SiSU object citation numbering and system
+
+** Hompages:
+  <http://www.jus.uio.no/sisu>
+  <http://www.sisudoc.org>
+
+** Git
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/db_remove.rb;hb=HEAD>
+
+=end
+module SiSU_DbRemove
+  class Remove
+    include SiSU_DbAction
+    def initialize(opt,conn,file,sql_type)
+      @opt,@conn,@file,@sql_type=opt,conn,file,sql_type
+      @md=SiSU_Param::Parameters.new(@opt).get
+      @fnb=@md.fnb
+      @db=SiSU_Env::InfoDb.new
+    end
+    def remove
+      driver_sqlite3=if @sql_type==:sqlite
+        (@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) \
+        ? true
+        : false
+      end
+      del_id=if driver_sqlite3
+        begin
+          remove_selected=%{
+            SELECT tid
+            FROM metadata_and_text
+            WHERE src_filename = '#{@md.fns}'
+            AND metadata_and_text.language_document_char = '#{@opt.lng}'
+          ;} # note, for .ssm: @md.fns (is set during runtime & is) != @opt.fns @md.opt.fns
+          @conn.get_first_value(remove_selected).to_i
+        rescue SQLite3::Exception => e
+          #not tested
+          puts "Exception occurred"
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).mark(e.inspect)
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:yellow).mark(
+            "\n" \
+            + 'Attempting to initialize db' + "\n" \
+            + 'Creating db tables'
+          )
+          sdb={
+            create: SiSU_DbDBI::Create.new(@opt,@conn,@file_maint,@sql_type),
+            index: SiSU_DbDBI::Index.new(@opt,@conn,@file_maint,@sql_type),
+          }
+          db_action(sdb).create
+        end
+      else
+        begin
+          remove_selected=%{
+            SELECT metadata_and_text.tid
+            FROM metadata_and_text
+            WHERE metadata_and_text.src_filename = '#{@md.fns}'
+            AND metadata_and_text.language_document_char = '#{@opt.lng}'
+          ;} # note, for .ssm: @md.fns (is set during runtime & is) != @opt.fns @md.opt.fns
+          x=@conn.exec(remove_selected)
+          x.field_values("tid")[0]
+        rescue PG::Error => e
+          err=[
+            e.result.error_field( PG::Result::PG_DIAG_SEVERITY ),
+            e.result.error_field( PG::Result::PG_DIAG_SQLSTATE ),
+            e.result.error_field( PG::Result::PG_DIAG_MESSAGE_PRIMARY ),
+            e.result.error_field( PG::Result::PG_DIAG_MESSAGE_DETAIL ),
+            e.result.error_field( PG::Result::PG_DIAG_MESSAGE_HINT ),
+            e.result.error_field( PG::Result::PG_DIAG_STATEMENT_POSITION ),
+            e.result.error_field( PG::Result::PG_DIAG_INTERNAL_POSITION ),
+            e.result.error_field( PG::Result::PG_DIAG_INTERNAL_QUERY ),
+            e.result.error_field( PG::Result::PG_DIAG_CONTEXT ),
+            e.result.error_field( PG::Result::PG_DIAG_SOURCE_FILE ),
+            e.result.error_field( PG::Result::PG_DIAG_SOURCE_LINE ),
+            e.result.error_field( PG::Result::PG_DIAG_SOURCE_FUNCTION ),
+          ]
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).mark("\n" + err.inspect)
+          if err[2] =~/relation "\S+?" does not exist/ \
+          or err.inspect =~/relation "\S+?" does not exist/
+            SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:yellow).mark(
+              "\n" \
+              + err[2] + "\n" \
+              + 'Attempting to initialize db' + "\n" \
+              + 'Creating db tables'
+            )
+            sdb={
+              create: SiSU_DbDBI::Create.new(@opt,@conn,@file_maint,@sql_type),
+              index: SiSU_DbDBI::Index.new(@opt,@conn,@file_maint,@sql_type),
+            }
+            db_action(sdb).create
+          end
+        end
+      end
+      if del_id
+        sql_entry=[
+          "DELETE FROM endnotes WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM endnotes_asterisk WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM endnotes_plus WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM doc_objects WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM urls WHERE metadata_tid = '#{del_id}';",
+          "DELETE FROM metadata_and_text WHERE metadata_and_text.tid = '#{del_id}';",
+        ]
+        if driver_sqlite3
+          @conn.transaction
+          sql_entry.each do |s|
+            begin
+              @conn.execute(s)
+            rescue
+              next
+            end
+          end
+          @conn.commit if driver_sqlite3
+        else
+          sql_entry.each do |s|
+            begin
+              @conn.exec_params(s)
+            rescue
+              next
+            end
+          end
+        end
+        if @opt.act[:maintenance][:set]==:on
+          @file.puts sql_entry
+        end
+      else
+        if (@opt.act[:verbose][:set]==:on \
+        || @opt.act[:verbose_plus][:set]==:on \
+        || @opt.act[:maintenance][:set]==:on)
+          SiSU_Screen::Ansi.new(
+            @opt.selections.str,
+            "no such file in database #{@db.psql.db}::#{@opt.fns}"
+          ).puts_grey
+        end
+      end
+    end
+  end
+end
+__END__

data/lib/sisu/db_select.rb

@@ -0,0 +1,230 @@
+# encoding: utf-8
+=begin
+
+* Name: SiSU
+
+** Description: documents, structuring, processing, publishing, search
+*** modules shared by the different db types, dbi, postgresql, sqlite
+
+** Author: Ralph Amissah
+  <ralph@amissah.com>
+  <ralph.amissah@gmail.com>
+
+** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
+  All Rights Reserved.
+
+** License: GPL 3 or later:
+
+  SiSU, a framework for document structuring, publishing and search
+
+  Copyright (C) Ralph Amissah
+
+  This program is free software: you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published by the Free
+  Software Foundation, either version 3 of the License, or (at your option)
+  any later version.
+
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program. If not, see <http://www.gnu.org/licenses/>.
+
+  If you have Internet connection, the latest version of the GPL should be
+  available at these locations:
+  <http://www.fsf.org/licensing/licenses/gpl.html>
+  <http://www.gnu.org/licenses/gpl.html>
+
+  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+** SiSU uses:
+  * Standard SiSU markup syntax,
+  * Standard SiSU meta-markup syntax, and the
+  * Standard SiSU object citation numbering and system
+
+** Hompages:
+  <http://www.jus.uio.no/sisu>
+  <http://www.sisudoc.org>
+
+** Git
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/db_select.rb;hb=HEAD>
+
+=end
+module SiSU_DbAction
+  def db_action(sdb)
+    @sdb=sdb
+    def createdb
+      @sdb[:create].output_dir?
+      begin
+        @sdb[:create].create_db
+      rescue
+        @sdb[:create].output_dir?
+      end
+    end
+    def drop
+      @sdb[:drop].drop.tables
+    end
+    def create
+      @sdb[:create].output_dir?
+      begin
+        @sdb[:create].create_table.metadata_and_text
+        @sdb[:create].create_table.doc_objects
+        @sdb[:create].create_table.endnotes
+        @sdb[:create].create_table.endnotes_asterisk
+        @sdb[:create].create_table.endnotes_plus
+        @sdb[:create].create_table.urls
+        @sdb[:index].create_indexes
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,'--sqlite').location
+        @sdb[:create].output_dir? do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      end
+    end
+    def import
+      db_exist?
+      @sdb[:import].marshal_load
+      tell=case @sql_type
+      when :sqlite
+        SiSU_Screen::Ansi.new(
+          @opt.act[:color_state][:set],
+          "sqlite3 #{@db.sqlite.db} database?"
+        )
+      when :pg
+        SiSU_Screen::Ansi.new(
+          @opt.act[:color_state][:set],
+          "pgaccess or psql #{@db.psql.db} database?"
+        )
+      else '???'
+      end
+      tell.puts_grey if @opt.act[:verbose][:set]==:on
+    end
+    def remove
+      db_exist?
+      @sdb[:remove_doc].remove
+    end
+    def update
+      remove
+      import
+    end
+    self
+  end
+end
+module SiSU_DbSelect
+  class Case
+    include SiSU_DbAction
+    def initialize(opt,conn='',sql_type=:pg)
+      @opt,@conn,@sql_type=opt,conn,sql_type
+      @db=SiSU_Env::InfoDb.new
+      @file_maint=sql_maintenance_file
+      @sdb={
+        create: SiSU_DbDBI::Create.new(@opt,@conn,@file_maint,@sql_type),
+        index: SiSU_DbDBI::Index.new(@opt,@conn,@file_maint,@sql_type),
+        drop: SiSU_DbDBI::Drop.new(@opt,@conn,@db,@sql_type),
+      }
+      if (@opt.act[:psql_import][:set]==:on \
+      || @opt.act[:psql_update][:set]==:on) \
+      or (@opt.act[:sqlite_import][:set]==:on \
+      || @opt.act[:sqlite_update][:set]==:on)
+        @sdb[:import]=SiSU_DbDBI::Import.new(@opt,@conn,@file_maint,@sql_type)
+        @sdb[:remove_doc]=SiSU_DbDBI::Remove.new(@opt,@conn,@file_maint,@sql_type)
+      elsif (@opt.act[:psql_remove][:set]==:on \
+      or @opt.act[:sqlite_remove][:set]==:on)
+        @sdb[:remove_doc]=SiSU_DbDBI::Remove.new(@opt,@conn,@file_maint,@sql_type)
+      end
+    end
+    def db_exist?
+      if @sql_type==:sqlite \
+      and (not (FileTest.file?(@db.sqlite.db)) \
+      or FileTest.zero?(@db.sqlite.db))
+        puts %{no connection with sqlite database established, you may need to run:\n} \
+        + %{    sisu --sqlite --createall\n} \
+        + %{  before attempting to populate the database}
+        SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:yellow).mark(
+          "\n" \
+          + 'Attempting to initialize db' + "\n" \
+          + 'Creating db tables'
+        )
+        db_action(@sdb).create
+      end
+      if @conn.is_a?(NilClass)
+        if @sql_type==:sqlite
+          puts %{no connection with sqlite database established, you may need to run:\n} \
+          + %{    sisu --sqlite --createall\n} \
+          + %{  before attempting to populate the database}
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:yellow).mark(
+            "\n" \
+            + 'Attempting to initialize db' + "\n" \
+            + 'Creating db tables'
+          )
+          db_action(@sdb).create
+          @db.sqlite.db
+        else
+          puts %{no connection with pg database established, you may need to run:\n} \
+          + %{    createdb "#{@db.psql.db}"\n} \
+          + %{  after that don't forget to run:\n} \
+          + %{    sisu --pg --createall\n} \
+          + %{  before attempting to populate the database}
+          @db.psql.db
+        end
+        exit
+      end
+    end
+    def sql_maintenance_file
+      file=if @opt.act[:maintenance][:set]==:on
+        if @opt.fns and not @opt.fns.empty?
+          @env=SiSU_Env::InfoEnv.new(@opt.fns) if @opt.fns
+          if @sql_type ==:sqlite
+            puts "\n#{@env.processing_path.sqlite}/#{@opt.fns}.sql"
+          end
+          @db=SiSU_Env::InfoDb.new
+          @job="sqlite3 #{@db.sqlite.db} < #{@env.processing_path.sqlite}/#{@opt.fns}.sql"
+          if @sql_type ==:sqlite
+            File.new("#{@env.processing_path.sqlite}/#{@opt.fns}.sql",'w+')
+          else
+            File.new("#{@env.processing_path.postgresql}/#{@opt.fns}.sql",'w+')
+          end
+        elsif @opt.fns \
+        and (@opt.act[:sqlite_create][:set] ==:on \
+        || @opt.act[:psql_create][:set] ==:on)
+          nil #sort variations later
+        else nil
+        end
+      else nil
+      end
+      file
+    end
+    def cases
+      if @opt.act[:psql_drop][:set] ==:on \
+      or @opt.act[:sqlite_drop][:set] ==:on
+        db_action(@sdb).drop
+      end
+      if @opt.act[:psql_createdb][:set] ==:on \
+      or @opt.act[:sqlite_createdb][:set] ==:on
+        db_action(@sdb).createdb
+      end
+      if @opt.act[:psql_create][:set] ==:on \
+      or @opt.act[:sqlite_create][:set] ==:on
+        db_action(@sdb).create
+      end
+      if @opt.act[:psql_update][:set] ==:on \
+      or @opt.act[:sqlite_update][:set] ==:on
+        db_action(@sdb).update
+      else
+        if @opt.act[:psql_remove][:set] ==:on \
+        or @opt.act[:sqlite_remove][:set] ==:on
+          db_action(@sdb).remove
+        end
+        if @opt.act[:psql_import][:set] ==:on \
+        or @opt.act[:sqlite_import][:set] ==:on
+          db_action(@sdb).import
+        end
+      end
+    end
+  end
+end
+__END__

data/lib/sisu/db_sqltxt.rb

@@ -0,0 +1,173 @@
+# encoding: utf-8
+=begin
+
+* Name: SiSU
+
+** Description: documents, structuring, processing, publishing, search
+*** system environment, resource control and configuration details
+
+** Author: Ralph Amissah
+  <ralph@amissah.com>
+  <ralph.amissah@gmail.com>
+
+** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
+  All Rights Reserved.
+
+** License: GPL 3 or later:
+
+  SiSU, a framework for document structuring, publishing and search
+
+  Copyright (C) Ralph Amissah
+
+  This program is free software: you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published by the Free
+  Software Foundation, either version 3 of the License, or (at your option)
+  any later version.
+
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program. If not, see <http://www.gnu.org/licenses/>.
+
+  If you have Internet connection, the latest version of the GPL should be
+  available at these locations:
+  <http://www.fsf.org/licensing/licenses/gpl.html>
+  <http://www.gnu.org/licenses/gpl.html>
+
+  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+** SiSU uses:
+  * Standard SiSU markup syntax,
+  * Standard SiSU meta-markup syntax, and the
+  * Standard SiSU object citation numbering and system
+
+** Hompages:
+  <http://www.jus.uio.no/sisu>
+  <http://www.sisudoc.org>
+
+** Git
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/db_sqltxt.rb;hb=HEAD>
+
+=end
+module SiSU_DbText
+  class Prepare
+    def special_character_escape(str)
+      str=str.gsub(/'/m,"''"). #string.gsub!(/'/,"\047") #string.gsub!(/'/,"\\'")
+        gsub(/(\\)/m,'\1\1'). #ok but with warnings, double backslash on sqlite #str.gsub!(/[\\]/m,'\\x5C') #ok but with warnings, but not for sqlite #str.gsub!(/(\\)/m,'\1') #ok for sqlite not for pgsql
+        gsub(/#{Mx[:br_line]}|#{Mx[:br_nl]}/m,"<br>\n").
+        gsub(/#{Mx[:tag_o]}\S+?#{Mx[:tag_c]}/m,''). #check
+        gsub(/#{Mx[:lnk_o]}\s*(\S+?\.(?:png|jpg))(?:\s+\d+x\d+)?(.+?)#{Mx[:lnk_c]}\S+/m,'[image: \1] \2').
+        gsub(/#{Mx[:lnk_o]}\s*(.+?)\s*#{Mx[:lnk_c]}(?:file|ftp):\/\/\S+?([.,!?]?(?:\s|$))/m,'\1\2').
+        gsub(/#{Mx[:lnk_o]}\s*(.+?)\s*#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/m,'\1')
+    end
+    def clean_searchable_text_from_document_objects(arr)
+      en=[]
+      arr=(arr.is_a?(String)) ? [ arr ] : arr
+      txt_arr=arr.each.map do |s|
+        s=s.gsub(/#{Mx[:fa_o]}[a-z]{1,4}#{Mx[:fa_o_c]}/m,'').
+            gsub(/#{Mx[:fa_c_o]}[a-z]{1,4}#{Mx[:fa_c]}/m,'').
+            gsub(/<br>/m,' ')
+        en << s.scan(/#{Mx[:en_a_o]}\s*(.+?)\s*#{Mx[:en_a_c]}/m)
+        s=s.gsub(/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/m,'').
+          gsub(/#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]}/m,'').
+          gsub(/ \s+/m,' ')
+        #p s if s =~/[^ \nA-Za-z0-9'"`?!#@$%^&*=+,.;:\[\]()<>{}‹›|\\\/~_-]/
+        s
+      end
+      txt_arr=txt_arr << en
+      txt=txt_arr.flatten.join("\n")
+      special_character_escape(txt)
+    end
+    def clean_document_objects_body(arr)
+      en=[]
+      arr=(arr.is_a?(String)) ? [ arr ] : arr
+      txt_arr=arr.each.map do |s|
+        en << s.scan(/#{Mx[:en_a_o]}\s*(.+?)\s*#{Mx[:en_a_c]}/m)
+        s=s.
+          gsub(/#{Mx[:en_a_o]}\s*(\d+).+?#{Mx[:en_a_c]}/m,
+            '<sup>\1</sup>').
+          gsub(/#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]}/m,'').
+          gsub(/ \s+/m,' ')
+        s
+      end
+      en_arr=en.flatten.each.map do |e|
+        e.sub(/^(\d+)\s*/,'<sup>\1</sup> ')
+      end
+      txt_arr=txt_arr << en_arr
+      txt=txt_arr.flatten.join("\n<br>")
+      special_character_escape(txt)
+    end
+    def clean_searchable_text_from_document_source(arr)
+      txt_arr,en=[],[]
+      arr=(arr.is_a?(String)) ? arr.split(/\n+/m) : arr
+      arr.each do |s|
+        s=s.gsub(/([*\/_-])\{(.+?)\}\1/m,'\2').
+          gsub(/^(?:block|group|poem|code)\{/m,'').
+          gsub(/^\}(?:block|group|poem|code)/m,'').
+          gsub(/\A(?:@\S+:\s+.+)\Z/m,'')
+        if s =~/^:A~/
+          if defined? @md.creator \
+          and defined? @md.creator.author \
+          and not @md.creator.author.empty?
+            s=s.gsub(/@author/,@md.creator.author)
+          else
+            SiSU_Screen::Ansi.new(
+              'v',
+              'WARNING Document Author information missing; provide @creator: :author:',
+              @md.fnb
+            ).warn unless @md.opt.act[:quiet][:set]==:on
+          end
+          if defined? @md.title \
+          and defined? @md.title.full \
+          and not @md.title.full.empty?
+            s=s.gsub(/@title/,@md.title.full)
+          else
+            SiSU_Screen::Ansi.new(
+              'v',
+              'WARNING Document Title missing; provide @title:',
+              @md.fnb
+            ).warn unless @md.opt.act[:quiet][:set]==:on
+          end
+        end
+        s=s.gsub(/^(?:_[1-9]\*?|_\*)\s+/m,'').
+          gsub(/^(?:[1-9]\~(\S+)?)\s+/m,'').
+          gsub(/^(?::?[A-C]\~(\S+)?)\s+/m,'').
+          gsub(/^%{1,3} .+/m,''). #removed even if contained in code block
+          gsub(/<br>/m,' ')
+        #en << s.scan(/~\{\s*(.+?)\s*\}~/m)
+        s=s.gsub(/~\{.+?\}~/m,'').
+          gsub(/ \s+/m,' ')
+        ##special_character_escape(s)
+        #p s if s =~/[^ \nA-Za-z0-9'"`?!#@$%^&*=+,.;:\[\]()<>{}‹›|\\\/~_-]/
+        s
+      end
+      txt_arr << arr << en
+      txt=txt_arr.flatten.join("\n")
+      txt=special_character_escape(txt)
+      txt
+    end
+    def strip_markup(str) #define rules, make same as in dal clean
+      str=str.gsub(/#{Mx[:fa_superscript_o]}(\d+)#{Mx[:fa_superscript_c]}/,'[\1]').
+        gsub(/(?:&nbsp\\;|#{Mx[:nbsp]})+/,' ').
+        gsub(/#{Mx[:tc_o]}#{Mx[:tc_p]}#{Mx[:tc_p]}\d+(.+)#{Mx[:tc_c]}/u,'\1').         #tables
+        gsub(/#{Mx[:tc_p]}#{Mx[:tc_p]}\d+#{Mx[:tc_p]}/u,' ').                          #tables
+        gsub(/#{Mx[:tc_p]}/u,' ').                                                     #tables tidy later
+        gsub(/<.+?>/,'').
+        gsub(/#{Mx[:lnk_o]}.+?\.(?:png|jpg|gif).+?#{Mx[:lnk_c]}(?:file|ftp)\/\/:\S+ /,' [image] '). # else image names found in search
+        gsub(/#{Mx[:lnk_o]}.+?\.(?:png|jpg|gif).+?#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/,' [image]'). # else image names found in search
+        gsub(/\s\s+/,' ').
+        strip
+    end
+    def unique_words(str)
+      a=str.scan(/[a-zA-Z0-9\\\/_-]{2,}/) #a=str.scan(/\S+{2,}/)
+      str=a.uniq.sort.join(' ')
+      str
+    end
+  end
+end
+__END__