sisu 7.0.0

data/lib/sisu/db_dbi.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+=begin
+
+* Name: SiSU
+
+** Description: documents, structuring, processing, publishing, search
+*** modules shared by the different db types, dbi, postgresql, sqlite
+
+** Author: Ralph Amissah
+  <ralph@amissah.com>
+  <ralph.amissah@gmail.com>
+
+** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
+  All Rights Reserved.
+
+** License: GPL 3 or later:
+
+  SiSU, a framework for document structuring, publishing and search
+
+  Copyright (C) Ralph Amissah
+
+  This program is free software: you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published by the Free
+  Software Foundation, either version 3 of the License, or (at your option)
+  any later version.
+
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program. If not, see <http://www.gnu.org/licenses/>.
+
+  If you have Internet connection, the latest version of the GPL should be
+  available at these locations:
+  <http://www.fsf.org/licensing/licenses/gpl.html>
+  <http://www.gnu.org/licenses/gpl.html>
+
+  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+** SiSU uses:
+  * Standard SiSU markup syntax,
+  * Standard SiSU meta-markup syntax, and the
+  * Standard SiSU object citation numbering and system
+
+** Hompages:
+  <http://www.jus.uio.no/sisu>
+  <http://www.sisudoc.org>
+
+** Git
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/db_dbi.rb;hb=HEAD>
+
+=end
+module  SiSU_DbDBI
+  require_relative 'db_columns'                                             # db_columns.rb
+  require_relative 'db_tests'                                               # db_tests.rb
+  require_relative 'db_create'                                              # db_create.rb
+  require_relative 'db_select'                                              # db_select.rb
+  require_relative 'db_indexes'                                             # db_indexes.rb
+  require_relative 'db_drop'                                                # db_drop.rb
+  require_relative 'db_remove'                                              # db_remove.rb
+  require_relative 'db_load_tuple'                                          # db_load_tuple.rb
+  require_relative 'db_import'                                              # db_import.rb
+  class ColumnSize < SiSU_DbColumns::ColumnSize                             # db_columns.rb
+  end
+  class Test < SiSU_DbTests::Test                                           # db_tests.rb
+  end
+  class Create <SiSU_DbCreate::Create                                       # db_create.rb
+  end
+  class Case <SiSU_DbSelect::Case                                           # db_select.rb
+  end
+  class Index <SiSU_DbIndex::Index                                          # db_indexes.rb
+  end
+  class Drop <SiSU_DbDrop::Drop                                             # db_drop.rb
+  end
+  class Remove <SiSU_DbRemove::Remove                                       # db_remove.rb
+  end
+  class LoadDocuments <SiSU_DbTuple::LoadDocuments                          # db_load_tuple.rb
+  end
+  class LoadMetadata <SiSU_DbTuple::LoadMetadata                            # db_load_tuple.rb
+  end
+  class LoadUrls <SiSU_DbTuple::LoadUrls                                    # db_update urls
+  end
+  class Import <SiSU_DbImport::Import #<SiSU_DB::ColumnSize                 # db_import.rb
+  end
+end
+__END__

data/lib/sisu/db_drop.rb

@@ -0,0 +1,207 @@
+# encoding: utf-8
+=begin
+
+* Name: SiSU
+
+** Description: documents, structuring, processing, publishing, search
+*** modules shared by the different db types, dbi, postgresql, sqlite
+
+** Author: Ralph Amissah
+  <ralph@amissah.com>
+  <ralph.amissah@gmail.com>
+
+** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
+  All Rights Reserved.
+
+** License: GPL 3 or later:
+
+  SiSU, a framework for document structuring, publishing and search
+
+  Copyright (C) Ralph Amissah
+
+  This program is free software: you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published by the Free
+  Software Foundation, either version 3 of the License, or (at your option)
+  any later version.
+
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program. If not, see <http://www.gnu.org/licenses/>.
+
+  If you have Internet connection, the latest version of the GPL should be
+  available at these locations:
+  <http://www.fsf.org/licensing/licenses/gpl.html>
+  <http://www.gnu.org/licenses/gpl.html>
+
+  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+** SiSU uses:
+  * Standard SiSU markup syntax,
+  * Standard SiSU meta-markup syntax, and the
+  * Standard SiSU object citation numbering and system
+
+** Hompages:
+  <http://www.jus.uio.no/sisu>
+  <http://www.sisudoc.org>
+
+** Git
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/db_drop.rb;hb=HEAD>
+
+=end
+module SiSU_DbDrop
+  require_relative 'utils_response'                   # utils_response.rb
+  class Drop
+    include SiSU_Response
+    def initialize(opt,conn,db_info,sql_type)
+      @opt,@conn,@db_info,@sql_type=opt,conn,db_info,sql_type
+      case @sql_type
+      when :sqlite
+        cascade=''
+      else
+        cascade='CASCADE'
+      end
+      @drop_table=[
+        "DROP TABLE metadata_and_text #{cascade};",
+        "DROP TABLE doc_objects #{cascade};",
+        "DROP TABLE urls #{cascade};",
+        "DROP TABLE endnotes #{cascade};",
+        "DROP TABLE endnotes_asterisk #{cascade};",
+        "DROP TABLE endnotes_plus #{cascade};",
+      ]
+    end
+    def drop
+      def tables                                                              #% drop all tables
+        begin
+          msg_sqlite="as not all disk space is recovered after dropping the database << #{@db_info.sqlite.db} >>, you may be better off deleting the file, and recreating it as necessary"
+          case @sql_type
+          when :sqlite
+            puts msg_sqlite
+            ans=response?('remove sql database?')
+            if ans \
+            and File.exist?(@db_info.sqlite.db)
+              @conn.close
+              File.unlink(@db_info.sqlite.db)
+              db=SiSU_Env::InfoDb.new
+              conn=db.sqlite.conn_sqlite3
+              sdb=SiSU_DbDBI::Create.new(@opt,conn,@db_info,@sql_type)
+              sdb_index=SiSU_DbDBI::Index.new(@opt,conn,@db_info,@sql_type)
+              sdb.output_dir?
+              begin
+                sdb.create_db
+                sdb.create_table.metadata_and_text
+                sdb.create_table.doc_objects
+                sdb.create_table.endnotes
+                sdb.create_table.endnotes_asterisk
+                sdb.create_table.endnotes_plus
+                sdb.create_table.urls
+                sdb_index.create_indexes
+              rescue
+                SiSU_Errors::Rescued.new($!,$@,'-D').location do
+                  __LINE__.to_s + ':' + __FILE__
+                end
+                sdb.output_dir?
+              end
+              exit
+            else
+              @conn.transaction
+              @drop_table.each do |d|
+                begin
+                  @conn.exec_params(d)
+                rescue
+                  next
+                end
+                end
+              @conn.commit
+            end
+          when :pg
+            @conn.transaction
+            @drop_table.each do |d|
+              begin
+                @conn.exec_params(d)
+              rescue
+                next
+              end
+            end
+            @conn.commit
+          end
+        rescue
+          case @sql_type
+          when :sqlite
+            ans=response?('remove sql database?')
+            if ans and File.exist?(@db_info.sqlite.db); File.unlink(@db_info.sqlite.db)
+            end
+          else
+            @drop_table.each do |d|
+              begin
+                @conn.exec_params(d)
+              rescue
+                next
+              end
+            end
+          end
+        ensure
+        end
+      end
+      def indexes
+        def conn_execute_array(sql_arr)
+          @conn.transaction do |conn|
+            sql_arr.each do |sql|
+              begin
+                conn.exec_params(sql)
+              rescue
+                next
+              end
+            end
+          end
+        end
+        def base                                                             #% drop base indexes
+          print "\n          drop documents common indexes\n" unless @opt.act[:quiet][:set]==:on
+          sql_arr=[
+            %{DROP INDEX idx_title;},
+            %{DROP INDEX idx_author;},
+            %{DROP INDEX idx_filename;},
+            %{DROP INDEX idx_topics;},
+            %{DROP INDEX idx_ocn;},
+            %{DROP INDEX idx_digest_clean;},
+            %{DROP INDEX idx_digest_all;},
+            %{DROP INDEX idx_lev0;},
+            %{DROP INDEX idx_lev1;},
+            %{DROP INDEX idx_lev2;},
+            %{DROP INDEX idx_lev3;},
+            %{DROP INDEX idx_lev4;},
+            %{DROP INDEX idx_lev5;},
+            %{DROP INDEX idx_lev6;},
+            %{DROP INDEX idx_endnote_nr;},
+            %{DROP INDEX idx_digest_en;},
+            %{DROP INDEX idx_endnote_nr_asterisk;},
+            %{DROP INDEX idx_endnote_asterisk;},
+            %{DROP INDEX idx_digest_en_asterisk;},
+            %{DROP INDEX idx_endnote_nr_plus;},
+            %{DROP INDEX idx_endnote_plus;},
+            %{DROP INDEX idx_digest_en_plus},
+          ]
+          conn_execute_array(sql_arr)
+        end
+        def text                                                             #% drop TEXT indexes, sqlite
+          print "\n          drop documents TEXT indexes\n" unless @opt.act[:quiet][:set]==:on
+          sql_arr=[
+            %{DROP INDEX idx_clean;},
+            %{DROP INDEX idx_endnote},
+          ]
+          conn_execute_array(sql_arr)
+        end
+        self
+      end
+      indexes.base
+      @opt.act[:psql][:set]==:on ? '' : indexes.text
+      self
+    end
+  end
+end
+__END__

data/lib/sisu/db_import.rb

@@ -0,0 +1,877 @@
+# encoding: utf-8
+=begin
+
+* Name: SiSU
+
+** Description: documents, structuring, processing, publishing, search
+*** modules shared by the different db types, dbi, postgresql, sqlite
+
+** Author: Ralph Amissah
+  <ralph@amissah.com>
+  <ralph.amissah@gmail.com>
+
+** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
+  All Rights Reserved.
+
+** License: GPL 3 or later:
+
+  SiSU, a framework for document structuring, publishing and search
+
+  Copyright (C) Ralph Amissah
+
+  This program is free software: you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published by the Free
+  Software Foundation, either version 3 of the License, or (at your option)
+  any later version.
+
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program. If not, see <http://www.gnu.org/licenses/>.
+
+  If you have Internet connection, the latest version of the GPL should be
+  available at these locations:
+  <http://www.fsf.org/licensing/licenses/gpl.html>
+  <http://www.gnu.org/licenses/gpl.html>
+
+  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+** SiSU uses:
+  * Standard SiSU markup syntax,
+  * Standard SiSU meta-markup syntax, and the
+  * Standard SiSU object citation numbering and system
+
+** Hompages:
+  <http://www.jus.uio.no/sisu>
+  <http://www.sisudoc.org>
+
+** Git
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
+  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/db_import.rb;hb=HEAD>
+
+=end
+module SiSU_DbImport
+  require_relative 'db_columns'                         # db_columns.rb
+  require_relative 'db_load_tuple'                      # db_load_tuple.rb
+  require_relative 'db_sqltxt'                          # db_sqltxt.rb
+  require_relative 'html_lite_shared'                   # html_lite_shared.rb
+  class Import < SiSU_DbText::Prepare
+    include SiSU_Param
+    include SiSU_Screen
+    include SiSU_DbAction
+    @@dl=nil
+    @@hname=nil
+    attr_accessor :tp
+    def initialize(opt,conn,file_maint,sql_type=:pg)
+      @opt,@conn,@file_maint,@sql_type=opt,conn,file_maint,sql_type
+      @cX=SiSU_Screen::Ansi.new(@opt.act[:color_state][:set]).cX
+      @env=SiSU_Env::InfoEnv.new(@opt.fns)
+      @dal="#{@env.processing_path.ao}"
+      @fnb=if @opt.fns.empty? \
+      or @opt.selections.str.empty?
+        ''
+      else
+        @md=SiSU_Param::Parameters.new(@opt).get
+        @md.fnb
+      end
+      @fnc="#{@dal}/#{@opt.fns}.content.rbm"
+      @@seg,@@seg_full='',''                                  #create? consider placing field just before clean text as opposed to seg which contains seg(.html) name info seg_full would contain seg info for levels 5 & 6 where available eg seg_full may be 7.3 (level 5) and 7.3.1 (level 6) where seg  is 7
+      @col=Hash.new('')
+      @col[:ocn]=''
+      @counter={}
+      @db=SiSU_Env::InfoDb.new
+      if @sql_type==:sqlite
+        @driver_sqlite3=
+        (@conn.inspect.match(/^(.{10})/)[1] \
+        == @db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) \
+        ? true
+        : false
+      end
+      sql='SELECT MAX(lid) FROM doc_objects'
+      begin
+        @col[:lid] ||=0
+        @col[:lid]=@driver_sqlite3 \
+        ? @conn.execute( sql ).join.to_i
+        : @conn.exec( sql ).getvalue(0,0).to_i
+      rescue
+        if @opt.act[:maintenance][:set]==:on
+          puts "#{__FILE__}:#{__LINE__}"
+        end
+      end
+      @col[:lid]=0 if @col[:lid].nil? or @col[:lid].to_s.empty?
+      sql='SELECT MAX(nid) FROM endnotes'
+      begin
+        @id_n=@driver_sqlite3 \
+        ? @conn.execute( sql ).join.to_i
+        : @conn.exec( sql ).getvalue(0,0).to_i
+        @id_n ||=0
+      rescue
+        if @opt.act[:maintenance][:set]==:on
+          puts "#{__FILE__}:#{__LINE__}"
+        end
+      end
+      @id_n =0 if @col[:lid].nil? or @col[:lid].to_s.empty?
+      @col[:lv0]=@col[:lv1]=@col[:lv2]=@col[:lv3]=@col[:lv4]=@col[:lv5]=@col[:lv6]=@col[:lv7]=0
+      @db=SiSU_Env::InfoDb.new
+      @pdf_fn=SiSU_Env::FileOp.new(@md).base_filename
+      @@dl ||=SiSU_Env::InfoEnv.new.digest.length
+    end
+    def marshal_load
+      require_relative 'ao'                               # ao.rb
+      @ao_array=SiSU_AO::Source.new(@opt).get             # ao file drawn here
+      if (@opt.act[:verbose][:set]==:on \
+      || @opt.act[:verbose_plus][:set]==:on \
+      || @opt.act[:maintenance][:set]==:on)
+        SiSU_Screen::Ansi.new(
+          @opt.act[:color_state][:set],
+          "#{@db.psql.db}::#{@opt.fns}"
+        ).puts_blue
+      end
+      SiSU_Screen::Ansi.new(
+        @opt.act[:color_state][:set],
+        'Marshal Load',
+        @fnc
+      ).puts_grey if @opt.act[:verbose][:set]==:on
+      select_first_match=%{
+        SELECT metadata_and_text.tid
+        FROM metadata_and_text
+        WHERE metadata_and_text.src_filename = '#{@md.fns}'
+        AND metadata_and_text.language_document_char = '#{@opt.lng}'
+      ;} # note, for .ssm: @md.fns (is set during runtime & is) != @opt.fns @md.opt.fns
+      file_exist=if @sql_type==:sqlite
+        begin
+          @conn.get_first_value(select_first_match)
+        rescue SQLite3::Exception => e
+          # not tested
+          puts "Exception occurred"
+          puts e
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:yellow).mark(
+            "\n" \
+            + 'Attempting to initialize db' + "\n" \
+            + 'Creating db tables'
+          )
+          sdb={
+            create: SiSU_DbDBI::Create.new(@opt,@conn,@file_maint,@sql_type),
+            index: SiSU_DbDBI::Index.new(@opt,@conn,@file_maint,@sql_type),
+          }
+          db_action(sdb).create
+        end
+      else
+        begin
+          @conn.exec(select_first_match).field_values("tid")[0]
+        rescue PG::Error => e
+          err=[
+            e.result.error_field( PG::Result::PG_DIAG_SEVERITY ),
+            e.result.error_field( PG::Result::PG_DIAG_SQLSTATE ),
+            e.result.error_field( PG::Result::PG_DIAG_MESSAGE_PRIMARY ),
+            e.result.error_field( PG::Result::PG_DIAG_MESSAGE_DETAIL ),
+            e.result.error_field( PG::Result::PG_DIAG_MESSAGE_HINT ),
+            e.result.error_field( PG::Result::PG_DIAG_STATEMENT_POSITION ),
+            e.result.error_field( PG::Result::PG_DIAG_INTERNAL_POSITION ),
+            e.result.error_field( PG::Result::PG_DIAG_INTERNAL_QUERY ),
+            e.result.error_field( PG::Result::PG_DIAG_CONTEXT ),
+            e.result.error_field( PG::Result::PG_DIAG_SOURCE_FILE ),
+            e.result.error_field( PG::Result::PG_DIAG_SOURCE_LINE ),
+            e.result.error_field( PG::Result::PG_DIAG_SOURCE_FUNCTION ),
+          ]
+          p err
+          if err[2] =~/relation "\S+?" does not exist/ \
+          or err.inspect =~/relation "\S+?" does not exist/
+            SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:yellow).mark(
+              "\n" \
+              + err[2] + "\n" \
+              + 'Attempting to initialize db' + "\n" \
+              + 'Creating db tables'
+            )
+            sdb={
+              create: SiSU_DbDBI::Create.new(@opt,@conn,@file_maint,@sql_type),
+              index: SiSU_DbDBI::Index.new(@opt,@conn,@file_maint,@sql_type),
+            }
+            db_action(sdb).create
+            retry
+          end
+        end
+      end
+      if not file_exist
+        t_d=[]                                                              # transaction_data
+        t_d << db_import_metadata
+        t_d << db_import_documents(@ao_array)
+        t_d << db_import_urls(@ao_array,@fnc)                              #import OID on/off
+        t_d=t_d.flatten
+        if (@opt.act[:verbose_plus][:set]==:on \
+        || @opt.act[:maintenance][:set]==:on)
+          puts @conn.class if defined? @conn.class
+          puts @conn.driver_name if defined? @conn.driver_name
+          puts @conn.driver if defined? @conn.driver
+        end
+        begin                                                               #% sql
+          if @sql_type==:sqlite
+            @conn.transaction do |conn|
+              t_d.each do |sql|
+                conn.execute(sql)
+              end
+            end
+            #also 'execute' works for sqlite
+            #@conn.execute("BEGIN")
+            #  t_d.each do |sql|
+            #    @conn.execute(sql)
+            #  end
+            #@conn.execute("COMMIT")
+          else
+            #'do' works for postgresql
+            @conn.exec("BEGIN")
+              t_d.each do |sql|
+                @conn.exec(sql)
+              end
+            @conn.exec("COMMIT")
+          end
+        rescue
+          SiSU_Errors::Rescued.new($!,$@,@opt.selections.str,@opt.fns).location do
+            __LINE__.to_s + ':' + __FILE__
+          end
+          sqlfn="#{@env.processing_path.sql}/#{@md.fnb}.sql"
+          sql=File.new(sqlfn,'w')
+          t_d.each {|i| sql.puts i}
+          p sqlfn
+          if @opt.act[:maintenance][:set]==:on
+            puts sql
+            p @conn.methods.sort
+            puts "#{__FILE__}:#{__LINE__}"
+          end
+        ensure
+        end
+      else
+        if file_exist
+          @db=SiSU_Env::InfoDb.new
+          puts "\nfile #{@opt.fns} in language code #{@opt.lng} already exists in database #{@db.psql.db}, use --update instead?"
+        end
+      end
+    end
+    def pf_db_import_transaction_open
+    end
+    def pf_db_import_transaction_close
+    end
+    def book_idx_hash_to_str(book_idx)
+      book_idx=book_idx ? book_idx : ''
+      book_idx_str,book_subidx_part='',''
+      if not book_idx.empty?
+        book_idx_str=''
+        book_idx.each_pair do |k0,v0|
+          book_idx_str << %{#{k0}+#{v0[:plus]}}
+          book_subidx_part=''
+          if v0[:sub].length > 0
+            v0[:sub].each do |subterms|
+               subterms.each_pair do |k1,v1|
+                 book_subidx_part << %{\n  #{k1}+#{v1[:plus]} | }
+               end
+            end
+            book_idx_str=book_idx_str + ':' + book_subidx_part
+          end
+        end
+      end
+      book_idx_str
+    end
+    def db_import_metadata                                                       #% import documents - populate database
+      if (@opt.act[:verbose][:set]==:on \
+      || @opt.act[:verbose_plus][:set]==:on \
+      || @opt.act[:maintenance][:set]==:on)
+        print %{ #{@cX.grey}import documents dbi_unit #{@cX.off} }
+      end
+      @tp={}
+      @md=SiSU_Param::Parameters.new(@opt).get
+#% sisutxt & fulltxt
+      if FileTest.exist?(@md.fns)
+        txt_arr=IO.readlines(@md.fns,'')
+        src=txt_arr.join("\n")
+        src=special_character_escape(src)
+        @tp[:sisutxt_f],@tp[:sisutxt_i]='sisutxt, ',"'#{src}', "
+        txt=clean_searchable_text_from_document_source(txt_arr)
+        #txt=special_character_escape(txt)
+        @tp[:fulltxt_f],@tp[:fulltxt_i]='fulltxt, ',"'#{txt}', "
+      end
+#% title
+      if defined? @md.title.full \
+      and @md.title.full=~/\S+/                                              # DublinCore 1 - title
+        #@tp[:title]=@md.title.full
+        #special_character_escape(@tp[:title])
+        #@tp[:title_f],@tp[:title_i]='title, ',"'#{@tp[:title]}', "
+        sql='SELECT MAX(tid) FROM metadata_and_text;'
+        begin
+          @@id_t ||=0
+          id_t=@driver_sqlite3 \
+          ? @conn.execute( sql ).join.to_i # { |x| id_t=x.join.to_i }
+          : @conn.exec( sql ).getvalue(0,0).to_i
+          @@id_t=id_t if id_t
+        rescue
+          if @opt.act[:maintenance][:set]==:on
+            puts "#{__FILE__} #{__LINE__}"
+          end
+        end
+        @@id_t+=1 #bug related, needs to be performed once at start of file, but consider moving, as, placed here it means program will fail if document header lacks @title:
+        if (@opt.act[:verbose][:set]==:on \
+        || @opt.act[:verbose_plus][:set]==:on \
+        || @opt.act[:maintenance][:set]==:on)
+          puts %{\n#{@cX.grey}Processing file number#{@cX.off}: #{@cX.green}#{@@id_t}#{@@cX.off}}
+        end
+      end
+      ################ CLEAR ##############
+      SiSU_DbDBI::Test.new(self,@opt).verify                          #% import title names, filenames (tuple)
+      t=SiSU_DbTuple::LoadMetadata.new(@conn,@@id_t,@md,@file_maint)
+      tuple=t.tuple
+      tuple
+    end
+    def db_import_documents(ao_array)                                     #% import documents - populate main database table, import into substantive database tables (tuple)
+      begin
+        @col[:tid]=@@id_t
+        @en,@en_ast,@en_pls,@tuple_array=[],[],[],[]
+        @col[:en_a],@col[:en_z]=nil,nil
+        ao_array.each do |data|
+          data.obj=data.obj.gsub(/#{Mx[:fa_bold_o]}(.+?)#{Mx[:fa_bold_c]}/,'\1').
+            gsub(/#{Mx[:fa_italics_o]}(.+?)#{Mx[:fa_italics_c]}/,'\1').
+            gsub(/#{Mx[:fa_underscore_o]}(.+?)#{Mx[:fa_underscore_c]}/,'\1').
+            gsub(/#{Mx[:fa_superscript_o]}(.+?)#{Mx[:fa_superscript_c]}/,'\1').
+            gsub(/#{Mx[:fa_subscript_o]}(.+?)#{Mx[:fa_subscript_c]}/,'\1').
+            gsub(/#{Mx[:fa_insert_o]}(.+?)#{Mx[:fa_insert_c]}/,'\1').
+            gsub(/#{Mx[:fa_cite_o]}(.+?)#{Mx[:fa_cite_c]}/,'\1').
+            gsub(/#{Mx[:fa_strike_o]}(.+?)#{Mx[:fa_strike_c]}/,'\1').
+            gsub(/#{Mx[:fa_monospace_o]}(.+?)#{Mx[:fa_monospace_c]}/,'\1').
+            gsub(/#{Mx[:gl_o]}(●)#{Mx[:gl_c]}\s*/,'\1 ').
+            gsub(/#{Mx[:tag_o]}\S+?#{Mx[:tag_c]}/,'') #check
+          @col[:seg]=@@seg
+          if data.of ==:para \
+          || data.of ==:heading \
+          || data.of ==:heading_insert \
+          || data.of ==:block \
+          || data.of ==:group      # regular text what of code-blocks grouped text etc.
+            notedata=data.obj.dup
+                                                                               #% :headings
+            if data.is==:heading \
+            && (data.ln.inspect=~/[0-3]/)
+              (
+                @col[:lev],
+                txt,@col[:ocn],
+                @col[:lev_an],
+                @col[:ocnd],@col[:ocns],
+                @col[:t_of],@col[:t_is],
+                @col[:node],@col[:parent],
+                @col[:digest_clean],@col[:digest_all]=
+                data.ln,
+                data.obj,data.ocn,
+                data.lv,
+                data.odv,data.osp,
+                data.of,data.is,
+                data.node,data.parent,
+                '',''
+              )
+              @col[:lid]+=1
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_minus
+              @col[:body]=clean_document_objects_body(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text_from_document_objects(plaintext)
+              book_idx=book_idx_hash_to_str(data.idx)
+              @col[:book_idx]=clean_searchable_text_from_document_objects(book_idx)
+              if @en[0] then @en_a,@en_z=@en[0].first,@en[0].last
+              end
+              if @en_ast[0] then @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last
+              end
+              if @en_pls[0] then @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last
+              end
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              case @col[:lev]
+              when /0/ then @col[:lv0]+=1
+              when /1/ then @col[:lv1]+=1
+              when /2/ then @col[:lv2]+=1
+              when /3/ then @col[:lv3]+=1
+              when /4/ then @col[:lv4]+=1
+              end
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==4
+              (
+                @@seg,txt,
+                @col[:ocn],@col[:lev_an],
+                @col[:ocnd],@col[:ocns],
+                @col[:t_of],@col[:t_is],
+                @col[:node],@col[:parent],
+                @col[:digest_clean],@col[:digest_all]=
+                data.name,data.obj,
+                data.ocn,data.lv,
+                data.odv,data.osp,
+                data.of,data.is,
+                data.node,data.parent,
+                '',''
+              )
+              @col[:seg]=@@seg
+              @col[:lv4]+=1
+              @col[:lid]+=1
+              @col[:lev]=4
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=clean_document_objects_body(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text_from_document_objects(plaintext)
+              book_idx=book_idx_hash_to_str(data.idx)
+              @col[:book_idx]=clean_searchable_text_from_document_objects(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==5
+              (
+                txt,
+                @col[:ocn],@col[:lev_an],
+                @col[:ocnd],@col[:ocns],
+                @col[:t_of],@col[:t_is],
+                @col[:node],@col[:parent],
+                @col[:digest_clean],@col[:digest_all]=
+                data.obj,
+                data.ocn,data.lv,
+                data.odv,data.osp,
+                data.of,data.is,
+                data.node,data.parent,
+                '',''
+              )
+              @@seg_full=data.name if data.is==:heading \
+              && data.ln==5 \
+              && data.name #check data.name
+              @@seg ||='' #nil # watch
+              @col[:seg]=@@seg
+              @col[:lv5]+=1
+              @col[:lid]+=1
+              @col[:lev]=5
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=clean_document_objects_body(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text_from_document_objects(plaintext)
+              book_idx=book_idx_hash_to_str(data.idx)
+              @col[:book_idx]=clean_searchable_text_from_document_objects(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==6
+              txt,       @col[:ocn],@col[:lev_an],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=
+                data.obj,data.ocn,  data.lv,      data.odv,   data.osp,   data.of,    data.is,    data.node,  data.parent,  '',                 ''
+              @@seg_full=data.name if data.is==:heading && data.ln==6 && data.name #check data.name
+              @@seg ||='' #nil # watch
+              @col[:seg]=@@seg
+              @col[:lv6]+=1
+              @col[:lid]+=1
+              @col[:lev]=6
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=clean_document_objects_body(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text_from_document_objects(plaintext)
+              book_idx=book_idx_hash_to_str(data.idx)
+              @col[:book_idx]=clean_searchable_text_from_document_objects(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+            elsif data.is==:heading \
+            && data.ln==7
+              txt,       @col[:ocn],@col[:lev_an],@col[:ocnd],@col[:ocns],@col[:t_of],@col[:t_is],@col[:node],@col[:parent],@col[:digest_clean],@col[:digest_all]=
+                data.obj,data.ocn,  data.lv,      data.odv,   data.osp,   data.of,    data.is,    data.node,  data.parent,  '',                 ''
+              @@seg_full=data.name if data.is==:heading && data.ln==7 && data.name #check data.name
+              @@seg ||='' #nil # watch
+              @col[:seg]=@@seg
+              @col[:lv7]+=1
+              @col[:lid]+=1
+              @col[:lev]=7
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              body=SiSU_FormatShared::CSS_Format.new(@md,data).lev4_plus
+              @col[:body]=clean_document_objects_body(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text_from_document_objects(plaintext)
+              book_idx=book_idx_hash_to_str(data.idx)
+              @col[:book_idx]=clean_searchable_text_from_document_objects(book_idx)
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @col[:lev]=@col[:plaintext]=@col[:body]=''
+                                                                               #% :structure :layout :comment
+            elsif data.of==:structure \
+            || data.of==:layout \
+            || data.of==:comment
+              #added watch
+                                                                               #% :
+            else                                                               #% regular text
+              @col[:lid]+=1
+              (
+                txt=''
+                txt,@col[:ocn],
+                @col[:ocnd],@col[:ocns],
+                @col[:t_of],@col[:t_is],
+                @col[:node],@col[:parent],
+                @col[:digest_clean],@col[:digest_all],
+                @col[:lev]=
+                data.obj,data.ocn,
+                data.odv,data.osp,
+                data.of,data.is,
+                '',data.parent,
+                '','',
+                9
+              )
+              @hname=if @col[:seg] \
+              and not @col[:seg].to_s.empty?
+                @@hname=@col[:seg].to_s
+              else @@hname
+              end
+              @env=SiSU_Env::InfoEnv.new(@md.fns)
+              @base_url="#{@env.url.root}/#{@md.fnb}/#{@hname}.html"
+              txt=endnotes(txt).extract_any
+              if @sql_type==:pg \
+              and txt.size > (SiSU_DbColumns::ColumnSize.new.document_clean - 1)             # examine pg build & remove limitation
+                puts "\n\nTOO LARGE (TXT - see error log)\n\n"
+                open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                  error.puts("\n#{@opt.fns}\nTEXT BODY\n#{@col[:body].size} object #{@col[:ocn]} -> #{@col[:body].slice(0..500)}")
+                end
+                txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+              end
+              @en_a,@en_z=@en[0].first,@en[0].last if @en[0]
+              @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0]
+              @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0]
+              body=if data.is==:table
+                SiSU_FormatShared::CSS_Format.new(@md,data).html_table
+              elsif data.is==:code
+                SiSU_FormatShared::CSS_Format.new(@md,data).code
+              elsif defined? data.indent \
+              and defined? data.hang \
+              and data.indent =~/[1-9]/ \
+              and data.indent == data.hang
+                SiSU_FormatShared::CSS_Format.new(@md,data).indent(data.indent)
+              elsif defined? data.indent \
+              and defined? data.hang \
+              and data.hang =~/[0-9]/ \
+              and data.indent != data.hang
+                SiSU_FormatShared::CSS_Format.new(@md,data).hang_indent(data.hang,data.indent)
+              else
+                SiSU_FormatShared::CSS_Format.new(@md,data).norm
+              end
+              @col[:body]=clean_document_objects_body(body)
+              plaintext=@col[:body].dup
+              plaintext=strip_markup(plaintext)
+              @col[:plaintext]=clean_searchable_text_from_document_objects(plaintext)
+              book_idx=book_idx_hash_to_str(data.idx)
+              @col[:book_idx]=clean_searchable_text_from_document_objects(book_idx)
+              t=SiSU_DbTuple::LoadDocuments.new(@conn,@col,@opt,@file_maint)
+              @tuple_array << t.tuple
+              @en,@en_ast,@en_pls=[],[],[]
+              @col[:en_a]=@col[:en_z]=nil
+              @col[:lev]=@col[:plaintext]=@col[:body]=@col[:words]=''
+            end
+            if notedata =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/                                         #% import into database endnotes tables
+              endnote_array=notedata.scan(/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/)
+              endnote_array.each do |inf|
+                if inf[/#{Mx[:en_a_o]}\d+.+?#{Mx[:en_a_c]}/]
+                  if inf[/#{Mx[:en_a_o]}(\d+)(.+?)#{Mx[:en_a_c]}/]
+                    nr,txt,digest_clean=$1,$2.strip,0
+                  end
+                  @id_n ||=0
+                  @id_n+=1
+                  txt=special_character_escape(txt)
+                  body=SiSU_FormatShared::CSS_Format.new(@md,data).endnote(nr,txt)
+                  txt=strip_markup(txt)
+                  if txt.size > (SiSU_DbColumns::ColumnSize.new.endnote_clean - 1)
+                    puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n"
+                    open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                      error.puts("\n#{@opt.fns}\nENDNOTE\n#{txt.size} object #{@col[:ocn]},#{@col[:ocnd]},#{@col[:ocns]} -> #{txt.slice(0..500)}")
+                    end
+                    txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+                  end
+                  if txt
+                    en={
+                      type: 'endnotes',
+                      id:      @id_n,
+                      lid:     @col[:lid],
+                      nr:      nr,
+                      txt:     txt,
+                      body:    body,
+                      ocn:     @col[:ocn],
+                      ocnd:    @col[:ocnd],
+                      ocns:    @col[:ocns],
+                      id_t:    @@id_t,
+                      hash:    digest_clean
+                    }
+                    t=SiSU_DbTuple::LoadEndnotes.new(@conn,en,@opt,@file_maint)
+                    @tuple_array << t.tuple
+                  end
+                end
+              end
+              word_mode=notedata.scan(/\S+/)
+            end
+            if notedata =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/                                      #% import into database endnotes tables
+              endnote_array=notedata.scan(/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/)
+              endnote_array.each do |inf|
+                if inf[/#{Mx[:en_b_o]}\*\d+.+?#{Mx[:en_b_c]}/]                    # dal new endnotes 2003w31/1
+                  if inf[/#{Mx[:en_b_o]}[*](\d+)(.+?)#{Mx[:en_b_c]}/]           # dal new endnotes 2003w31/1
+                    nr,txt,digest_clean=$1,$2.strip,0
+                  end
+                  @id_n+=1
+                  txt=special_character_escape(txt)
+                  body=SiSU_FormatShared::CSS_Format.new(@md,data).endnote(nr,txt)
+                  txt=strip_markup(txt)
+                  if txt.size > (SiSU_DbColumns::ColumnSize.new.endnote_clean - 1)
+                    puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n"
+                    open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                      error.puts("\n#{@opt.fns}\nENDNOTE\n#{txt.size} object #{@col[:ocn]},#{@col[:ocnd]},#{@col[:ocns]} -> #{txt.slice(0..500)}")
+                    end
+                    txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+                  end
+                  if txt
+                    en={
+                      type: 'endnotes_asterisk',
+                      id:      @id_n,
+                      lid:     @col[:lid],
+                      nr:      nr,
+                      txt:     txt,
+                      body:    body,
+                      ocn:     @col[:ocn],
+                      ocnd:    @col[:ocnd],
+                      ocns:    @col[:ocns],
+                      id_t:    @@id_t,
+                      hash:    digest_clean
+                    }
+                    t=SiSU_DbTuple::LoadEndnotes.new(@conn,en,@opt,@file_maint)
+                    @tuple_array << t.tuple
+                  end
+                end
+              end
+              word_mode=notedata.scan(/\S+/)
+            end
+            if notedata =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/                                           #% import into database endnotes tables
+              endnote_array=notedata.scan(/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/)
+              endnote_array.each do |inf|
+                if inf[/#{Mx[:en_b_o]}\+\d+.+?#{Mx[:en_b_c]}/]                        # dal new endnotes 2003w31/1
+                  if inf[/#{Mx[:en_b_o]}[+](\d+)(.+?)#{Mx[:en_b_c]}/]               # dal new endnotes 2003w31/1
+                    nr,txt,digest_clean=$1,$2.strip,0
+                  end
+                  @id_n+=1
+                  txt=special_character_escape(txt)
+                  body=SiSU_FormatShared::CSS_Format.new(@md,data).endnote(nr,txt)
+                  txt=strip_markup(txt)
+                  if txt.size > (SiSU_DbColumns::ColumnSize.new.endnote_clean - 1)
+                    puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n"
+                    open("#{Dir.pwd}/pg_documents_error_log",'a') do |error|
+                      error.puts("\n#{@opt.fns}\nENDNOTE\n#{txt.size} object #{@col[:ocn]},#{@col[:ocnd]},#{@col[:ocns]} -> #{txt.slice(0..500)}")
+                    end
+                    txt=%{\n\nLARGE TEXT BLOCK OMITTED\n\n}
+                  end
+                  if txt
+                    en={
+                      type: 'endnotes_plus',
+                      id:      @id_n,
+                      lid:     @col[:lid],
+                      nr:      nr,
+                      txt:     txt,
+                      body:    body,
+                      ocn:     @col[:ocn],
+                      ocnd:    @col[:ocnd],
+                      ocns:    @col[:ocns],
+                      id_t:    @@id_t,
+                      hash:    digest_clean
+                    }
+                    t=SiSU_DbTuple::LoadEndnotes.new(@conn,en,@opt,@file_maint)
+                    @tuple_array << t.tuple
+                  end
+                end
+              end
+              word_mode=notedata.scan(/\S+/)
+            end
+          end
+        end
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.selections.str,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+      @tuple_array
+    end
+    def endnotes(txt)
+      @txt=txt
+      def extract_any
+        if @txt =~/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})[*+]?(\d+)\s+.+?(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/
+          endnotes(@txt).range
+          @en << endnotes(@txt).standard if @txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/
+          @en_ast << endnotes(@txt).asterisk if @txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/
+          @en_pls << endnotes(@txt).plus if @txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/
+          @txt=endnotes(@txt).clean_text
+        end
+        @txt
+      end
+      def standard
+        (@txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/) \
+        ? @txt.scan(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/)
+        : nil
+      end
+      def asterisk
+        (@txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/) \
+        ? @txt.scan(/#{Mx[:en_b_o]}[*](\d+).+?#{Mx[:en_b_c]}/)
+        : nil
+      end
+      def plus
+        (@txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/) \
+        ? @txt.scan(/#{Mx[:en_b_o]}[+](\d+).+?#{Mx[:en_b_c]}/)
+        : nil
+      end
+      def clean_text(base_url=nil)
+        @txt=if base_url
+          @txt.gsub(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/,%{<sup><a href="#{base_url}#_\\1" name="-\\1">\\1</a></sup>}).
+            gsub(/#{Mx[:en_b_o]}([*]\d+).+?#{Mx[:en_b_c]}/,%{<sup><a href="#{base_url}#_\\1" name="-\\1">\\1</a></sup>}).
+            gsub(/#{Mx[:en_b_o]}([+]\d+).+?#{Mx[:en_b_c]}/,%{<sup><a href="#{base_url}#_\\1" name="-\\1">\\1</a></sup>})
+        else
+          @txt.gsub(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/,'<sup>\1</sup>').
+            gsub(/#{Mx[:en_b_o]}([*]\d+).+?#{Mx[:en_b_c]}/,'<sup>\1</sup>').
+            gsub(/#{Mx[:en_b_o]}([+]\d+).+?#{Mx[:en_b_c]}/,'<sup>\1</sup>')
+        end
+        @txt
+      end
+      def range
+        @col[:en_a]=@col[:en_z]=nil
+        if @txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}([*]\d+).+?#{Mx[:en_b_c]}|#{Mx[:en_b_o]}([+]\d+).+?#{Mx[:en_b_c]}/
+          word_array=@txt.scan(/\S+/)
+          word_array.each do |w|
+            if w[/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})[*+]?(\d+)\s+.+?(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/]                                                # not tested since change 2003w31
+              @col[:en_a]=$1 unless @col[:en_a]
+              @col[:en_z]=@col[:en_a].dup unless @col[:en_a]
+              @col[:en_z]=$1 if @col[:en_a]
+            end
+          end
+        end
+        @col
+      end
+      self
+    end
+    def db_import_urls(dbi_unit,content)                                           #% import documents OID - populate database
+      begin
+        @fnc=content
+        @env=SiSU_Env::InfoEnv.new(@opt.fns)
+        f,u={},{}
+        if @fnb.empty? \
+        or @fnb.nil?
+          p 'file output path error' #remove
+        end
+        if FileTest.file?("#{@md.file.output_path.txt.dir}/#{@md.file.base_filename.txt}")==true
+          f[:txt],u[:txt]='plaintext,', "'#{@md.file.output_path.txt.url}/#{@md.file.base_filename.txt}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.html_seg.dir}/#{@md.file.base_filename.html_segtoc}")==true
+          f[:html_toc],u[:html_toc]='html_toc,', "'#{@md.file.output_path.html_seg.url}/#{@md.file.base_filename.html_segtoc}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.html_scroll.dir}/#{@md.file.base_filename.html_scroll}")==true
+          f[:html_doc],u[:html_doc]='html_doc,', "'#{@md.file.output_path.html_scroll.url}/#{@md.file.base_filename.html_scroll}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.xhtml.dir}/#{@md.file.base_filename.xhtml}")==true
+          f[:xhtml],u[:xhtml]='xhtml,', "'#{@md.file.output_path.xhtml.url}/#{@md.file.base_filename.xhtml}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.xml_sax.dir}/#{@md.file.base_filename.xml_sax}")==true
+          f[:xml_sax],u[:xml_sax]='xml_sax,', "'#{@md.file.output_path.xml_sax.url}/#{@md.file.base_filename.xml_sax}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.xml_dom.dir}/#{@md.file.base_filename.xml_dom}")==true
+          f[:xml_dom],u[:xml_dom]='xml_dom,', "'#{@md.file.output_path.xml_dom.url}/#{@md.file.base_filename.xml_dom}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.epub.dir}/#{@md.file.base_filename.epub}")==true
+          f[:epub],u[:epub]='epub,', "'#{@md.file.output_path.epub.url}/#{@md.file.base_filename.epub}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.odt.dir}/#{@md.file.base_filename.odt}")==true
+          f[:odf],u[:odf]='odf,', "'#{@md.file.output_path.odt.url}/#{@md.file.base_filename.odt}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_p_a4}")==true #\
+        #or FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_p_letter}")==true
+          f[:pdf_p],u[:pdf_p]='pdf_p,', "'#{@md.file.output_path.pdf.url}/#{@pdf_fn.pdf_p_a4}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_l_a4}")==true #\
+        #or FileTest.file?("#{@md.file.output_path.pdf.dir}/#{@pdf_fn.pdf_l_letter}")==true
+          f[:pdf_l],u[:pdf_l]='pdf_l,', "'#{@md.file.output_path.pdf.url}/#{@pdf_fn.pdf_l_a4}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.html_concordance.dir}/#{@md.file.base_filename.html_concordance}")==true
+          f[:concordance],u[:concordance]='concordance,', "'#{@md.file.output_path.html_concordance.url}/#{@md.file.base_filename.html_concordance}',"
+        end
+        #if FileTest.file?("#{@md.file.output_path.x.dir}/#{@md.file.base_filename.x}")==true
+        #  f[:latex_p],u[:latex_p]='latex_p,', "'#{@md.file.output_path.x.url}/#{@md.file.base_filename.x}',"
+        #end
+        ##if FileTest.file?("#{out}/#{@fnb}/#{@opt.fns}.tex")==true
+        ##  f[:latex_p],u[:latex_p]='latex_p,', "'#{base}/#{@fnb}/#{@opt.fns}.tex',"
+        ##end
+        #if FileTest.file?("#{@md.file.output_path.x.dir}/#{@md.file.base_filename.x}")==true
+        #  f[:latex_l],u[:latex_l]='latex_l,', "'#{@md.file.output_path.x.url}/#{@md.file.base_filename.x}',"
+        #end
+        ##if FileTest.file?("#{out}/#{@fnb}/#{@opt.fns}.landscape.tex")==true
+        ##  f[:latex_l],u[:latex_l]='latex_l,', "'#{base}/#{@fnb}/#{@opt}.fns}.landscape.tex',"
+        ##end
+        if FileTest.file?("#{@md.file.output_path.digest.dir}/#{@md.file.base_filename.digest}")==true
+          f[:digest],u[:digest]='digest,', "'#{@md.file.output_path.digest.url}/#{@md.file.base_filename.digest}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.manifest.dir}/#{@md.file.base_filename.manifest}")==true #revisit, was to be text, this is html
+          f[:manifest],u[:manifest]='manifest,', "'#{@md.file.output_path.manifest.url}/#{@md.file.base_filename.manifest}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.src.dir}/#{@md.file.base_filename.src}")==true
+          f[:markup],u[:markup]='markup,', "'#{@md.file.output_path.src.url}/#{@md.file.base_filename.src}',"
+        end
+        if FileTest.file?("#{@md.file.output_path.sisupod.dir}/#{@md.file.base_filename.sisupod}")==true
+          f[:sisupod],u[:sisupod]='sisupod,', "'#{@md.file.output_path.sisupod.url}/#{@md.file.base_filename.sisupod}',"
+        end
+        t=SiSU_DbTuple::LoadUrls.new(@conn,f,u,@@id_t,@opt,@file_maint)
+        tuple=t.tuple
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.selections.str,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+      tuple
+    end
+  end
+end
+__END__