echi-converter 0.3.3 → 0.3.4

data/db/migrate/008_create_echi_vdns.rb

@@ -0,0 +1,27 @@
+class CreateEchiVdns < ActiveRecord::Migration
+  def self.up
+    #We create the table from the one defined in the application.yml file
+    create_table "echi_vdns", :force => true do |t|
+      @@echi_schema["echi_vdns"].each do | field |
+        case field["type"]
+        when 'int'
+          t.column field["name"], :integer, :limit => field["length"], :precision => field["length"], :scale => 0
+        when 'str'
+          t.column field["name"], :string, :limit => field["length"]
+        when 'datetime'
+          t.column field["name"], :datetime
+        when 'bool'
+          t.column field["name"], :string, :limit => 1
+        when 'bool_int'
+          t.column field["name"], :string, :limit => 1
+        end
+      end
+    end
+    add_index "echi_vdns", "vdn"
+  end
+
+  def self.down
+    remove_index "echi_vdns", "vdn"
+    drop_table "echi_vdns"
+  end
+end

data/examples/db_connect_test.rb

@@ -0,0 +1,25 @@
+#A test script that allows one to do a quick database connection test 
+#to ensure their database connectivity is configured properly
+#require AR
+require 'rubygems'
+require 'active_record'
+
+# connect to the Database
+ActiveRecord:: Base.establish_connection(
+  :adapter => "oracle",
+  :database => "192.168.10.3/orapr1",
+  :username => "myname",
+  :password => "mypass"
+)
+
+#define a simple model
+class EchiRecord < ActiveRecord::Base
+set_table_name "PCO_ECHIRECORD" # comment out if not using oracle
+end
+
+begin
+  record=EchiRecord.find(:first)
+  puts record.inspect
+rescue => err
+  puts err
+end

data/lib/database.rb

@@ -4,6 +4,15 @@ end
 class EchiLog < ActiveRecord::Base
 end
 
-#The EchiAgent database table may not always be present
+#These database tables may not always be present
 class EchiAgent < ActiveRecord::Base
 end
+
+class EchiAuxReason < ActiveRecord::Base
+end
+
+class EchiCwc < ActiveRecord::Base
+end
+
+class EchiVdn < ActiveRecord::Base
+end

data/lib/echi-converter.rb

@@ -5,6 +5,8 @@ require 'net/ftp'
 require 'net/smtp'
 require 'fileutils'
 require 'uuidtools'
+require 'thread'
+require $workingdir + '/ftp_fetcher.rb'
 
 class Logger
   #Change the logging format to include a timestamp
@@ -32,7 +34,7 @@ module EchiConverter
     end
     begin
       ActiveRecord::Base.establish_connection(YAML::load(File.open(databaseconfig))) 
-      @log.info "Successfully connected to the database"
+      @log.info "Initialized the database"
     rescue => err
       @log.fatal "Could not connect to the database - " + err
       send_email_alert "DATABASE"
@@ -97,6 +99,70 @@ module EchiConverter
     return directory_month
   end
   
+  #Method to get FTP files
+  def get_ftp_files
+    filelist_fetcher = FtpFetcher.new
+    filequeue = filelist_fetcher.fetch_list @log
+    
+    if filequeue == nil
+      return -1
+    end
+    
+    if $config["max_ftp_sessions"] > 1 && filequeue.length > 4
+      if $config["max_ftp_sessions"] > filequeue.length
+        @log.info "Using " + filequeue.length.to_s + " ftp sessions to fetch files"
+        my_threads = []
+        cnt = 0
+        while cnt < filequeue.length
+          my_threads << Thread.new do
+            fetcher = Fetcher.new
+            result = fetcher.fetch_ftp_files filequeue, @log
+          end
+          cnt += 1
+        end
+        my_threads.each { |aThread|  aThread.join }
+      else
+        @log.info "Using " + $config["max_ftp_sessions"].to_s + " ftp sessions to fetch files"
+        my_threads = []
+        cnt = 0
+        while cnt < $config["max_ftp_sessions"]
+          my_threads << Thread.new do
+            fetcher = FtpFetcher.new
+            result = fetcher.fetch_ftp_files filequeue, @log
+          end
+          cnt += 1
+        end
+        my_threads.each { |aThread|  aThread.join }
+      end
+    else
+      @log.info "Using a single ftp session to fetch the files"
+      fetcher = FtpFetcher.new
+      result = fetcher.fetch_ftp_files filequeue, @log
+    end
+    if result == false
+      send_email_alert "FTP"
+    end
+  end
+  
+  #Method to write to the log table
+  def log_processed_file type, filedata
+    begin 
+      echi_log = EchiLog.new
+      echi_log.filename = filedata["name"]
+      if type == 'BINARY'
+        echi_log.filenumber = filedata["number"]
+        echi_log.version = filedata["version"]
+      end
+      echi_log.records = filedata["cnt"]
+      echi_log.processedat = Time.now
+      echi_log.save
+    rescue => err
+      @log.info "Error creating ECHI_LOG entry - " + err
+      return -1
+    end
+    return 0
+  end
+  
   #Method for parsing the various datatypes from the ECH file
   def dump_binary type, length
     case type
@@ -149,288 +215,233 @@ module EchiConverter
     @log.debug "File_number " + filenumber.to_s
     fileversion = dump_binary 'int', 4
     @log.debug "Version " + fileversion.to_s
-
-    if $config["echi_process_log"] == "Y"
-      #Log the file
-      begin
-        echi_log = EchiLog.new
-        echi_log.filename = filename
-        echi_log.filenumber = filenumber
-        echi_log.version = fileversion
-      rescue => err
-        @log.info "Error creating ECHI_LOG entry - " + err
-      end
-    end
     
-    #Perform a transaction for each file, including the log table
-    #in order to commit as one atomic action upon success
-    EchiRecord.transaction do
-      bool_cnt = 0
-      @record_cnt = 0
-      while @binary_file.eof == FALSE do 
-        @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
-        echi_record = EchiRecord.new
-        @echi_schema["fields"].each do | field |
-          #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
-          if field["type"] == 'bool'
-            if bool_cnt == 0
-              bytearray = dump_binary field["type"], field["length"]
-            end
-            #Ensure we parse the bytearray and set the appropriate flags
-            #We need to make sure the entire array is not nil, in order to do Y/N
-            #if Nil we then set all no
-            if bytearray != nil
-              if bytearray.slice(bool_cnt,1) == 1
-                value = 'Y'
-              else
+    begin
+      #Perform a transaction for each file, including the log table
+      #in order to commit as one atomic action upon success
+      EchiRecord.transaction do
+        bool_cnt = 0
+        @record_cnt = 0
+        while @binary_file.eof == FALSE do 
+          @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+          echi_record = EchiRecord.new
+          @echi_schema["echi_records"].each do | field |
+            #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
+            if field["type"] == 'bool'
+              if bool_cnt == 0
+                bytearray = dump_binary field["type"], field["length"]
+              end
+              #Ensure we parse the bytearray and set the appropriate flags
+              #We need to make sure the entire array is not nil, in order to do Y/N
+              #if Nil we then set all no
+              if bytearray != nil
+                if bytearray.slice(bool_cnt,1) == 1
+                  value = 'Y'
+                else
+                  value = 'N'
+                end
+              else 
                 value = 'N'
               end
-            else 
-              value = 'N'
-            end
-            bool_cnt += 1
-            if bool_cnt == 8
-              bool_cnt = 0
+              bool_cnt += 1
+              if bool_cnt == 8
+                bool_cnt = 0
+              end
+            else
+              #Process 'standard' fields
+              value = dump_binary field["type"], field["length"]
+              @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
             end
-          else
-            #Process 'standard' fields
-            value = dump_binary field["type"], field["length"]
-            @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
+            echi_record[field["name"]] = value
           end
-          echi_record[field["name"]] = value
-        end
-        echi_record.save
+          echi_record.save
       
-        #Scan past the end of line record
-        @binary_file.read(1)
-        @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
-        @record_cnt += 1
+          #Scan past the end of line record
+          @binary_file.read(1)
+          @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+          @record_cnt += 1
+        end
+        @binary_file.close
       end
-      @binary_file.close
+    rescue => err
+        @log.info "Error processing ECHI file - " + err
     end
-    
+          
     #Move the file to the processed directory
     FileUtils.mv(echi_file, @processeddirectory)
     
     if $config["echi_process_log"] == "Y"
-      #Finish logging the details on the file
-      begin
-        echi_log.records = @record_cnt
-        echi_log.processedat = Time.now
-        echi_log.save
-      rescue => err
-        @log.info "Error inserting ECHI_LOG entry - " + err
-      end
+      log_processed_file "BINARY", { "name" => filename, "number" => filenumber, "version" => fileversion, "cnt" => @record_cnt }
     end
     
     return @record_cnt
   end
-  
-  def connect_ftpsession
-    #Open ftp connection
-    begin
-      if $config["echi_connect_type"] == 'ftp'
-        ftp_session = Net::FTP.new($config["echi_host"])
-        ftp_session.login $config["echi_username"], $config["echi_password"]
-        @log.info "Successfully connected to the ECHI FTP server"
-      else
-        #Stub for possible SSH support in the future
-        #session = Net::SSH.start(config["echi_host"], config["echi_port"], config["echi_username"], config["echi_password"])
-        @log.fatal "SSH currently not supported, please use FTP for accessing the ECHI server"
-        exit
-      end
-    rescue => err
-      @log.fatal "Could not connect with the FTP server - " + err
-      send_email_alert "FTP"
-      return -1
-    end
-    return ftp_session
-  end
-  
-  #Connect to the ftp server and fetch the files each time
-  def fetch_ftp_files
-   attempts = 0
-   ftp_session = -1
-   while ftp_session == -1 do
-     ftp_session = connect_ftpsession
-     if ftp_session == -1
-       sleep 5
-     end
-     attempts += 1
-     if $config["echi_ftp_retry"] == attempts
-       ftp_session = 0
-     end
-   end
-   if ftp_session != 0
-     begin
-       if $config["echi_ftp_directory"] != nil
-         ftp_session.chdir($config["echi_ftp_directory"])
-       end
-       files = ftp_session.list('chr*')
-       
-       #Also fetch the agname.dat file if it is configured to be processed
-       if $config["echi_update_agent_data"] == "Y"
-         files = files + ftp_session.list("agname.dat")
-       end
-       
-       file_cnt = 0
-       files.each do | file |
-         file_data = file.split(' ')
-         
-         local_filename = $workingdir + '/../files/to_process/' + file_data[8]
-         ftp_session.getbinaryfile(file_data[8], local_filename)
-         if $config["echi_ftp_delete"] == 'Y'
-           begin
-             ftp_session.delete(file_data[8])
-           rescue => err
-             @log.fatal err
-           end
-         end
-         file_cnt += 1
-       end
-       ftp_session.close
-    rescue => err
-      @log.fatal "Could not fetch from ftp server - " + err
-    end
-   end
-   return 
-  end
 
   def process_ascii filename
     echi_file = $workingdir + "/../files/to_process/" + filename
   
-    if $config["echi_process_log"] == "Y"
-      #Log the file
-      begin
-        echi_log = EchiLog.new
-        echi_log.filename = filename
-        #echi_log.filenumber = filenumber
-        #echi_log.version = fileversion
-      rescue => err
-        @log.info "Error creating ECHI_LOG entry - " + err
-      end
-    end
-  
-    #Perform a transaction for each file, including the log table
-    #in order to commit as one atomic action upon success
-    EchiRecord.transaction do
-      @record_cnt = 0
-      FasterCSV.foreach(echi_file) do |row|
-        if row != nil
-          @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
-          echi_record = EchiRecord.new
-          cnt = 0
-          @echi_schema["fields"].each do | field |
-            if field["type"] == "bool" || field["type"] == "bool_int"
-              case row[cnt]
-              when "0"
-                echi_record[field["name"]] = "N"
-              when "1"
-                echi_record[field["name"]] = "Y"
-              end
-              @log.debug field["name"] + ' == ' + row[cnt]
-            else
-              echi_record[field["name"]] = row[cnt]
-              if row[cnt] != nil
+    begin
+      #Perform a transaction for each file, including the log table
+      #in order to commit as one atomic action upon success
+      EchiRecord.transaction do
+        @record_cnt = 0
+        FasterCSV.foreach(echi_file) do |row|
+          if row != nil
+            @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+            echi_record = EchiRecord.new
+            cnt = 0
+            @echi_schema["echi_records"].each do | field |
+              if field["type"] == "bool" || field["type"] == "bool_int"
+                case row[cnt]
+                when "0"
+                  echi_record[field["name"]] = "N"
+                when "1"
+                  echi_record[field["name"]] = "Y"
+                end
                 @log.debug field["name"] + ' == ' + row[cnt]
+              else
+                echi_record[field["name"]] = row[cnt]
+                if row[cnt] != nil
+                  @log.debug field["name"] + ' == ' + row[cnt]
+                end
               end
+              cnt += 1
             end
-            cnt += 1
+            echi_record.save
+            @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+            @record_cnt += 1
           end
-          echi_record.save
-          @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
-          @record_cnt += 1
         end
       end
+    rescue => err
+      @log.info "Error processing ECHI file - " + err
     end
   
     #Move the file to the processed directory
     FileUtils.mv(echi_file, @processeddirectory)
   
     if $config["echi_process_log"] == "Y"
-      #Finish logging the details on the file
-      begin
-        echi_log.records = @record_cnt
-        echi_log.processedat = Time.now
-        echi_log.save
-      rescue => err
-        @log.info "Error inserting ECHI_LOG entry - " + err
-      end
+      log_processed_file nil, { "name" => filename, "cnt" => @record_cnt }
     end
   
     return @record_cnt
   end
 
-  def insert_agent_data field
+  def insert_dat_data tablename, row
  
     begin
-      echi_agent = EchiAgent.new
-      echi_agent.group_id = field[0]
-      echi_agent.login_id = field[1]
-      echi_agent.name = field[2]
-      echi_agent.save
+      case tablename
+      when "echi_agents"
+        echi_dat_record = EchiAgent.new
+      when "echi_aux_reasons"
+        echi_dat_record = EchiAuxReason.new
+      when "echi_cwcs"
+        echi_dat_record = EchiCwc.new
+      when "echi_vdns"
+        echi_dat_record = EchiVdn.new
+      end
+      cnt = 0
+      @echi_schema[tablename].each do | field |
+        echi_dat_record[field["name"]] = row[cnt]
+        cnt += 1
+      end
+      echi_dat_record.save
     rescue => err
-      @log.info "Unable to insert agent record - " + err
+      @log.info "Unable to insert " + tablename + " file record - " + err
     end
  
   end
   
-  #Method to insert data into 'echi_agents' based on agname.dat
-  def process_agent_data
-    agent_file = $workingdir + "/../files/to_process/agname.dat"
-        
-      if File.exists?(agent_file)
-        EchiAgent.transaction do
-          @record_cnt = 0
-          File.open(agent_file).each do |row|
-            if row != nil
-              field = row.rstrip.split('|')
-              @log.debug '<====================START AGENT RECORD ' + @record_cnt.to_s + ' ====================>'
-              agent = EchiAgent.find(:first, :conditions => [ "login_id = ? AND group_id = ?", field[1], field[0]])
-              if agent != nil
-                if agent.name != field[2]
-                  agent.name = field[2]
-                  agent.update
-                  @record_cnt += 1
-                  @log.debug "Updated record - " + field.inspect
-                else
-                  @log.debug "No update required for - " + field.inspect
-                end
-              else
-                insert_agent_data field
-                @record_cnt += 1
-                @log.debug "Inserted new record - " + field.inspect
-              end
-            end
-            @log.debug '<====================STOP AGENT RECORD ' + @record_cnt.to_s + ' ====================>'
-          end
+  #Process the appropriate table name
+  def process_proper_table file
+    @record_cnt = 0
+    process_file = File.open(file["filename"])
+    process_file.each do |row|
+      if row != nil
+        field = row.rstrip.split('|')
+        @log.debug '<====================START ' + file["name"] + ' RECORD ' + @record_cnt.to_s + ' ====================>'
+        case file["name"]
+        when "echi_agents"
+          record = EchiAgent.find(:first, :conditions => [ "login_id = ? AND group_id = ?", field[1], field[0]])
+        when "echi_aux_reasons"
+          record = EchiAuxReason.find(:first, :conditions => [ "aux_reason = ? AND group_id = ?", field[1], field[0]])
+        when "echi_cwcs"
+          record = EchiCwc.find(:first, :conditions => [ "cwc = ? AND group_id = ?", field[1], field[0]])
+        when "echi_vdns"
+          record = EchiVdn.find(:first, :conditions => [ "vdn = ? AND group_id = ?", field[1], field[0]])
         end
-        @agent_file_processed = Time.now
-        #Move the file to the processed directory
-        begin
-          agname_new_filename = "agname_" + UUID.timestamp_create.to_s + ".dat"
-          target_file = @processeddirectory + "/" + agname_new_filename
-          FileUtils.mv(agent_file, target_file)
-        rescue => err
-          @log.info "Issue with agname_*.dat filename - " + err
+        if record != nil
+          if record.name != field[2]
+            record.name = field[2]
+            record.update
+            @record_cnt += 1
+            @log.debug "Updated record - " + field.inspect
+          else
+            @log.debug "No update required for - " + field.inspect
+          end
+        else
+          insert_dat_data file["name"], field
+          @record_cnt += 1
+          @log.debug "Inserted new record - " + field.inspect
         end
-        if $config["echi_process_log"] == "Y"
-          #Log the file
-          begin
-            echi_log = EchiLog.new
-            echi_log.filename = agname_new_filename
-            #echi_log.filenumber = filenumber
-            #echi_log.version = fileversion
-            #Finish logging the details on the file
-            echi_log.records = @record_cnt
-            echi_log.processedat = Time.now
-            echi_log.save
-          rescue => err
-            @log.info "Error creating ECHI_LOGS entry - " + err
+      end
+      @log.debug '<====================STOP ' + file["name"] + ' RECORD ' + @record_cnt.to_s + ' ====================>'
+    end
+    process_file.close
+    
+    case file["name"]
+    when "echi_agents"
+      filename_elements = $config["echi_agent_dat"].split(".")
+    when "echi_aux_reasons"
+      filename_elements = $config["echi_aux_rsn_dat"].split(".")
+    when "echi_cwcs"
+      filename_elements = $config["echi_cwc_dat"].split(".")
+    when "echi_vdns"
+      filename_elements = $config["echi_vdn_dat"].split(".")
+    end
+    new_filename = filename_elements[0] + "_" + UUID.timestamp_create.to_s + "." + filename_elements[1]
+    target_file = @processeddirectory + "/" + new_filename
+    begin
+      FileUtils.mv(file["filename"], target_file)
+      if $config["echi_process_log"] == "Y"
+        log_processed_file nil, { "name" => new_filename, "cnt" => @record_cnt }
+      end
+    rescue => err
+      @log.info "Unable to move processed file - " + err
+    end
+  end
+  
+  #Method to insert data into 'echi_agents' based on agname.dat
+  def process_dat_files
+    dat_files = Array.new
+    dat_files[0] = { "name" => "echi_agents", "filename" => $workingdir + "/../files/to_process/"  + $config["echi_agent_dat"] }
+    dat_files[1] = { "name" => "echi_aux_reasons", "filename" => $workingdir + "/../files/to_process/"  + $config["echi_aux_rsn_dat"] }
+    dat_files[2] = { "name" =>"echi_cwcs", "filename" => $workingdir + "/../files/to_process/"  + $config["echi_cwc_dat"] }
+    dat_files[3] = { "name" =>"echi_vdns", "filename" => $workingdir + "/../files/to_process/"  + $config["echi_vdn_dat"] }
+    
+    dat_files.each do |file|
+      if File.exists?(file["filename"])
+        case file["name"]
+        when "echi_agents"
+          EchiAgent.transaction do
+            process_proper_table file
+          end
+        when "echi_aux_reasons"
+          EchiAuxReason.transaction do
+            process_proper_table file
+          end
+        when "echi_cwcs"
+          EchiCwc.transaction do
+            process_proper_table file
+          end
+        when "echi_vdns"
+          EchiVdn.transaction do
+            process_proper_table file
           end
         end
       end
     end
+  end
 
   require $workingdir + '/echi-converter/version.rb'
 end