echi-converter 0.1.1 → 0.2.0

data/History.txt

@@ -61,8 +61,21 @@
 
 * Major enhancement(s):
 * Minor enhancement(s):
-  * Removed some extraneous comments to the log and added others
+  * Removed some extraneous comments to the log
   * Added 'standard' schema definitions for revision 12 & 13 to the config directory
 * Bug fix(es):
   * Fixed bug #14020 issue with binary offset
-  * In binary handling properly insert a Y/N for the booleans
+  * In binary handling properly insert a Y/N for the booleans
+
+== 0.2.0 2007-10-21
+
+* Major enhancement(s):
+  * FR#14853 - Added support for ActiveRecord transactions on a per file basis
+* Minor enhancement(s):
+  * FR#14855 - Store processed files in a year/month directory structure based on processed date (ie - 2007/10)
+  * If the 'echi_ftp_directory' option in application.yml is left blank/nil, the system will not attempt a change directory
+  * The system now builds the files to process array from the to_process directory instead of the ftp list, to allow for better recovery if a failure occurs
+  * FR#14997 - Added support for sending email alerts if ECHI fails to connect to either the database or an ftp server
+* Bug fix(es):
+  * Fixed bug #14585 requiring Hoe for the install
+  * Will now properly toggle between the standard schema and the Presence schema (ie - override ActiveRecord defaults)

data/Manifest.txt

@@ -6,6 +6,7 @@ Rakefile
 lib/main.rb
 lib/echi-converter/version.rb
 lib/database.rb
+lib/database_presence.rb
 lib/main.rb
 lib/echi-converter.rb
 scripts/txt2html

data/Rakefile

@@ -79,6 +79,7 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
     ['activesupport', '>=1.4.2'],
     ['daemons', '>=1.0.7'],
     ['fastercsv', '>=1.2.0'],
+    ['hoe', '>=1.2.2'],
     ['rake', '>=0.7.3']
   ]
   #p.spec_extras = {}    # A hash of extra values to set in the gemspec.

data/config/application.yml

@@ -3,10 +3,10 @@
 #Connection details for the Avaya CMS/ECHI host
 echi_host:      localhost
 echi_port:      22
-echi_username:  your_uname
-echi_password:  your_passwd
+echi_username:  anonymous
+echi_password:  
 echi_connect_type:  ftp #only ftp supported now, possible for ssh in the future
-echi_ftp_directory:  /Users/ftp/anonymous
+echi_ftp_directory:  #/Users/ftp/anonymous #If blank/nil, the system will not do a CD at the start
 echi_ftp_retry: 3
 echi_ftp_delete:  N #to not delete the files off of the FTP server set to N
 echi_schema: extended_version13.yml
@@ -35,6 +35,12 @@ log_number: 10
 #The size of each individual log file
 log_length: 10240000
 
+#Email address to send alerts to, specifically failures to connect to the DB or FTP servers
+#currently does not support an SMTP server that requires authentication
+alert_email_address: jgoecke@presenceco.com
+smtp_server: mail.presenceco.com
+smtp_port: 25
+
 #Special settings for a specific application database
 pco_process: N #Set to yes or no if running with a PCO database
 pco_record_tablename: PCO_ECHIRECORD

data/config/install_files.yml

@@ -34,6 +34,7 @@ files_to_copy:
     - name: examples/extended_version12/chr0003
     - name: examples/extended_version12/chr0003.txt
     - name: lib/database.rb
+    - name: lib/database_presence.rb
     - name: lib/echi-converter.rb
     - name: lib/main.rb
     - name: lib/echi-converter/version.rb
@@ -41,6 +42,7 @@ files_to_copy:
     
 files_to_upgrade:
     - name: lib/database.rb
+    - name: lib/database_presence.rb
     - name: lib/echi-converter.rb
     - name: lib/main.rb
     - name: lib/echi-converter/version.rb

data/lib/database.rb

@@ -1,25 +1,5 @@
 class EchiRecord < ActiveRecord::Base
-  #Determine our working directory
-  workingdirectory = File.expand_path(File.dirname(__FILE__))
-  #Open the configuration file
-  configfile = workingdirectory + '/../config/application.yml' 
-  config = YAML::load(File.open(configfile))
-  
-  if config["pco_process"] == 'Y'
-    set_table_name config["pco_record_tablename"]
-    set_sequence_name config["pco_record_seqname"]
-  end
 end
 
 class EchiLog < ActiveRecord::Base
-  #Determine our working directory
-  workingdirectory = File.expand_path(File.dirname(__FILE__))
-  #Open the configuration file
-  configfile = workingdirectory + '/../config/application.yml' 
-  config = YAML::load(File.open(configfile))
-  
-  if config["pco_process"] == 'Y'
-    set_table_name @config["pco_log_tablename"]
-    set_sequence_name @config["pco_log_seqname"]
-  end
 end

data/lib/database_presence.rb

@@ -0,0 +1,9 @@
+class EchiRecord < ActiveRecord::Base
+  set_table_name config["pco_record_tablename"]
+  set_sequence_name config["pco_record_seqname"]
+end
+
+class EchiLog < ActiveRecord::Base
+  set_table_name @config["pco_log_tablename"]
+  set_sequence_name @config["pco_log_seqname"]
+end

data/lib/echi-converter.rb

@@ -2,6 +2,7 @@ require 'rubygems'
 require 'active_record'
 require 'faster_csv'
 require 'net/ftp'
+require 'net/smtp'
 require 'fileutils'
 
 class Logger
@@ -34,6 +35,7 @@ module EchiConverter
       @log.info "Successfully connected to the database"
     rescue => err
       @log.fatal "Could not connect to the database - " + err
+      send_email_alert "DATABASE"
     end
   end
   
@@ -55,6 +57,46 @@ module EchiConverter
     end
   end
   
+  #Method to send alert emails
+  def send_email_alert reason
+    begin
+      Net::SMTP.start(@config["smtp_server"], @config["smtp_port"]) do |smtp|
+        smtp.open_message_stream('donotreply@echi-converter.rubyforge.org', [@config["alert_email_address"]]) do |f|
+          f.puts "From: donotreply@echi-converter.rubyforge.org"
+          f.puts "To: " + @config['alert_email_address']
+          f.puts "Subject: ECHI-Converter Failure"
+          case reason 
+          when "DATABASE"
+            f.puts "Failed to connect to the database."
+          when "FTP"
+            f.puts "Failed to connect to the ftp server."
+          end
+            f.puts " "
+            f.puts "Please check the ECHI-Converter environment as soon as possible."
+        end
+      end
+    rescue => err
+      @log.warn err
+    end
+  end
+  
+  #Set the working directory to copy processed files to, if it does not exist creat it
+  #Directory names based on year/month so as not to exceed 5K files in a single directory
+  def set_directory working_directory
+    time = Time.now
+    directory_year = working_directory + "/../files/processed/" + time.year.to_s 
+    directory_month = directory_year + "/" + time.month.to_s
+    
+    if File.exists?(directory_month) == false
+      if File.exists?(directory_year) == false
+        Dir.mkdir(directory_year)
+      end
+      Dir.mkdir(directory_month)
+    end      
+    
+    return directory_month
+  end
+  
   #Method for parsing the various datatypes from the ECH file
   def dump_binary type, length
     case type
@@ -116,61 +158,64 @@ module EchiConverter
       echi_log.version = fileversion
     end
     
-    bool_cnt = 0
-    record_cnt = 0
-    while @binary_file.eof == FALSE do 
-      @log.debug '<====================START RECORD ' + record_cnt.to_s + ' ====================>'
-      echi_record = EchiRecord.new
-      @echi_schema["fields"].each do | field |
-        #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
-        if field["type"] == 'bool'
-          if bool_cnt == 0
-            bytearray = dump_binary field["type"], field["length"]
-          end
-          #Ensure we parse the bytearray and set the appropriate flags
-          #We need to make sure the entire array is not nil, in order to do Y/N
-          #if Nil we then set all no
-          if bytearray != nil
-            if bytearray.slice(bool_cnt,1) == 1
-              value = 'Y'
-            else
+    #Perform a transaction for each file, including the log table
+    #in order to commit as one atomic action upon success
+    EchiRecord.transaction do
+      bool_cnt = 0
+      @record_cnt = 0
+      while @binary_file.eof == FALSE do 
+        @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+        echi_record = EchiRecord.new
+        @echi_schema["fields"].each do | field |
+          #We handle the 'boolean' fields differently, as they are all encoded as bits in a single 8-bit byte
+          if field["type"] == 'bool'
+            if bool_cnt == 0
+              bytearray = dump_binary field["type"], field["length"]
+            end
+            #Ensure we parse the bytearray and set the appropriate flags
+            #We need to make sure the entire array is not nil, in order to do Y/N
+            #if Nil we then set all no
+            if bytearray != nil
+              if bytearray.slice(bool_cnt,1) == 1
+                value = 'Y'
+              else
+                value = 'N'
+              end
+            else 
               value = 'N'
             end
-          else 
-            value = 'N'
-          end
-          bool_cnt += 1
-          if bool_cnt == 8
-            bool_cnt = 0
+            bool_cnt += 1
+            if bool_cnt == 8
+              bool_cnt = 0
+            end
+          else
+            #Process 'standard' fields
+            value = dump_binary field["type"], field["length"]
+            @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
           end
-        else
-          #Process 'standard' fields
-          value = dump_binary field["type"], field["length"]
-          @log.debug field["name"] + " { type => #{field["type"]} & length => #{field["length"]} } value => " + value.to_s
+          echi_record[field["name"]] = value
         end
-        echi_record[field["name"]] = value
-      end
-      echi_record.save
+        echi_record.save
       
-      #Scan past the end of line record
-      @binary_file.read(1)
-      @log.debug '<====================STOP RECORD ' + record_cnt.to_s + ' ====================>'
-      record_cnt += 1
+        #Scan past the end of line record
+        @binary_file.read(1)
+        @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+        @record_cnt += 1
+      end
+      @binary_file.close
     end
-    @binary_file.close
     
     #Move the file to the processed directory
-    destination_directory = @workingdirectory + '/../files/processed/'
-    FileUtils.mv(echi_file, destination_directory)
+    FileUtils.mv(echi_file, @processeddirectory)
     
     if @config["echi_process_log"] == "Y"
       #Finish logging the details on the file
-      echi_log.records = record_cnt
+      echi_log.records = @record_cnt
       echi_log.processed_at = Time.now
       echi_log.save
     end
     
-    return record_cnt
+    return @record_cnt
   end
   
   def connect_ftpsession
@@ -188,6 +233,7 @@ module EchiConverter
       end
     rescue => err
       @log.fatal "Could not connect with the FTP server - " + err
+      send_email_alert "FTP"
       return -1
     end
     return ftp_session
@@ -210,7 +256,9 @@ module EchiConverter
    end
    if ftp_session != 0
      begin
-       ftp_session.chdir(@config["echi_ftp_directory"])
+       if @config["echi_ftp_directory"] != nil
+         ftp_session.chdir(@config["echi_ftp_directory"])
+       end
        files = ftp_session.list('chr*')
        file_cnt = 0
        files.each do | file |
@@ -249,47 +297,50 @@ def process_ascii filename
     #echi_log.version = fileversion
   end
   
-  record_cnt = 0
-  FasterCSV.foreach(echi_file) do |row|
-    if row != nil
-      @log.debug '<====================START RECORD ' + record_cnt.to_s + ' ====================>'
-      echi_record = EchiRecord.new
-      cnt = 0
-      @echi_schema["fields"].each do | field |
-        if field["type"] == "bool" || field["type"] == "bool_int"
-          case row[cnt]
-          when "0"
-            echi_record[field["name"]] = "N"
-          when "1"
-            echi_record[field["name"]] = "Y"
-          end
-          @log.debug field["name"] + ' == ' + row[cnt]
-        else
-          echi_record[field["name"]] = row[cnt]
-          if row[cnt] != nil
+  #Perform a transaction for each file, including the log table
+  #in order to commit as one atomic action upon success
+  EchiRecord.transaction do
+    @record_cnt = 0
+    FasterCSV.foreach(echi_file) do |row|
+      if row != nil
+        @log.debug '<====================START RECORD ' + @record_cnt.to_s + ' ====================>'
+        echi_record = EchiRecord.new
+        cnt = 0
+        @echi_schema["fields"].each do | field |
+          if field["type"] == "bool" || field["type"] == "bool_int"
+            case row[cnt]
+            when "0"
+              echi_record[field["name"]] = "N"
+            when "1"
+              echi_record[field["name"]] = "Y"
+            end
             @log.debug field["name"] + ' == ' + row[cnt]
+          else
+            echi_record[field["name"]] = row[cnt]
+            if row[cnt] != nil
+              @log.debug field["name"] + ' == ' + row[cnt]
+            end
           end
+          cnt += 1
         end
-        cnt += 1
+        echi_record.save
+        @log.debug '<====================STOP RECORD ' + @record_cnt.to_s + ' ====================>'
+        @record_cnt += 1
       end
-      echi_record.save
-      @log.debug '<====================STOP RECORD ' + record_cnt.to_s + ' ====================>'
-      record_cnt += 1
     end
   end
   
   #Move the file to the processed directory
-  destination_directory = @workingdirectory + '/../files/processed/' + filename
-  FileUtils.mv(echi_file, destination_directory)
+  FileUtils.mv(echi_file, @processeddirectory)
   
   if @config["echi_process_log"] == "Y"
     #Finish logging the details on the file
-    echi_log.records = record_cnt
+    echi_log.records = @record_cnt
     echi_log.processed_at = Time.now
     echi_log.save
   end
   
-  return record_cnt
+  return @record_cnt
 end
 
 require @workingdirectory + '/echi-converter/version.rb'

data/lib/echi-converter/version.rb

@@ -1,8 +1,8 @@
 module EchiConverter #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 1
-    TINY  = 1
+    MINOR = 2
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/main.rb

@@ -11,7 +11,11 @@ configfile = @workingdirectory + '/../config/application.yml'
 @config = YAML::load(File.open(configfile))
 
 #Load ActiveRecord Models
-require @workingdirectory + '/database.rb'
+if @config["pco_process"] == 'Y'
+  require @workingdirectory + '/database_presence.rb'
+else
+  require @workingdirectory + '/database.rb'
+end
 
 #Load the configured schema
 schemafile = @workingdirectory + "/../config/" + @config["echi_schema"]
@@ -29,12 +33,21 @@ end
 
 loop do
   #Process the files
-  files = fetch_ftp_files
-  files.each do | file |
-    if @config["echi_format"] == 'BINARY'
-      record_cnt = convert_binary_file file
-    elsif @config["echi_format"] == 'ASCII'
-      record_cnt = process_ascii file
+  ftp_files = fetch_ftp_files
+  #Grab filenames from the to_process directory after an FTP fetch, so if the 
+  #system fails it may pick up where it left off
+  to_process_dir = @workingdirectory + "/../files/to_process/"
+  
+  #Establish where to copy the processed files to
+  @processeddirectory = set_directory(@workingdirectory)
+  
+  Dir.entries(to_process_dir).each do | file |
+    if file != "." && file != ".."
+      if @config["echi_format"] == 'BINARY'
+        record_cnt = convert_binary_file file
+      elsif @config["echi_format"] == 'ASCII'
+        record_cnt = process_ascii file
+      end
     end
     @log.info "Processed file #{file} with #{record_cnt.to_s} records"
   end

data/website/index.html

@@ -33,7 +33,7 @@
     <h1>ECHI Converter</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/echi-converter"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/echi-converter" class="numbers">0.1.1</a>
+      <a href="http://rubyforge.org/projects/echi-converter" class="numbers">0.2.0</a>
     </div>
     <h1>&#x2192; &#8216;echi-converter&#8217;</h1>
 
@@ -61,7 +61,7 @@
 		<li>Generate your schema via ActiveRecord Migrations</li>
 		<li>Fetch Binary or <span class="caps">ASCII CSV</span> files from the Avaya <span class="caps">CMS</span> platform via <span class="caps">FTP</span></li>
 		<li>Convert from the defined Binary format to <span class="caps">ASCII</span></li>
-		<li>Insert the records into the defined database table</li>
+		<li>Insert the records into the defined database table using database transactions, via ActiveRecord, on a per file basis to support recovery on failure</li>
 		<li>Change schema structure via <span class="caps">YML</span> configuration file to accommodate various releases of the <span class="caps">ECHI</span> format</li>
 	</ol>
 
@@ -86,6 +86,7 @@
 		<li><a href="http://daemons.rubyforge.org/">Daemons v1.0.7+</a></li>
 		<li><a href="http://fastercsv.rubyforge.org/">FasterCSV v1.2.0+</a></li>
 		<li><a href="http://rake.rubyforge.org/">Rake v0.7.3+</a></li>
+		<li><a href="http://rubyforge.org/projects/seattlerb/">Hoe v1.2.2+</a></li>
 	</ol>
 
 
@@ -175,6 +176,12 @@
 	</ol>
 
 
+	<h2>Screencast</h2>
+
+
+	<p>You may view the screencast on howto install and use the <span class="caps">ECHI</span>-Converter <a href="http://www.screencast.com/t/lQQkIVkUZMr">here</a>.</p>
+
+
 	<h2>Forum</h2>
 
 
@@ -201,7 +208,7 @@
 
 	<p>Comments are welcome. Send an email to <a href="mailto:jason@goecke.net">jason [at] goecke.net</a>.</p>
     <p class="coda">
-      <a href="mailto:drnicwilliams@gmail.com">Dr Nic</a>, 17th September 2007<br>
+      <a href="mailto:drnicwilliams@gmail.com">Dr Nic</a>, 23rd October 2007<br>
       Theme extended from <a href="http://rb2js.rubyforge.org/">Paul Battley</a>
     </p>
 </div>

data/website/index.txt

@@ -20,7 +20,7 @@ The utility provides the following capabilities:
 	# Generate your schema via ActiveRecord Migrations
 	# Fetch Binary or ASCII CSV files from the Avaya CMS platform via FTP
 	# Convert from the defined Binary format to ASCII
-	# Insert the records into the defined database table
+	# Insert the records into the defined database table using database transactions, via ActiveRecord, on a per file basis to support recovery on failure
 	# Change schema structure via YML configuration file to accommodate various releases of the ECHI format
 
 # Table names:
@@ -37,6 +37,7 @@ h2. Requirements
 	# "Daemons v1.0.7+":http://daemons.rubyforge.org/
 	# "FasterCSV v1.2.0+":http://fastercsv.rubyforge.org/
 	# "Rake v0.7.3+":http://rake.rubyforge.org/
+    # "Hoe v1.2.2+":http://rubyforge.org/projects/seattlerb/
 
 
 h2. Installing
@@ -97,6 +98,9 @@ Items to be done to move this out of alpha stage:
 
 # 'echi-converter create' works fine with OSX/Linux, need to test validate with a Win32 platform, therefore currently recommended to only run on a Linux/OSX platform, could use "VMWare":http://www.vmware.com to do this on Windows.
 
+h2. Screencast
+
+You may view the screencast on howto install and use the ECHI-Converter "here":http://www.screencast.com/t/lQQkIVkUZMr.
 
 h2. Forum
 

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: echi-converter
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
-date: 2007-09-17 00:00:00 -07:00
+  version: 0.2.0
+date: 2007-10-23 00:00:00 -07:00
 summary: ECHI Conversion Utility - Provides a utility to fetch Avaya CMS / ECHI binary files, convert them and insert into a database table via ActiveRecord
 require_paths: 
 - lib
@@ -37,6 +37,7 @@ files:
 - lib/main.rb
 - lib/echi-converter/version.rb
 - lib/database.rb
+- lib/database_presence.rb
 - lib/echi-converter.rb
 - scripts/txt2html
 - setup.rb