echi-converter 0.0.2 → 0.0.3

data/History.txt

@@ -1,6 +1,6 @@
 == 0.0.1 2007-07-22
 
-* 1 major enhancement:
+* 1 major enhancement(s):
   * Initial release
   * Alpha phase
 
@@ -12,6 +12,17 @@
   * Changed BOOLEAN fields to char(1) with Y/N possible values (FR#12741)
   * Added the ability to toggle deleting or leaving the FTP files (FR#12702)
   * Removed some extraneous debug info
-* 1 bug fix:
-  * Bug # 12528 - Handle segstart/segstop dates properly
-  * Bug #12746 - Properly parse Bytearray for Boolean fields
+* 1 Bug fix(es):
+  * Bug #12528 - Handle segstart/segstop dates properly
+  * Bug #12746 - Properly parse Bytearray for Boolean fields
+
+== 0.0.3 2007-08-10
+
+* Major enhancement(s):
+  * Added support for import of ASCII CSV files (FR#12997)
+* Minor enhancement(s):
+  * Added support of processing 'chr' files only (FR#12996)
+  * Added a configuration element to turn logging of file processing to a table on/off
+  * Removed the net/ssh library include, as not using sftp for now
+* Bug fix(es):
+  * Added proper error handling when ftp user does not have delete rights

data/Manifest.txt

@@ -21,6 +21,7 @@ website/template.rhtml
 config/application.yml
 config/database.yml
 config/extended_version12.yml
+config/extended_version13.yml
 db/migrate/001_create_echi_records.rb
 db/migrate/002_create_echi_logs.rb
 bin/echi-converter-create

data/config/application.yml

@@ -8,8 +8,10 @@ echi_password:  sanfran2007!
 echi_connect_type:  ftp #only ftp supported now, possible for ssh in the future
 echi_ftp_directory:  /Users/ftp/anonymous
 echi_ftp_retry: 3
-echi_ftp_delete:  Y #to not delete the files off of the FTP server set to N
+echi_ftp_delete:  N #to not delete the files off of the FTP server set to N
 echi_schema: extended_version13.yml
+echi_format: ASCII #valid settings are ASCII or BINARY
+echi_process_log: N #valid is Y/N to turn it on or off
 
 #Currently only ftp supported, but may want to add UUCP/Serial later
 echi_xfer_type: ftp

data/config/extended_version13.yml

@@ -0,0 +1,234 @@
+#ECH File construct, order is important, as the application sequentially reads the file
+#Version 12 - Extended Schema
+fields:
+    - name: callid
+      type: int
+      length: 4
+    - name: acwtime
+      type: int
+      length: 4
+    - name: onholdtime
+      type: int
+      length: 4
+    - name: consulttime
+      type: int
+      length: 4
+    - name: disptime
+      type: int
+      length: 4
+    - name: duration
+      type: int
+      length: 4
+    - name: segstart
+      type: datetime
+      length: 4
+    - name: segstop
+      type: datetime
+      length: 4
+    - name: talktime
+      type: int
+      length: 4
+    - name: netintime
+      type: int
+      length: 4
+    - name: origholdtime
+      type: int
+      length: 4
+    - name: queuetime
+      type: int
+      length: 4
+    - name: ringtime
+      type: int
+      length: 4
+    - name: dispivector
+      type: int
+      length: 2
+    - name: dispsplit
+      type: int
+      length: 2
+    - name: firstivector
+      type: int
+      length: 2
+    - name: split1
+      type: int
+      length: 2
+    - name: split2
+      type: int
+      length: 2
+    - name: split3
+      type: int
+      length: 2
+    - name: trunkgroup
+      type: int
+      length: 2
+    - name: tk_locid
+      type: int
+      length: 2
+    - name: orig_locid
+      type: int
+      length: 2
+    - name: answer_locid
+      type: int
+      length: 2
+    - name: obs_locid
+      type: int
+      length: 2
+    - name: uui_len
+      type: int
+      length: 2
+    - name: assist
+      type: bool
+      length: 1
+    - name: audio_difficulty
+      type: bool
+      length: 1
+    - name: conference
+      type: bool
+      length: 1
+    - name: da_queued
+      type: bool
+      length: 1
+    - name: hold_abn
+      type: bool
+      length: 1
+    - name: malicious
+      type: bool
+      length: 1
+    - name: observing_call
+      type: bool
+      length: 1
+    - name: transferred
+      type: bool
+      length: 1
+    - name: agent_released
+      type: bool_int
+      length: 1
+    - name: acdnum
+      type: int
+      length: 1
+    - name: call_disp
+      type: int
+      length: 1
+    - name: disppriority
+      type: int
+      length: 1
+    - name: holds
+      type: int
+      length: 1
+    - name: segment
+      type: int
+      length: 1
+    - name: ansreason
+      type: int
+      length: 1
+    - name: origreason
+      type: int
+      length: 1
+    - name: dispsklevel
+      type: int
+      length: 1
+    - name: events0
+      type: int
+      length: 1
+    - name: events1
+      type: int
+      length: 1
+    - name: events2
+      type: int
+      length: 1
+    - name: events3
+      type: int
+      length: 1
+    - name: events4
+      type: int
+      length: 1
+    - name: events5
+      type: int
+      length: 1
+    - name: events6
+      type: int
+      length: 1
+    - name: events7
+      type: int
+      length: 1
+    - name: events8
+      type: int
+      length: 1
+    - name: ucid
+      type: str
+      length: 21
+    - name: dispvdn
+      type: str
+      length: 8
+    - name: eqloc
+      type: str
+      length: 10
+    - name: firstvdn 
+      type: str
+      length: 8
+    - name: orig_logid
+      type: str
+      length: 10
+    - name: ans_logid
+      type: str
+      length: 10
+    - name: last_observer
+      type: str
+      length: 10
+    - name: dialed_number
+      type: str
+      length: 25
+    - name: calling_party
+      type: str
+      length: 13
+    - name: collect_digits
+      type: str
+      length: 17
+    - name: cwc_digits
+      type: str
+      length: 17
+    - name: calling_II
+      type: str
+      length: 3
+    - name: cwcs0
+      type: str
+      length: 17
+    - name: cwcs1
+      type: str
+      length: 17
+    - name: cwcs2
+      type: str
+      length: 17
+    - name: cwcs3
+      type: str
+      length: 17
+    - name: cwcs4
+      type: str
+      length: 17
+    - name: vdn2
+      type: str
+      length: 8
+    - name: vdn3
+      type: str
+      length: 8
+    - name: vdn4
+      type: str
+      length: 8
+    - name: vdn5
+      type: str
+      length: 8
+    - name: vdn6
+      type: str
+      length: 8
+    - name: vdn7
+      type: str
+      length: 8
+    - name: vdn8
+      type: str
+      length: 8
+    - name: vdn9
+      type: str
+      length: 8
+    - name: asaiuui
+      type: str
+      length: 96

data/lib/echi-converter.rb

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'active_record'
 require 'faster_csv'
 require 'net/ftp'
-require 'net/ssh'
+#require 'net/ssh'
 #require 'net/sftp'
 require 'fileutils'
 
@@ -110,11 +110,13 @@ module EchiConverter
     fileversion = dump_binary 'int', 4
     @log.debug "Version " + fileversion.to_s
 
-    #Log the file
-    echi_log = EchiLog.new
-    echi_log.filename = filename
-    echi_log.filenumber = filenumber
-    echi_log.version = fileversion
+    if @config["echi_process_log"] == "Yes"
+      #Log the file
+      echi_log = EchiLog.new
+      echi_log.filename = filename
+      echi_log.filenumber = filenumber
+      echi_log.version = fileversion
+    end
     
     bool_cnt = 0
     record_cnt = 0
@@ -158,10 +160,12 @@ module EchiConverter
     destination_directory = @workingdirectory + '/../files/processed/'
     FileUtils.mv(echi_file, destination_directory)
     
-    #Finish logging the details on the file
-    echi_log.records = record_cnt
-    echi_log.processed_at = Time.now
-    echi_log.save
+    if @config["echi_process_log"] == "Yes"
+      #Finish logging the details on the file
+      echi_log.records = record_cnt
+      echi_log.processed_at = Time.now
+      echi_log.save
+    end
     
     return record_cnt
   end
@@ -214,7 +218,11 @@ module EchiConverter
          ftp_session.getbinaryfile(remote_filename, local_filename)
          files_to_process[file_cnt] = remote_filename
          if @config["echi_ftp_delete"] == 'Y'
-           ftp_session.delete(remote_filename)
+           begin
+             ftp_session.delete(remote_filename)
+           rescue => err
+             @log.fatal err
+           end
          end
          file_cnt += 1
        end
@@ -227,4 +235,54 @@ module EchiConverter
   end
 end
 
+def process_ascii filename
+  echi_file = @workingdirectory + "/../files/to_process/" + filename
+  
+  if @config["echi_process_log"] == "Yes"
+    #Log the file
+    echi_log = EchiLog.new
+    echi_log.filename = filename
+    echi_log.filenumber = filenumber
+    echi_log.version = fileversion
+  end
+  
+  record_cnt = 0
+  FasterCSV.foreach(echi_file) do |row|
+    if row != nil
+      @log.debug '<====================START RECORD====================>'
+      echi_record = EchiRecord.new
+      cnt = 0
+      @echi_schema["fields"].each do | field |
+        if field["type"] == "bool" || field["type"] == "bool_int"
+          case row[cnt]
+          when "0"
+            echi_record[field["name"]] = "N"
+          when "1"
+            echi_record[field["name"]] = "Y"
+          end
+          @log.debug field["name"] + ' == ' + row[cnt]
+        else
+          echi_record[field["name"]] = row[cnt]
+          if row[cnt] != nil
+            @log.debug field["name"] + ' == ' + row[cnt]
+          end
+        end
+        cnt += 1
+      end
+      echi_record.save
+      @log.debug '<====================STOP RECORD====================>'
+      record_cnt += 1
+    end
+  end
+  
+  if @config["echi_process_log"] == "Yes"
+    #Finish logging the details on the file
+    echi_log.records = record_cnt
+    echi_log.processed_at = Time.now
+    echi_log.save
+  end
+  
+  return record_cnt
+end
+
 require @workingdirectory + '/echi-converter/version.rb'

data/lib/echi-converter/version.rb

@@ -2,7 +2,7 @@ module EchiConverter #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 0
-    TINY  = 2
+    TINY  = 3
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/main.rb

@@ -32,7 +32,11 @@ loop do
   #Process the files
   files = fetch_ftp_files
   files.each do | file |
-    record_cnt = convert_binary_file file
+    if @config["echi_format"] == 'BINARY'
+      record_cnt = convert_binary_file file
+    elsif @config["echi_format"] == 'ASCII'
+      record_cnt = process_ascii file
+    end
     @log.info "Processed file #{file} with #{record_cnt.to_s} records"
   end
 

data/website/index.html

@@ -33,7 +33,7 @@
     <h1>ECHI Converter</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/echi-converter"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/echi-converter" class="numbers">0.0.2</a>
+      <a href="http://rubyforge.org/projects/echi-converter" class="numbers">0.0.3</a>
     </div>
     <h1>&#x2192; &#8216;echi-converter&#8217;</h1>
 
@@ -59,7 +59,7 @@
 	<ol>
 	<li>Support of ActiveRecord (means you may use Oracle, MySQL, MS-SQL, Postgres, <span class="caps">DB2</span>, etc)</li>
 		<li>Generate your schema via ActiveRecord Migrations</li>
-		<li>Fetch Binary files from the Avaya <span class="caps">CMS</span> platform via <span class="caps">FTP</span></li>
+		<li>Fetch Binary or <span class="caps">ASCII CSV</span> files from the Avaya <span class="caps">CMS</span> platform via <span class="caps">FTP</span></li>
 		<li>Convert from the defined Binary format to <span class="caps">ASCII</span></li>
 		<li>Insert the records into the defined database table</li>
 		<li>Change schema structure via <span class="caps">YML</span> configuration file to accommodate various releases of the <span class="caps">ECHI</span> format</li>
@@ -176,6 +176,7 @@
 
 	<ol>
 	<li>&#8216;echi-converter-create&#8217; works fine with <span class="caps">OSX</span>/Linux, need to test validate with a Win32 platform</li>
+		<li>Determine what to do with the files on the <span class="caps">FTP</span> server of the Avaya <span class="caps">CMS</span>, delete them or leave them and add methods to ensure no duplication</li>
 	</ol>
 
 
@@ -205,7 +206,7 @@
 
 	<p>Comments are welcome. Send an email to <a href="mailto:jason@goecke.net">jason [at] goecke.net</a>.</p>
     <p class="coda">
-      <a href="mailto:drnicwilliams@gmail.com">Dr Nic</a>, 2nd August 2007<br>
+      <a href="mailto:drnicwilliams@gmail.com">Dr Nic</a>, 11th August 2007<br>
       Theme extended from <a href="http://rb2js.rubyforge.org/">Paul Battley</a>
     </p>
 </div>

data/website/index.txt

@@ -18,7 +18,7 @@ The utility provides the following capabilities:
 	
 	# Support of ActiveRecord (means you may use Oracle, MySQL, MS-SQL, Postgres, DB2, etc)
 	# Generate your schema via ActiveRecord Migrations
-	# Fetch Binary files from the Avaya CMS platform via FTP
+	# Fetch Binary or ASCII CSV files from the Avaya CMS platform via FTP
 	# Convert from the defined Binary format to ASCII
 	# Insert the records into the defined database table
 	# Change schema structure via YML configuration file to accommodate various releases of the ECHI format
@@ -95,6 +95,7 @@ h2. ToDo
 Items to be done to move this out of alpha stage:
 
 # 'echi-converter-create' works fine with OSX/Linux, need to test validate with a Win32 platform
+# Determine what to do with the files on the FTP server of the Avaya CMS, delete them or leave them and add methods to ensure no duplication
 
 
 h2. Forum

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: echi-converter
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
-date: 2007-08-02 00:00:00 -07:00
+  version: 0.0.3
+date: 2007-08-11 00:00:00 -07:00
 summary: ECHI Conversion Utility - Provides a utility to fetch Avaya CMS / ECHI binary files, convert them and insert into a database table via ActiveRecord
 require_paths: 
 - lib
@@ -51,6 +51,7 @@ files:
 - config/application.yml
 - config/database.yml
 - config/extended_version12.yml
+- config/extended_version13.yml
 - db/migrate/001_create_echi_records.rb
 - db/migrate/002_create_echi_logs.rb
 - bin/echi-converter-create