dbf 0.3.0 → 0.4.0

data/lib/dbf/reader.rb

@@ -1,21 +1,28 @@
-require 'rubygems'
-require 'breakpoint'
 module DBF
   
   DBF_HEADER_SIZE = 32
   FPT_HEADER_SIZE = 512
   FPT_BLOCK_HEADER_SIZE = 8
   DATE_REGEXP = /([\d]{4})([\d]{2})([\d]{2})/
-  VERSION_DESCRIPTIONS = {"02" => "FoxBase", "03" => "dBase III without memo file", "04" => "dBase IV without memo file",
-    "05" => "dBase V without memo file", "30" => "Visual FoxPro", "31" => "Visual FoxPro with AutoIncrement field",
-    "7b" => "dBase IV with memo file", "83" => "dBase III with memo file", "8b" => "dBase IV with memo file",
-    "8e" => "dBase IV with SQL table", "f5" => "FoxPro with memo file", "fb" => "FoxPro without memo file"}
+  VERSION_DESCRIPTIONS = {
+    "02" => "FoxBase",
+    "03" => "dBase III without memo file",
+    "04" => "dBase IV without memo file",
+    "05" => "dBase V without memo file",
+    "30" => "Visual FoxPro",
+    "31" => "Visual FoxPro with AutoIncrement field",
+    "7b" => "dBase IV with memo file",
+    "83" => "dBase III with memo file",
+    "8b" => "dBase IV with memo file",
+    "8e" => "dBase IV with SQL table",
+    "f5" => "FoxPro with memo file",
+    "fb" => "FoxPro without memo file"
+  }
     
   class DBFError < StandardError; end
   class UnpackError < DBFError; end
   
   class Reader
-    
     attr_reader :field_count
     attr_reader :fields
     attr_reader :record_count
@@ -24,7 +31,7 @@ module DBF
     
     def initialize(file)
       @data_file = File.open(file, 'rb')
-      @memo_file = File.open(file.gsub(/dbf$/i, 'fpt'), 'rb') rescue File.open(file.gsub(/dbf$/i, 'FPT'), 'rb') rescue nil
+      @memo_file = open_memo(file)
       reload!
     end
     
@@ -38,6 +45,17 @@ module DBF
       @memo_file ? true : false
     end
     
+    def open_memo(file)
+      %w(fpt FPT dbt DBT).each do |extension|
+        filename = file.sub(/dbf$/i, extension)
+        if File.exists?(filename)
+          @memo_file_format = extension.downcase.to_sym
+          return File.open(filename)
+        end
+      end
+      nil
+    end
+    
     def field(field_name)
       @fields.detect {|f| f.name == field_name.to_s}
     end
@@ -45,23 +63,42 @@ module DBF
     def memo(start_block)
       @memo_file.rewind
       @memo_file.seek(start_block * @memo_block_size)
-      memo_type, memo_size, memo_string = @memo_file.read(@memo_block_size).unpack("NNa56")
-      if memo_size > @memo_block_size - FPT_BLOCK_HEADER_SIZE
-        memo_string << @memo_file.read(memo_size - @memo_block_size + FPT_BLOCK_HEADER_SIZE)
+      if @memo_file_format == :fpt
+        memo_type, memo_size, memo_string = @memo_file.read(@memo_block_size).unpack("NNa56")
+        if memo_size > @memo_block_size - FPT_BLOCK_HEADER_SIZE
+          memo_string << @memo_file.read(memo_size - @memo_block_size + FPT_BLOCK_HEADER_SIZE)
+        end
+      else
+        if version == "83" # dbase iii
+          memo_string = ""
+          loop do
+            memo_string << block = @memo_file.read(512)
+            break if block.strip.size < 512
+          end
+        end
       end
       memo_string
     end
     
+    # An array of all the records contained in the database file
     def records
       seek_to_record(0)
-      @records ||= Array.new(@record_count) { build_record if active_record? }
+      @records ||= Array.new(@record_count) do |i|
+        if active_record?
+          build_record
+        else
+          seek_to_record(i + 1)
+          nil
+        end
+      end
     end
     
     alias_method :rows, :records
     
+    # Jump to record
     def record(index)
       seek_to_record(index)
-      build_record if active_record?
+      active_record? ? build_record : nil
     end
     
     alias_method :row, :record
@@ -81,14 +118,16 @@ module DBF
       @fields.each do |field| 
         case field.type
         when 'N' # number
-          if field.decimal == 0
-            record[field.name] = unpack_integer(field) rescue nil
-          else
-            record[field.name] = unpack_float(field) rescue nil
-          end
+          record[field.name] = field.decimal == 0 ? unpack_integer(field) : unpack_float(field) rescue nil
         when 'D' # date
           raw = unpack_string(field).to_s.strip
-          record[field.name] = raw.empty? ? nil : Date.new(*raw.match(DATE_REGEXP).to_a.slice(1,3).map {|n| n.to_i}) rescue nil
+          unless raw.empty?
+            begin
+              record[field.name] = Time.gm(*raw.match(DATE_REGEXP).to_a.slice(1,3).map {|n| n.to_i})
+            rescue
+              record[field.name] = Date.new(*raw.match(DATE_REGEXP).to_a.slice(1,3).map {|n| n.to_i}) rescue nil
+            end
+          end
         when 'M' # memo
           starting_block = unpack_integer(field)
           record[field.name] = starting_block == 0 ? nil : memo(starting_block) rescue nil
@@ -108,12 +147,17 @@ module DBF
     end
     
     def get_field_descriptors
-      @fields = Array.new(@field_count) {|i| Field.new(*@data_file.read(32).unpack('a10xax4cc'))}
+      @fields = Array.new(@field_count) {|i| Field.new(*@data_file.read(32).unpack('a10xax4CC'))}
     end
     
     def get_memo_header_info
       @memo_file.rewind
-      @memo_next_available_block, @memo_block_size = @memo_file.read(FPT_HEADER_SIZE).unpack('Nxxn')
+      if @memo_file_format == :fpt
+        @memo_next_available_block, @memo_block_size = @memo_file.read(FPT_HEADER_SIZE).unpack('Nxxn')
+      else
+        @memo_block_size = 512
+        @memo_next_available_block = File.size(@memo_file.path) / @memo_block_size
+      end
     end
     
     def seek(offset)

data/test/common.rb

@@ -10,6 +10,10 @@ module CommonTests
       assert_equal @controls[:has_memo_file], @dbf.has_memo_file?
     end
     
+    def test_memo_file_format
+      assert_equal @controls[:memo_file_format], @dbf.instance_eval("@memo_file_format")
+    end
+    
     def test_records
       assert_kind_of Array, @dbf.records
       assert_kind_of Array, @dbf.rows
@@ -59,7 +63,7 @@ module CommonTests
     
     def test_date_fields
       @controls[:testable_date_field_names].each do |name|
-        assert(@dbf.records.any? {|record| record[name].is_a?(Date)})
+        assert(@dbf.records.any? {|record| record[name].is_a?(Date) || record[name].is_a?(Time)})
       end
     end
     
@@ -81,7 +85,7 @@ module CommonTests
 
     def test_memo_fields
       @controls[:testable_memo_field_names].each do |name|
-        assert(@dbf.records.any? {|record| record[name].is_a?(String)})
+        assert(@dbf.records.any? {|record| record[name].is_a?(String) && record[name].size > 1})
       end
     end
   

data/test/dbase_iii_read_test.rb

@@ -10,6 +10,7 @@ class DBaseIIIReadTest < Test::Unit::TestCase
     @controls = {
       :version => "03",
       :has_memo_file => false,
+      :memo_file_format => nil,
       :field_count => 31,
       :record_count => 14,
       :record_length => 590,

data/test/foxpro_read_test.rb

@@ -10,6 +10,7 @@ class FoxproReadTest < Test::Unit::TestCase
     @controls = {
       :version => "f5",
       :has_memo_file => true,
+      :memo_file_format => :fpt,
       :field_count => 59,
       :record_count => 975,
       :record_length => 969,

data/test/performance.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.dirname(__FILE__) + "/../lib/")
+require 'dbf'
+require 'profiler'
+
+dbf = DBF::Reader.new(File.join(File.dirname(__FILE__),'databases', 'foxpro.dbf'))
+
+Profiler__::start_profile
+
+dbf.records
+
+Profiler__::stop_profile
+Profiler__::print_profile($stdout)

metadata

@@ -3,9 +3,9 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: dbf
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
-date: 2006-08-12 00:00:00 -07:00
-summary: A library for reading DBase (or XBase, Clipper, Foxpro, etc) database files
+  version: 0.4.0
+date: 2006-10-07 00:00:00 -07:00
+summary: A library for reading dBase (or xBase, Clipper, Foxpro, etc) database files
 require_paths: 
 - lib
 email: keithm@infused.org
@@ -15,7 +15,7 @@ description:
 autorequire: 
 default_executable: 
 bindir: bin
-has_rdoc: false
+has_rdoc: true
 required_ruby_version: !ruby/object:Gem::Version::Requirement 
   requirements: 
   - - ">"
@@ -36,6 +36,7 @@ files:
 - test/databases
 - test/dbase_iii_read_test.rb
 - test/foxpro_read_test.rb
+- test/performance.rb
 - test/databases/dbase_iii.dbf
 - test/databases/foxpro.dbf
 - test/databases/foxpro.fpt