dbf 1.5.2 → 1.5.3

data/Gemfile.lock

@@ -3,92 +3,28 @@ PATH
   specs:
     dbf (1.5.2)
       activesupport (~> 3.0.0)
-      fastercsv (= 1.5.3)
-      i18n (~> 0.4.2)
+      fastercsv (= 1.5.4)
+      i18n (~> 0.5.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    Saikuro (1.1.0)
-    activesupport (3.0.3)
-    arrayfields (4.7.4)
-    chronic (0.2.3)
-      hoe (>= 1.2.1)
-    churn (0.0.12)
-      chronic (~> 0.2.3)
-      hirb
-      json_pure
-      main
-      ruby_parser (~> 2.0.4)
-      sexp_processor (~> 3.0.3)
-    colored (1.2)
+    activesupport (3.0.5)
     diff-lcs (1.1.2)
-    fastercsv (1.5.3)
-    fattr (2.2.0)
-    flay (1.4.1)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    flog (2.5.0)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    hirb (0.3.5)
-    hoe (2.7.0)
-      rake (>= 0.8.7)
-      rubyforge (>= 2.0.4)
-    i18n (0.4.2)
-    json_pure (1.4.6)
-    main (4.3.0)
-      arrayfields (>= 4.7.4)
-      fattr (>= 2.1.0)
-    metric_fu (2.0.1)
-      Saikuro (>= 1.1.0)
-      activesupport (>= 2.0.0)
-      chronic (~> 0.2.3)
-      churn (>= 0.0.7)
-      flay (>= 1.2.1)
-      flog (>= 2.2.0)
-      rails_best_practices (>= 0.3.16)
-      rcov (>= 0.8.3.3)
-      reek (>= 1.2.6)
-      roodi (>= 2.1.0)
-    progressbar (0.9.0)
-    rails_best_practices (0.5.0)
-      activesupport
-      colored (~> 1.2)
-      progressbar (~> 0.9.0)
-      ruby_parser (~> 2.0.4)
-    rake (0.8.7)
-    rcov (0.9.9)
-    reek (1.2.8)
-      ruby2ruby (~> 1.2)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    roodi (2.1.0)
-      ruby_parser
-    rspec (2.1.0)
-      rspec-core (~> 2.1.0)
-      rspec-expectations (~> 2.1.0)
-      rspec-mocks (~> 2.1.0)
-    rspec-core (2.1.0)
-    rspec-expectations (2.1.0)
+    fastercsv (1.5.4)
+    i18n (0.5.0)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.1)
+    rspec-expectations (2.5.0)
       diff-lcs (~> 1.1.2)
-    rspec-mocks (2.1.0)
-    ruby2ruby (1.2.5)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    ruby_parser (2.0.5)
-      sexp_processor (~> 3.0)
-    rubyforge (2.0.4)
-      json_pure (>= 1.1.7)
-    sexp_processor (3.0.5)
+    rspec-mocks (2.5.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (~> 3.0.0)
   dbf!
-  fastercsv (= 1.5.3)
-  i18n (~> 0.4.2)
-  metric_fu (= 2.0.1)
-  rspec (= 2.1.0)
+  rspec (= 2.5.0)

data/MIT-LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2006-2010 Keith Morrison <keithm@infused.org>
+Copyright (c) 2006-2011 Keith Morrison <keithm@infused.org>
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -115,7 +115,7 @@ for a full list of supported column types.
 
 ## License
 
-Copyright (c) 2006-2010 Keith Morrison <keithm@infused.org>
+Copyright (c) 2006-2011 Keith Morrison <keithm@infused.org>
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

data/Rakefile

@@ -24,8 +24,8 @@ task :console do
   sh "irb -rubygems -I lib -r dbf.rb"
 end
 
-require 'metric_fu'
-MetricFu::Configuration.run do |config|
-  config.rcov[:test_files] = ['spec/**/*_spec.rb']  
-  config.rcov[:rcov_opts] << "-Ispec"
-end
+# require 'metric_fu'
+# MetricFu::Configuration.run do |config|
+#   config.rcov[:test_files] = ['spec/**/*_spec.rb']  
+#   config.rcov[:rcov_opts] << "-Ispec"
+# end

data/lib/dbf.rb

@@ -5,7 +5,6 @@ require 'active_support/core_ext/date/conversions'
 require 'active_support/core_ext/time/conversions'
 require 'active_support/core_ext/date_time/conversions'
 require 'active_support/core_ext/string/conversions'
-require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/string/inflections'
 require 'active_support/core_ext/enumerable'
 

data/lib/dbf/column.rb

@@ -1,91 +1,87 @@
 module DBF
   class ColumnLengthError < StandardError; end
   class ColumnNameError < StandardError; end
-  
+
   # DBF::Column stores all the information about a column including its name,
   # type, length and number of decimal places (if any)
   class Column
     attr_reader :name, :type, :length, :decimal
-    
+
     # Initialize a new DBF::Column
     #
     # @param [String] name
     # @param [String] type
     # @param [Fixnum] length
     # @param [Fixnum] decimal
-    def initialize(name, type, length, decimal)
-      @name, @type, @length, @decimal = clean(name), type, length, decimal
-      
+    def initialize(name, type, length, decimal, encoding=nil)
+      @name, @type, @length, @decimal, @encoding = clean(name), type, length, decimal, encoding
+
       raise ColumnLengthError, "field length must be greater than 0" unless length > 0
       raise ColumnNameError, "column name cannot be empty" if @name.length == 0
     end
-    
+
     # Cast value to native type
     #
     # @param [String] value
-    # @return [Fixnum, Float, Date, DateTime, Boolean, String] 
+    # @return [Fixnum, Float, Date, DateTime, Boolean, String]
     def type_cast(value)
       case type
         when 'N' then unpack_number(value)
         when 'I' then unpack_unsigned_long(value)
-        when 'F' then unpack_float(value)
+        when 'F' then value.to_f
         when 'D' then decode_date(value)
         when 'T' then decode_datetime(value)
         when 'L' then boolean(value)
-        else          value.to_s.strip
+        else          encode_string(value.to_s).strip
       end
     end
-    
+
     def memo?
       @memo ||= type == 'M'
     end
-    
+
     # Schema definition
     #
     # @return [String]
     def schema_definition
-      "\"#{name.underscore}\", #{schema_data_type}\n"
+      "\"#{underscored_name}\", #{schema_data_type}\n"
     end
-    
+
     def underscored_name
       @underscored_name ||= name.underscore
     end
-    
+
     private
-    
+
     def decode_date(value) #nodoc
       value.gsub!(' ', '0')
       value.blank? ? nil : value.to_date
     rescue
       nil
     end
-    
+
     def decode_datetime(value) #nodoc
       days, milliseconds = value.unpack('l2')
       seconds = milliseconds / 1000
       DateTime.jd(days, seconds/3600, seconds/60 % 60, seconds % 60) rescue nil
     end
-    
+
     def unpack_number(value) #nodoc
-      decimal.zero? ? unpack_integer(value) : value.to_f
+      decimal.zero? ? value.to_i : value.to_f
     end
-    
-    def unpack_float(value) #nodoc
-      value.to_f
-    end
-    
-    def unpack_integer(value) #nodoc
-      value.to_i
-    end
-    
+
     def unpack_unsigned_long(value) #nodoc
       value.unpack('V')[0]
     end
-    
+
     def boolean(value) #nodoc
       value.strip =~ /^(y|t)$/i ? true : false
     end
-    
+
+    def encode_string(value)
+      @encoding ? value.force_encoding(@encoding).encode(Encoding.default_external) : value
+    end
+
     def schema_data_type #nodoc
       case type
       when "N", "F"
@@ -104,13 +100,13 @@ module DBF
         ":string, :limit => #{length}"
       end
     end
-    
+
     def clean(s) #nodoc
       first_null = s.index("\x00")
       s = s[0, first_null] if first_null
       s.gsub(/[^\x20-\x7E]/, "")
     end
-    
+
   end
-  
+
 end

data/lib/dbf/encodings.yml

@@ -0,0 +1,61 @@
+# inspired by http://trac.osgeo.org/gdal/ticket/2864
+
+"01": cp437        # U.S. MS–DOS
+"02": cp850        # International MS–DOS
+"03": cp1252       # Windows ANSI
+"08": cp865        # Danish OEM
+"09": cp437        # Dutch OEM
+"0a": cp850        # Dutch OEM*
+"0b": cp437        # Finnish OEM
+"0d": cp437        # French OEM
+"0e": cp850        # French OEM*
+"0f": cp437        # German OEM
+"10": cp850        # German OEM*
+"11": cp437        # Italian OEM
+"12": cp850        # Italian OEM*
+"13": cp932        # Japanese Shift-JIS
+"14": cp850        # Spanish OEM*
+"15": cp437        # Swedish OEM
+"16": cp850        # Swedish OEM*
+"17": cp865        # Norwegian OEM
+"18": cp437        # Spanish OEM
+"19": cp437        # English OEM (Britain)
+"1a": cp850        # English OEM (Britain)*
+"1b": cp437        # English OEM (U.S.)
+"1c": cp863        # French OEM (Canada)
+"1d": cp850        # French OEM*
+"1f": cp852        # Czech OEM
+"22": cp852        # Hungarian OEM
+"23": cp852        # Polish OEM
+"24": cp860        # Portuguese OEM
+"25": cp850        # Portuguese OEM*
+"26": cp866        # Russian OEM
+"37": cp850        # English OEM (U.S.)*
+"40": cp852        # Romanian OEM
+"4d": cp936        # Chinese GBK (PRC)
+"4e": cp949        # Korean (ANSI/OEM)
+"4f": cp950        # Chinese Big5 (Taiwan)
+"50": cp874        # Thai (ANSI/OEM)
+"57": cp1252       # ANSI
+"58": cp1252       # Western European ANSI
+"59": cp1252       # Spanish ANSI
+"64": cp852        # Eastern European MS–DOS
+"65": cp866        # Russian MS–DOS
+"66": cp865        # Nordic MS–DOS
+"67": cp861        # Icelandic MS–DOS
+"6a": cp737        # Greek MS–DOS (437G)
+"6b": cp857        # Turkish MS–DOS
+"6c": cp863        # French–Canadian MS–DOS
+"78": cp950        # Taiwan Big 5
+"79": cp949        # Hangul (Wansung)
+"7a": cp936        # PRC GBK
+"7b": cp932        # Japanese Shift-JIS
+"7c": cp874        # Thai Windows/MS–DOS
+"86": cp737        # Greek OEM
+"87": cp852        # Slovenian OEM
+"88": cp857        # Turkish OEM
+"c8": cp1250       # Eastern European Windows
+"c9": cp1251       # Russian Windows
+"ca": cp1254       # Turkish Windows
+"cb": cp1253       # Greek Windows
+"cc": cp1257       # Baltic Windows

data/lib/dbf/memo.rb

@@ -47,10 +47,12 @@ module DBF
         build_dbt_83_memo(start_block)
       when "8b" # dbase iv
         build_dbt_8b_memo(start_block)
+      else
+        nil
       end
     end
     
-    def build_dbt_83_memo(start_block)
+    def build_dbt_83_memo(start_block) #nodoc
       @data.seek offset(start_block)
       memo_string = ""
       begin
@@ -60,7 +62,7 @@ module DBF
       memo_string
     end
     
-    def build_dbt_8b_memo(start_block)
+    def build_dbt_8b_memo(start_block) #nodoc
       @data.seek offset(start_block)
       @data.read(@data.read(BLOCK_HEADER_SIZE).unpack("x4L").first)
     end

data/lib/dbf/table.rb

@@ -1,14 +1,14 @@
 module DBF
 
-  # DBF::Table is the primary interface to a single DBF file and provides 
+  # DBF::Table is the primary interface to a single DBF file and provides
   # methods for enumerating and searching the records.
-  
+
   # TODO set record_length to length of actual used column lengths
   class Table
     include Enumerable
-    
+
     DBF_HEADER_SIZE = 32
-    
+
     VERSION_DESCRIPTIONS = {
       "02" => "FoxBase",
       "03" => "dBase III without memo file",
@@ -23,10 +23,11 @@ module DBF
       "f5" => "FoxPro with memo file",
       "fb" => "FoxPro without memo file"
     }
-    
-    attr_reader :version                # Internal dBase version number
-    attr_reader :record_count           # Total number of records
-    
+
+    attr_reader   :version              # Internal dBase version number
+    attr_reader   :record_count         # Total number of records
+    attr_accessor :encoding             # Source encoding (for ex. :cp1251)
+
     # Opens a DBF::Table
     # Example:
     #   table = DBF::Table.new 'data.dbf'
@@ -37,13 +38,13 @@ module DBF
       get_header_info
       @memo = open_memo(path)
     end
-    
+
     # Closes the table and memo file
     def close
       @memo && @memo.close
       @data.close
     end
-    
+
     # Calls block once for each record in the table. The record may be nil
     # if the record has been marked as deleted.
     #
@@ -51,7 +52,7 @@ module DBF
     def each
       @record_count.times {|i| yield record(i)}
     end
-    
+
     # Retrieve a record by index number.
     # The record will be nil if it has been deleted, but not yet pruned from
     # the database.
@@ -62,18 +63,18 @@ module DBF
       seek_to_record(index)
       current_record
     end
-    
+
     alias_method :row, :record
-    
+
     # Human readable version description
     #
     # @return [String]
     def version_description
       VERSION_DESCRIPTIONS[version]
     end
-    
+
     # Generate an ActiveRecord::Schema
-    # 
+    #
     # xBase data types are converted to generic types as follows:
     # - Number columns with no decimals are converted to :integer
     # - Number columns with decimals are converted to :float
@@ -99,40 +100,39 @@ module DBF
       columns.each do |column|
         s << "    t.column #{column.schema_definition}"
       end
-      s << "  end\nend"        
+      s << "  end\nend"
       s
     end
-    
+
     # Dumps all records to a CSV file.  If no filename is given then CSV is
     # output to STDOUT.
     #
     # @param [optional String] path Defaults to basename of dbf file
     def to_csv(path = nil)
-      path = default_csv_path unless path
-      csv_class.open(path, 'w', :force_quotes => true) do |csv|
+      csv_class.open(path || default_csv_path, 'w', :force_quotes => true) do |csv|
         csv << columns.map {|c| c.name}
         each {|record| csv << record.to_a}
       end
     end
-    
+
     # Find records using a simple ActiveRecord-like syntax.
     #
     # Examples:
     #   table = DBF::Table.new 'mydata.dbf'
-    #   
+    #
     #   # Find record number 5
     #   table.find(5)
     #
     #   # Find all records for Keith Morrison
     #   table.find :all, :first_name => "Keith", :last_name => "Morrison"
-    # 
+    #
     #   # Find first record
     #   table.find :first, :first_name => "Keith"
     #
     # The <b>command</b> may be a record index, :all, or :first.
     # <b>options</b> is optional and, if specified, should be a hash where the keys correspond
     # to column names in the database.  The values will be matched exactly with the value
-    # in the database.  If you specify more than one key, all values must match in order 
+    # in the database.  If you specify more than one key, all values must match in order
     # for the record to be returned.  The equivalent SQL would be "WHERE key1 = 'value1'
     # AND key2 = 'value2'".
     #
@@ -151,23 +151,24 @@ module DBF
         find_first(options)
       end
     end
-    
+
     # Retrieves column information from the database
     def columns
-      return @columns if @columns
-      
-      @data.seek(DBF_HEADER_SIZE)
-      @columns = []
-      column_count = (@header_length - DBF_HEADER_SIZE + 1) / DBF_HEADER_SIZE
-      column_count.times do
-        name, type, length, decimal = @data.read(32).unpack('a10 x a x4 C2')
-        @columns << Column.new(name.strip, type, length, decimal) if length > 0
+      @columns ||= begin
+        column_count = (@header_length - DBF_HEADER_SIZE + 1) / DBF_HEADER_SIZE
+
+        @data.seek(DBF_HEADER_SIZE)
+        columns = []
+        column_count.times do
+          name, type, length, decimal = @data.read(32).unpack('a10 x a x4 C2')
+          columns << Column.new(name.strip, type, length, decimal, @encoding) if length > 0
+        end
+        columns
       end
-      @columns
     end
-    
+
     private
-    
+
     def open_memo(path) #nodoc
       %w(fpt FPT dbt DBT).each do |extname|
         filename = path.sub(/#{File.extname(path)[1..-1]}$/, extname)
@@ -177,7 +178,7 @@ module DBF
       end
       nil
     end
-    
+
     def find_all(options) #nodoc
       map do |record|
         if record.try(:match?, options)
@@ -186,43 +187,45 @@ module DBF
         end
       end.compact
     end
-    
+
     def find_first(options) #nodoc
-      each do |record|
-        return record if record.try(:match?, options)
-      end
-      nil
+      detect {|record| record.try(:match?, options)}
     end
-    
+
     def deleted_record? #nodoc
       @data.read(1).unpack('a') == ['*']
     end
-    
-    def current_record
+
+    def current_record #nodoc
       deleted_record? ? nil : DBF::Record.new(@data.read(@record_length), columns, version, @memo)
     end
-    
+
     def get_header_info #nodoc
       @data.rewind
-      @version, @record_count, @header_length, @record_length = @data.read(DBF_HEADER_SIZE).unpack('H2 x3 V v2')
+      @version, @record_count, @header_length, @record_length, encoding_key =
+        @data.read(DBF_HEADER_SIZE).unpack("H2 x3 V v2 x17H2")
+      @encoding = self.class.encodings[encoding_key] if "".respond_to? :encoding
     end
-    
+
     def seek(offset) #nodoc
       @data.seek @header_length + offset
     end
-  
+
     def seek_to_record(index) #nodoc
       seek index * @record_length
     end
-    
+
     def csv_class #nodoc
       CSV.const_defined?(:Reader) ? FCSV : CSV
     end
-    
+
     def default_csv_path #nodoc
       File.basename(@data.path, '.dbf') + '.csv'
     end
-    
+
+    def self.encodings
+      @encodings ||= YAML.load_file(File.expand_path("../encodings.yml", __FILE__))
+    end
   end
-  
-end
+
+end

data/lib/dbf/version.rb

@@ -1,3 +1,3 @@
 module DBF
-  VERSION = '1.5.2'
+  VERSION = '1.5.3'
 end

data/spec/dbf/record_spec.rb

@@ -1,29 +1,29 @@
 require "spec_helper"
 
 describe DBF::Record do
-  
+
   describe '#to_a' do
     it 'should return an ordered array of attribute values' do
       table = DBF::Table.new "#{DB_PATH}/dbase_8b.dbf"
-      
+
       record = table.record(0)
       record.to_a.should == ["One", 1.0, Date.new(1970, 1, 1), true, 1.23456789012346, "First memo\r\n\037 \037 \037 \037 "]
-      
+
       record = table.record(9)
       record.to_a.should == ["Ten records stored in this database", 10.0, nil, false, 0.1, nil]
     end
   end
-  
+
   describe '#==' do
     before do
       table = DBF::Table.new "#{DB_PATH}/dbase_8b.dbf"
       @record = table.record(9)
     end
-    
+
     it 'should be false if other does not have attributes' do
       (@record == mock('other')).should be_false
     end
-    
+
     it 'should be true if other attributes match' do
       attributes = {:x => 1, :y => 2}
       @record.stub!(:attributes).and_return(attributes)
@@ -31,10 +31,10 @@ describe DBF::Record do
       (@record == other).should be_true
     end
   end
-  
+
   describe 'column accessors' do
     let(:table) { DBF::Table.new "#{DB_PATH}/dbase_8b.dbf"}
-    
+
     it 'should define accessor methods for each column' do
       record = table.find(0)
       record.should respond_to(:character)
@@ -42,4 +42,15 @@ describe DBF::Record do
     end
   end
 
+  describe 'column data for table' do
+    let(:table) { DBF::Table.new "#{DB_PATH}/cp1251.dbf"}
+
+    let(:record) { table.find(0) }
+    it 'should automatically encodes to default system encoding' do
+      if "".respond_to? :encoding
+        record.name.encoding.should == Encoding.default_external
+        record.name.encode("UTF-8").unpack("H4").should == ["d0b0"] # russian a
+      end
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dbf
 version: !ruby/object:Gem::Version 
-  hash: 7
-  prerelease: false
+  hash: 5
+  prerelease: 
   segments: 
   - 1
   - 5
-  - 2
-  version: 1.5.2
+  - 3
+  version: 1.5.3
 platform: ruby
 authors: 
 - Keith Morrison
@@ -15,8 +15,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-22 00:00:00 -08:00
-default_executable: dbf
+date: 2011-04-06 00:00:00 -07:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
@@ -45,9 +45,9 @@ dependencies:
         hash: 11
         segments: 
         - 0
-        - 4
-        - 2
-        version: 0.4.2
+        - 5
+        - 0
+        version: 0.5.0
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -58,12 +58,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 11
         segments: 
         - 1
         - 5
-        - 3
-        version: 1.5.3
+        - 4
+        version: 1.5.4
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -74,30 +74,14 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 11
+        hash: 27
         segments: 
         - 2
-        - 1
+        - 5
         - 0
-        version: 2.1.0
+        version: 2.5.0
   type: :development
   version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: metric_fu
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        hash: 13
-        segments: 
-        - 2
-        - 0
-        - 1
-        version: 2.0.1
-  type: :development
-  version_requirements: *id005
 description: A small fast library for reading dBase, xBase, Clipper and FoxPro database files.
 email: keithm@infused.org
 executables: 
@@ -107,6 +91,7 @@ extensions: []
 extra_rdoc_files: 
 - README.md
 - CHANGELOG.md
+- MIT-LICENSE
 files: 
 - CHANGELOG.md
 - Gemfile
@@ -118,6 +103,7 @@ files:
 - docs/supported_types.markdown
 - lib/dbf/attributes.rb
 - lib/dbf/column.rb
+- lib/dbf/encodings.yml
 - lib/dbf/memo.rb
 - lib/dbf/record.rb
 - lib/dbf/table.rb
@@ -127,6 +113,7 @@ files:
 - spec/dbf/file_formats_spec.rb
 - spec/dbf/record_spec.rb
 - spec/dbf/table_spec.rb
+- spec/fixtures/cp1251.dbf
 - spec/fixtures/dbase_03.dbf
 - spec/fixtures/dbase_30.dbf
 - spec/fixtures/dbase_30.fpt
@@ -171,7 +158,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Read xBase files