dullard 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76b6de08a6a72d691a3f3deb06f8a82018c90049
-  data.tar.gz: d56a5c10a24db975ce01c0e3e352739e128508f1
+  metadata.gz: 9d02ed091d258d5d690ba9e3ba350c392488664e
+  data.tar.gz: 9f61befa4e3003f7e469a26ff7c4412ffa4d6e55
 SHA512:
-  metadata.gz: 943f8b15089aab39fb627fc84a45523e093b60e6980666cb139be7102d99bb2adf3f71a2ed32d5d693784d4612199390de7a62e7dd06de4d80495a7de832ccbc
-  data.tar.gz: 3402538efd1a687823a4acc8e4a9fe811d381de1e20d40d354746cacb2e2ff557b6eb6259c5446f05b89375c61cb81efe0516fdc7cf26ec22800ed92325c45eb
+  metadata.gz: 87d7d94d530d337ba92623146e5d0a115e44b04d15b1ee91fcf4c49bc76e90ace74bbf29f6e707ad3853d3e314e000cb4a3ec96a9b256576755adaeacce8022c
+  data.tar.gz: 2d10be78032321162cc1e2416771946c02cecea129f2959f12ec9a3bc49e2e9d68563ac18b9896159fef1a097d1d90fa5bf6ac7698985a2ec1fc3afcf8248297

data/Rakefile

@@ -1,2 +1,7 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/dullard.gemspec

@@ -6,6 +6,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["ted@shlashdot.org"]
   gem.summary       = %q{A fast XLSX parser using Nokogiri}
   gem.homepage      = "http://github.com/thirtyseven/dullard"
+  gem.license       = "MIT"
 
   gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.files         = `git ls-files`.split("\n")
@@ -14,7 +15,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Dullard::VERSION
 
-  gem.add_development_dependency "rspec", "~> 2.6"
-  gem.add_dependency "nokogiri", "~> 1.5"
-  gem.add_dependency "rubyzip", "~> 0.9.6"
+  gem.add_development_dependency "rspec", "~> 2.14"
+  gem.add_development_dependency "rake", "~> 10.1"
+  gem.add_dependency "nokogiri", "~> 1.6"
+  gem.add_dependency "rubyzip", "~> 1.1"
 end

data/lib/dullard/reader.rb

@@ -1,12 +1,86 @@
-require 'zip/zipfilesystem'
+require 'zip/filesystem'
 require 'nokogiri'
 
 module Dullard; end
 
 class Dullard::Workbook
-  def initialize(file)
+  # Code borrowed from Roo (https://github.com/hmcgowan/roo/blob/master/lib/roo/excelx.rb)
+  # Some additional formats added by Paul Hendryx (phendryx@gmail.com) that are common in LibreOffice.
+  FORMATS = {
+    'general' => :float,
+    '0' => :float,
+    '0.00' => :float,
+    '#,##0' => :float,
+    '#,##0.00' => :float,
+    '0%' => :percentage,
+    '0.00%' => :percentage,
+    '0.00E+00' => :float,
+    '# ?/?' => :float, #??? TODO:
+    '# ??/??' => :float, #??? TODO:
+    'mm-dd-yy' => :date,
+    'd-mmm-yy' => :date,
+    'd-mmm' => :date,
+    'mmm-yy' => :date,
+    'h:mm am/pm' => :date,
+    'h:mm:ss am/pm' => :date,
+    'h:mm' => :time,
+    'h:mm:ss' => :time,
+    'm/d/yy h:mm' => :date,
+    '#,##0 ;(#,##0)' => :float,
+    '#,##0 ;[red](#,##0)' => :float,
+    '#,##0.00;(#,##0.00)' => :float,
+    '#,##0.00;[red](#,##0.00)' => :float,
+    'mm:ss' => :time,
+    '[h]:mm:ss' => :time,
+    'mmss.0' => :time,
+    '##0.0e+0' => :float,
+    '@' => :float,
+    #-- zusaetzliche Formate, die nicht standardmaessig definiert sind:
+    "yyyy\\-mm\\-dd" => :date,
+    'dd/mm/yy' => :date,
+    'hh:mm:ss' => :time,
+    "dd/mm/yy\\ hh:mm" => :datetime,
+    'm/d/yy' => :date,
+    'mm/dd/yy' => :date,
+    'mm/dd/yyyy' => :date,
+  }
+
+  STANDARD_FORMATS = { 
+    0 => 'General',
+    1 => '0',
+    2 => '0.00',
+    3 => '#,##0',
+    4 => '#,##0.00',
+    9 => '0%',
+    10 => '0.00%',
+    11 => '0.00E+00',
+    12 => '# ?/?',
+    13 => '# ??/??',
+    14 => 'mm-dd-yy',
+    15 => 'd-mmm-yy',
+    16 => 'd-mmm',
+    17 => 'mmm-yy',
+    18 => 'h:mm AM/PM',
+    19 => 'h:mm:ss AM/PM',
+    20 => 'h:mm',
+    21 => 'h:mm:ss',
+    22 => 'm/d/yy h:mm',
+    37 => '#,##0 ;(#,##0)',
+    38 => '#,##0 ;[Red](#,##0)',
+    39 => '#,##0.00;(#,##0.00)',
+    40 => '#,##0.00;[Red](#,##0.00)',
+    45 => 'mm:ss',
+    46 => '[h]:mm:ss',
+    47 => 'mmss.0',
+    48 => '##0.0E+0',
+    49 => '@',
+  }
+
+  def initialize(file, user_defined_formats = {})
     @file = file
-    @zipfs = Zip::ZipFile.open(@file)
+    @zipfs = Zip::File.open(@file)
+    @user_defined_formats = user_defined_formats
+    read_styles
   end
 
   def sheets
@@ -33,6 +107,51 @@ class Dullard::Workbook
     @string_table
   end
 
+  def read_styles
+    doc = Nokogiri::XML(@zipfs.file.open("xl/styles.xml"))
+    
+    @num_formats = {}
+    @cell_xfs = []
+    
+    doc.css('/styleSheet/numFmts/numFmt').each do |numFmt|
+      numFmtId = numFmt.attributes['numFmtId'].value.to_i
+      formatCode = numFmt.attributes['formatCode'].value
+      @num_formats[numFmtId] = formatCode
+    end
+
+    doc.css('/styleSheet/cellXfs/xf').each do |xf|
+      numFmtId = xf.attributes['numFmtId'].value.to_i
+      @cell_xfs << numFmtId
+    end
+  end
+
+  
+  # Code borrowed from Roo (https://github.com/hmcgowan/roo/blob/master/lib/roo/excelx.rb)
+  # convert internal excelx attribute to a format
+  def attribute2format(s)
+    id = @cell_xfs[s.to_i].to_i
+    result = @num_formats[id]
+
+    if result == nil
+      if STANDARD_FORMATS.has_key? id
+        result = STANDARD_FORMATS[id]
+      end
+    end
+
+    result.downcase
+  end
+
+  # Code borrowed from Roo (https://github.com/hmcgowan/roo/blob/master/lib/roo/excelx.rb)
+  def format2type(format)
+    if FORMATS.has_key? format
+      FORMATS[format]
+    elsif @user_defined_formats.has_key? format
+      @user_defined_formats[format]
+    else
+      :float
+    end
+  end
+
   def zipfs
     @zipfs
   end
@@ -57,12 +176,13 @@ class Dullard::Sheet
   end
 
   def rows
-    Enumerator.new(rows_size) do |y|
+    Enumerator.new(row_count) do |y|
       next unless @file
       @file.rewind
       shared = false
       row = nil
       column = nil
+      cell_type = nil
       Nokogiri::XML::Reader(@file).each do |node|
         case node.node_type
         when Nokogiri::XML::Reader::TYPE_ELEMENT
@@ -72,6 +192,11 @@ class Dullard::Sheet
             column = 0
             next
           when "c"
+            if node.attributes['t'] != 's' && node.attributes['t'] != 'b'
+              cell_format_index = node.attributes['s'].to_i
+              cell_type = @workbook.format2type(@workbook.attribute2format(cell_format_index))
+            end
+
             rcolumn = node.attributes["r"]
             if rcolumn
               rcolumn.delete!("0-9")
@@ -91,7 +216,21 @@ class Dullard::Sheet
           end
         end
         value = node.value
+
         if value
+          case cell_type
+            when :datetime
+            when :time
+            when :date
+              value = (DateTime.new(1899,12,30) + value.to_f)
+            when :percentage # ? TODO
+            when :float
+              value = value.to_f
+            else
+              # leave as string
+          end
+          cell_type = nil
+
           row << (shared ? string_lookup(value.to_i) : value)
         end
       end
@@ -113,14 +252,9 @@ class Dullard::Sheet
     end
   end
 
-  private
-  def path
-    "xl/worksheets/sheet#{@index}.xml"
-  end
-
-  def rows_size
-    if defined? @rows_size
-      @rows_size
+  def row_count
+    if defined? @row_count
+      @row_count
     elsif @file
       @file.rewind
       Nokogiri::XML::Reader(@file).each do |node|
@@ -128,13 +262,19 @@ class Dullard::Sheet
           case node.name
           when "dimension"
             if ref = node.attributes["ref"]
-              break @rows_size = ref.scan(/\d+$/).first.to_i
+              break @row_count = ref.scan(/\d+$/).first.to_i
             end
           when "sheetData"
-            break @rows_size = nil
+            break @row_count = nil
           end
         end
       end
     end
   end
+
+  private
+  def path
+    "xl/worksheets/sheet#{@index}.xml"
+  end
+
 end

data/lib/dullard/version.rb

@@ -1,3 +1,3 @@
 module Dullard
-  VERSION = "0.0.5"
+  VERSION = "0.1.0"
 end

data/spec/dullard_spec.rb

@@ -0,0 +1,73 @@
+require 'dullard'
+
+describe "dullard," do
+  before(:each) do
+    @file = File.open(File.expand_path("../test.xlsx", __FILE__))
+  end
+
+  describe "when it has no user defined formats," do
+    before(:each) do
+      @xlsx = Dullard::Workbook.new @file
+    end
+
+    it "can open a file" do
+      @xlsx.should_not be_nil
+    end
+
+    it "can find sheets" do
+      @xlsx.sheets.count.should == 1
+    end
+
+    it "reads the right number of columns, even with blanks" do
+      rows = @xlsx.sheets[0].rows
+      rows.next.count.should == 300
+      rows.next.count.should == 9
+      rows.next.count.should == 1
+    end
+
+    it "reads the right number of rows" do
+      @xlsx.sheets[0].row_count.should == 117
+    end
+
+    it "reads the right number of rows from the metadata when present" do
+      @xlsx.sheets[0].rows.size.should == 117
+    end
+
+    it "reads date/time properly" do
+      count = 0
+      @xlsx.sheets[0].rows.each do |row|
+        count += 1
+
+        if count == 116
+          row[0].strftime("%Y-%m-%d %H:%M:%S").should == "2012-10-18 00:00:00"
+          row[1].strftime("%Y-%m-%d %H:%M:%S").should == "2012-10-18 00:17:58"
+          row[2].strftime("%Y-%m-%d %H:%M:%S").should == "2012-07-01 21:18:48"
+          row[3].strftime("%Y-%m-%d %H:%M:%S").should == "2012-07-01 21:18:52"
+        end
+      end
+      count.should == 117
+    end
+  end
+
+  describe "when it has user defined formats," do
+    before(:each) do
+      @xlsx = Dullard::Workbook.new @file, {'GENERAL' => :string, 'm/d/yyyy' => :date, 'M/D/YYYY' => :date,}
+    end
+
+    it "converts the user defined formatted cells properly" do
+      count = 0
+      @xlsx.sheets[0].rows.each do |row|
+        count += 1
+
+        if count == 117
+          row[0].should == 'teststring'
+          row[1].strftime("%Y-%m-%d %H:%M:%S").should == "2012-10-18 00:00:00"
+          row[2].strftime("%Y-%m-%d %H:%M:%S").should == "2012-10-18 00:17:58"
+          row[3].strftime("%Y-%m-%d %H:%M:%S").should == "2012-07-01 21:18:48"
+          row[4].strftime("%Y-%m-%d %H:%M:%S").should == "2012-07-01 21:18:52"
+        end
+      end
+      count.should == 117
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dullard
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Ted Kaplan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-21 00:00:00.000000000 Z
+date: 2014-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -16,42 +16,56 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.1'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.6
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.6
+        version: '1.1'
 description: 
 email:
 - ted@shlashdot.org
@@ -68,10 +82,11 @@ files:
 - lib/dullard.rb
 - lib/dullard/reader.rb
 - lib/dullard/version.rb
-- specs/dullard_spec.rb
-- specs/test.xlsx
+- spec/dullard_spec.rb
+- spec/test.xlsx
 homepage: http://github.com/thirtyseven/dullard
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -93,4 +108,6 @@ rubygems_version: 2.0.5
 signing_key: 
 specification_version: 4
 summary: A fast XLSX parser using Nokogiri
-test_files: []
+test_files:
+- spec/dullard_spec.rb
+- spec/test.xlsx

data/specs/dullard_spec.rb

@@ -1,34 +0,0 @@
-require 'dullard'
-
-describe "dullard" do
-  before(:each) do
-    @file = File.open(File.expand_path("../test.xlsx", __FILE__))
-    @xlsx = Dullard::Workbook.new @file
-  end
-  it "can open a file" do
-    @xlsx.should_not be_nil
-  end
-
-  it "can find sheets" do
-    @xlsx.sheets.count.should == 1
-  end
-
-  it "reads the right number of columns, even with blanks" do
-    rows = @xlsx.sheets[0].rows
-    rows.next.count.should == 300
-    rows.next.count.should == 9
-    rows.next.count.should == 1
-  end
-
-  it "reads the right number of rows" do
-    count = 0
-    @xlsx.sheets[0].rows.each do |row|
-      count += 1
-    end
-    count.should == 115
-  end
-
-  it "reads the right number of rows from the metadata when present" do
-    @xlsx.sheets[0].rows.size.should == 115
-  end
-end