roo 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d2a5a56902bf60eef9028eddb5617ae403b866f
-  data.tar.gz: 3fad0f4206fd3058b3a420b63da7393b530f5f59
+  metadata.gz: 975d999c227f6bc66a6819e0e69c3ebceb3c76c9
+  data.tar.gz: 9b790645ff9ad0e2a8e1a84c9deb465bf5e5154b
 SHA512:
-  metadata.gz: 91ca5af7282631e668e104e407fb6affd40bc56792809b7129e46de4dd9e4af8133aa6c1eb18f2a9ac8fc7d9470ae51bfb304bff9d920893f1b391b40f6dd162
-  data.tar.gz: 9efee7d9ae6fec31bd965a7ce49fcd57452b0b61bb8ad0eb89a21640e4d3f0bbd5cfbb018f6f567b8475050550eb684dd6800aa80716876e2c0b747ae8adecd2
+  metadata.gz: 7b197a0ae99ceeb81ff7a8bf12099944f0b2667dbc25ee85380d64bd1117309cc50e98eece758824ee9d5c0b022b354b3c64a10357035281ee7191f79cb479d1
+  data.tar.gz: a73ae23bc6419fcacad79d85496e6349ef0b45956382cc1e93af0f4be57614248a1e1db33dcc87774a2b8cbbb58098d8526ecc4267c79c86f242acec161815dc

data/.travis.yml

@@ -1,12 +1,16 @@
 language: ruby
 rvm:
-  - 2.0.0
-  - 2.1.8
   - 2.2.4
+  - 2.3.1
   - ruby-head
   - jruby-19mode # JRuby in 1.9 mode
   - rbx-2
 matrix:
+  include:
+    - rvm: 2.0.0
+      gemfile: Gemfile_ruby2
+    - rvm: 2.1.8
+      gemfile: Gemfile_ruby2
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-19mode

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## Unreleased
+
+## [2.5.0] 2016-08-21
+### Fixed
+- Remove tempdirs via finalizers on garbage collection. This cleans them up in all known cases, rather than just when the #close method is called. The #close method can be used to cleanup early. [329](https://github.com/roo-rb/roo/pull/329)
+- Fixed README.md typo [318](https://github.com/roo-rb/roo/pull/318)
+- Parse sheets in ODS files once to improve performance [320](https://github.com/roo-rb/roo/pull/320)
+- Fix some Cell conversion issues [324](https://github.com/roo-rb/roo/pull/324) and [331](https://github.com/roo-rb/roo/pull/331)
+- Improved memory performance [332](https://github.com/roo-rb/roo/pull/332)
+- Added `no_hyperlinks` option to improve streamig performance [319](https://github.com/roo-rb/roo/pull/319) and [333](https://github.com/roo-rb/roo/pull/333)
+
+### Deprecations
+- Roo::Base::TEMP_PREFIX should be accessed via Roo::TEMP_PREFIX
+- The private Roo::Base#make_tempdir is now available at the class level in
+  classes that use tempdirs, added via Roo::Tempdir
+=======
+### Added
+- Discard hiperlinks lookups to allow streaming parsing without loading whole files
+
 ## [2.4.0] 2016-05-14
 ### Fixed  
 - Fixed opening spreadsheets with charts [315](https://github.com/roo-rb/roo/pull/315)

data/Gemfile

@@ -6,6 +6,7 @@ group :test do
   # additional testing libs
   gem 'webmock'
   gem 'shoulda'
+  gem 'activesupport', '< 5.0.0'
   gem 'rspec', '>= 3.0.0'
   gem 'vcr'
   gem 'simplecov', '>= 0.9.0', require: false

data/Gemfile_ruby2

@@ -0,0 +1,27 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  # additional testing libs
+  gem 'webmock'
+  gem 'shoulda'
+  gem 'rspec', '>= 3.0.0'
+  gem 'vcr'
+  gem 'simplecov', '>= 0.9.0', require: false
+  gem 'coveralls', require: false
+  # gem "pry"
+  gem "activesupport", "~> 4.2.0"
+end
+
+group :local_development do
+  gem "listen", "~> 3.0.6"
+  gem 'terminal-notifier-guard', require: false if RUBY_PLATFORM.downcase.include?('darwin')
+  gem 'guard-rspec', '>= 4.3.1', require: false
+  gem 'guard-minitest', require: false
+  gem 'guard-bundler', require: false
+  gem 'guard-preek', require: false
+  gem 'guard-rubocop', require: false
+  gem 'guard-reek', github: 'pericles/guard-reek', require: false
+  gem 'pry'
+end

data/README.md

@@ -50,7 +50,7 @@ ods.sheet(0).row(1)
 
 # Set the last sheet as the default sheet.
 ods.default_sheet = ods.sheets.last
-ods.default_sheet = s.sheets[3]
+ods.default_sheet = ods.sheets[2]
 ods.default_sheet = 'Sheet 3'
 
 # Iterate through each sheet

data/lib/roo.rb

@@ -9,6 +9,8 @@ module Roo
   autoload :Excelx,       'roo/excelx'
   autoload :CSV,          'roo/csv'
 
+  TEMP_PREFIX = 'roo_'.freeze
+
   CLASS_FOR_EXTENSION = {
     ods: Roo::OpenOffice,
     xlsx: Roo::Excelx,

data/lib/roo/base.rb

@@ -9,7 +9,6 @@ require 'roo/utils'
 class Roo::Base
   include Enumerable
 
-  TEMP_PREFIX = 'roo_'.freeze
   MAX_ROW_COL = 999_999.freeze
   MIN_ROW_COL = 0.freeze
 
@@ -18,6 +17,11 @@ class Roo::Base
   # sets the line with attribute names (default: 1)
   attr_accessor :header_line
 
+  def self.TEMP_PREFIX
+    warn '[DEPRECATION] please access TEMP_PREFIX via Roo::TEMP_PREFIX'
+    Roo::TEMP_PREFIX
+  end
+
   def initialize(filename, options = {}, _file_warning = :error, _tmpdir = nil)
     @filename = filename
     @options = options
@@ -32,14 +36,12 @@ class Roo::Base
     @last_column = {}
 
     @header_line = 1
-  rescue => e # clean up any temp files, but only if an error was raised
-    close
-    raise e
   end
 
   def close
-    return nil unless @tmpdirs
-    @tmpdirs.each { |dir| ::FileUtils.remove_entry(dir) }
+    if self.class.respond_to?(:finalize_tempdirs)
+      self.class.finalize_tempdirs(object_id)
+    end
     nil
   end
 
@@ -449,10 +451,6 @@ class Roo::Base
 
   private
 
-  def track_tmpdir!(tmpdir)
-    (@tmpdirs ||= []) << tmpdir
-  end
-
   def clean_sheet_if_need(options)
     return unless options[:clean]
     options.delete(:clean)
@@ -538,7 +536,7 @@ class Roo::Base
   def find_basename(filename)
     if uri?(filename)
       require 'uri'
-      uri = URI::parse filename
+      uri = URI.parse filename
       File.basename(uri.path)
     elsif !is_stream?(filename)
       File.basename(filename)
@@ -546,10 +544,15 @@ class Roo::Base
   end
 
   def make_tmpdir(prefix = nil, root = nil, &block)
+    warn '[DEPRECATION] extend Roo::Tempdir and use its .make_tempdir instead'
     prefix = "#{TEMP_PREFIX}#{prefix}"
+    root ||= ENV['ROO_TMP']
 
-    ::Dir.mktmpdir(prefix, root || ENV['ROO_TMP'], &block).tap do |result|
-      block_given? || track_tmpdir!(result)
+    if block_given?
+      # folder is deleted at end of block
+      ::Dir.mktmpdir(prefix, root, &block)
+    else
+      self.class.make_tempdir(self, prefix, root)
     end
   end
 

data/lib/roo/csv.rb

@@ -56,7 +56,7 @@ class Roo::CSV < Roo::Base
 
   def each_row(options, &block)
     if uri?(filename)
-      make_tmpdir do |tmpdir|
+      ::Dir.mktmpdir(Roo::TEMP_PREFIX, ENV['ROO_TMP']) do |tmpdir|
         tmp_filename = download_uri(filename, tmpdir)
         CSV.foreach(tmp_filename, options, &block)
       end

data/lib/roo/excelx.rb

@@ -1,11 +1,14 @@
 require 'nokogiri'
 require 'zip/filesystem'
 require 'roo/link'
+require 'roo/tempdir'
 require 'roo/utils'
 require 'forwardable'
 
 module Roo
   class Excelx < Roo::Base
+    extend Roo::Tempdir
+
     require 'set'
     extend Forwardable
 
@@ -36,13 +39,14 @@ module Roo
       cell_max = options.delete(:cell_max)
       sheet_options = {}
       sheet_options[:expand_merged_ranges] = (options[:expand_merged_ranges] || false)
+      sheet_options[:no_hyperlinks] = (options[:no_hyperlinks] || false)
 
       unless is_stream?(filename_or_stream)
         file_type_check(filename_or_stream, %w[.xlsx .xlsm], 'an Excel 2007', file_warning, packed)
         basename = find_basename(filename_or_stream)
       end
 
-      @tmpdir = make_tmpdir(basename, options[:tmpdir_root])
+      @tmpdir = self.class.make_tempdir(self, basename, options[:tmpdir_root])
       @shared = Shared.new(@tmpdir)
       @filename = local_filename(filename_or_stream, @tmpdir, packed)
       process_zipfile(@filename || filename_or_stream)
@@ -64,9 +68,9 @@ module Roo
       end
 
       super
-    rescue => e # clean up any temp files, but only if an error was raised
-      close
-      raise e
+    rescue
+      self.class.finalize_tempdirs(object_id)
+      raise
     end
 
     def method_missing(method, *args)

data/lib/roo/excelx/extractor.rb

@@ -8,7 +8,6 @@ module Roo
       private
 
       def doc
-        @doc ||=
         if doc_exists?
           ::Roo::Utils.load_xml(@path).remove_namespaces!
         end

data/lib/roo/excelx/shared_strings.rb

@@ -3,6 +3,14 @@ require 'roo/excelx/extractor'
 module Roo
   class Excelx
     class SharedStrings < Excelx::Extractor
+
+      COMMON_STRINGS = {
+        t: "t",
+        r: "r",
+        html_tag_open: "<html>",
+        html_tag_closed: "</html>"
+      }
+
       def [](index)
         to_a[index]
       end
@@ -26,18 +34,17 @@ module Roo
       def fix_invalid_shared_strings(doc)
         invalid = { '_x000D_'  => "\n" }
         xml = doc.to_s
+        return doc unless xml[/#{invalid.keys.join('|')}/]
 
-        if xml[/#{invalid.keys.join('|')}/]
-          @doc = ::Nokogiri::XML(xml.gsub(/#{invalid.keys.join('|')}/, invalid))
-        end
+        ::Nokogiri::XML(xml.gsub(/#{invalid.keys.join('|')}/, invalid))
       end
 
       def extract_shared_strings
         return [] unless doc_exists?
 
-        fix_invalid_shared_strings(doc)
+        document = fix_invalid_shared_strings(doc)
         # read the shared strings xml document
-        doc.xpath('/sst/si').map do |si|
+        document.xpath('/sst/si').map do |si|
           shared_string = ''
           si.children.each do |elem|
             case elem.name

data/lib/roo/excelx/sheet_doc.rb

@@ -39,8 +39,13 @@ module Roo
       def each_cell(row_xml)
         return [] unless row_xml
         row_xml.children.each do |cell_element|
-          key = ::Roo::Utils.ref_to_key(cell_element['r'])
-          yield cell_from_xml(cell_element, hyperlinks(@relationships)[key])
+          # If you're sure you're not going to need this hyperlinks you can discard it
+          hyperlinks = unless @options[:no_hyperlinks]
+                         key = ::Roo::Utils.ref_to_key(cell_element['r'])
+                         hyperlinks(@relationships)[key]
+                       end
+
+          yield cell_from_xml(cell_element, hyperlinks)
         end
       end
 
@@ -90,10 +95,9 @@ module Roo
         cell_xml.children.each do |cell|
           case cell.name
           when 'is'
-            cell.children.each do |inline_str|
-              if inline_str.name == 't'
-                return Excelx::Cell.create_cell(:string, inline_str.content, formula, style, hyperlink, coordinate)
-              end
+            content_arr = cell.search('t').map(&:content)
+            unless content_arr.empty?
+              return Excelx::Cell.create_cell(:string, content_arr.join(''), formula, style, hyperlink, coordinate)
             end
           when 'f'
             formula = cell.content
@@ -101,6 +105,8 @@ module Roo
             return create_cell_from_value(value_type, cell, formula, format, style, hyperlink, base_date, coordinate)
           end
         end
+
+        Excelx::Cell::Empty.new(coordinate)
       end
 
       def create_cell_from_value(value_type, cell, formula, format, style, hyperlink, base_date, coordinate)
@@ -157,8 +163,9 @@ module Roo
       end
 
       def extract_hyperlinks(relationships)
-        # FIXME: select the valid hyperlinks and then map those.
-        Hash[doc.xpath('/worksheet/hyperlinks/hyperlink').map do |hyperlink|
+        return {} unless (hyperlinks = doc.xpath('/worksheet/hyperlinks/hyperlink'))
+
+        Hash[hyperlinks.map do |hyperlink|
           if hyperlink.attribute('id') && (relationship = relationships[hyperlink.attribute('id').text])
             [::Roo::Utils.ref_to_key(hyperlink.attributes['ref'].to_s), relationship.attribute('Target').text]
           end

data/lib/roo/open_office.rb

@@ -3,10 +3,13 @@ require 'nokogiri'
 require 'cgi'
 require 'zip/filesystem'
 require 'roo/font'
+require 'roo/tempdir'
 require 'base64'
 
 module Roo
   class OpenOffice < Roo::Base
+    extend Roo::Tempdir
+
     ERROR_MISSING_CONTENT_XML = 'file missing required content.xml'.freeze
     XPATH_FIND_TABLE_STYLES   = "//*[local-name()='automatic-styles']".freeze
     XPATH_LOCAL_NAME_TABLE    = "//*[local-name()='table']".freeze
@@ -19,15 +22,27 @@ module Roo
 
       @only_visible_sheets = options[:only_visible_sheets]
       file_type_check(filename, '.ods', 'an Roo::OpenOffice', file_warning, packed)
-      @tmpdir   = make_tmpdir(find_basename(filename), options[:tmpdir_root])
+      @tmpdir   = self.class.make_tempdir(self, find_basename(filename), options[:tmpdir_root])
       @filename = local_filename(filename, @tmpdir, packed)
       # TODO: @cells_read[:default] = false
       open_oo_file(options)
       super(filename, options)
       initialize_default_variables
-    rescue => e # clean up any temp files, but only if an error was raised
-      close
-      raise e
+
+      unless @table_display.any?
+        doc.xpath(XPATH_FIND_TABLE_STYLES).each do |style|
+          read_table_styles(style)
+        end
+      end
+
+      @sheet_names = doc.xpath(XPATH_LOCAL_NAME_TABLE).map do |sheet|
+        if !@only_visible_sheets || @table_display[attribute(sheet, 'style-name')]
+          sheet.attributes['name'].value
+        end
+      end.compact
+    rescue
+      self.class.finalize_tempdirs(object_id)
+      raise
     end
 
     def open_oo_file(options)
@@ -132,16 +147,7 @@ module Roo
     end
 
     def sheets
-      unless @table_display.any?
-        doc.xpath(XPATH_FIND_TABLE_STYLES).each do |style|
-          read_table_styles(style)
-        end
-      end
-      doc.xpath(XPATH_LOCAL_NAME_TABLE).map do |sheet|
-        if !@only_visible_sheets || @table_display[attribute(sheet, 'style-name')]
-          sheet.attributes['name'].value
-        end
-      end.compact
+      @sheet_names
     end
 
     # version of the Roo::OpenOffice document

data/lib/roo/tempdir.rb

@@ -0,0 +1,26 @@
+module Roo
+  module Tempdir
+    def finalize_tempdirs(object_id)
+      if @tempdirs && (dirs_to_remove = @tempdirs[object_id])
+        @tempdirs[object_id] = nil
+        dirs_to_remove.each do |dir|
+          ::FileUtils.remove_entry(dir)
+        end
+      end
+    end
+
+    def make_tempdir(object, prefix, root)
+      root ||= ENV['ROO_TMP']
+      # folder is cleaned up in .finalize_tempdirs
+      ::Dir.mktmpdir("#{Roo::TEMP_PREFIX}#{prefix}", root).tap do |tmpdir|
+        @tempdirs ||= {}
+        if @tempdirs[object.object_id]
+          @tempdirs[object.object_id] << tmpdir
+        else
+          @tempdirs[object.object_id] = [tmpdir]
+          ObjectSpace.define_finalizer(object, method(:finalize_tempdirs))
+        end
+      end
+    end
+  end
+end

data/lib/roo/version.rb

@@ -1,3 +1,3 @@
 module Roo
-  VERSION = '2.4.0'
+  VERSION = '2.5.0'
 end

data/test/excelx/cell/test_datetime.rb

@@ -1,7 +1,4 @@
 require 'test_helper'
-# require 'pry'
-# require 'active_support'
-# require 'active_support/time'
 
 class TestRooExcelxCellDateTime < Minitest::Test
   def test_cell_value_is_datetime

data/test/test_roo.rb

@@ -17,6 +17,8 @@
 # with the wrong spreadsheet class
 #STDERR.reopen "/dev/null","w"
 
+Encoding.default_external = "UTF-8"
+
 require 'test_helper'
 require 'stringio'
 
@@ -2089,6 +2091,19 @@ where the expected result is
     end
   end
 
+  def test_finalize
+    tempdirs = []
+    begin
+      with_each_spreadsheet(:name=>'numbers1') do |oo|
+        tempdirs << oo.instance_variable_get('@tmpdir')
+      end
+      GC.start
+    end
+    tempdirs.each do |tempdir|
+      assert !File.exists?(tempdir), "Expected #{tempdir} to be cleaned up, but it still exists"
+    end
+  end
+
   def test_cleanup_on_error
     old_temp_files = Dir.open(Dir.tmpdir).to_a
     with_each_spreadsheet(:name=>'non_existent_file', :ignore_errors=>true) do |oo|; end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: roo
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Thomas Preymesser
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-14 00:00:00.000000000 Z
+date: 2016-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -101,6 +101,7 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
+- Gemfile_ruby2
 - Guardfile
 - LICENSE
 - README.md
@@ -139,6 +140,7 @@ files:
 - lib/roo/link.rb
 - lib/roo/open_office.rb
 - lib/roo/spreadsheet.rb
+- lib/roo/tempdir.rb
 - lib/roo/utils.rb
 - lib/roo/version.rb
 - roo.gemspec