libis-tools 0.9.44 → 0.9.45

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f31ca736bcd249250f63da7beb8077f62884d13
-  data.tar.gz: 39082c83a169c8d1327d04b7abf0cc4be359c5e9
+  metadata.gz: 8e5a13d476bc924187b3f55cfc73ee1ca625cc81
+  data.tar.gz: a303e11aac907b4b2872bfd6d29081468d964d37
 SHA512:
-  metadata.gz: 50863594daeb15513f04c45362499b48530b68d44a5488632a23eeb1624cbe1a9822ea007559fe76aab3561cfcfd51ec1259971aed8c096786389e33a2c7b98a
-  data.tar.gz: 148b11b3ed23b84b99175642ea662d0f43e0c1d3c797cbfd889b99440c66e51935975332f4fc41d707722bc4936f87d421bef131fbeeebdcf61165ea0ea113e3
+  metadata.gz: 8f13fccc9695b41c540c784d9e154a86879c1fcfe0fbec35bf9d1b1a5ea5133ea2409ad199da2cb88af97fbac21bca442ab3263a1ea835e426cb3d6693b7186e
+  data.tar.gz: baa9a0c4a0fa1ca8aa4de0f8c9f7cb23b1fa419fbabdad307b6a748e2b2d20402f7f348622114b92fe5838093c386a9d136ae3b56487f9a6fd2dc4ac961a3bdc

data/.travis.yml

@@ -5,7 +5,7 @@ rvm:
   - 2.1.0
   - 2.2.0
   - ruby-head
-  - jruby-9.0.1.0
+  - jruby-9.1.5.0
 jdk:
   - openjdk7
   - oraclejdk7
@@ -26,6 +26,7 @@ matrix:
       jdk: oraclejdk8
   allow_failures:
     - rvm: ruby-head
+    - rvm: jruby-9.1.5.0
 branches:
   only:
       - master

data/lib/libis/tools/extend/roo.rb

@@ -1,29 +1,43 @@
 require 'roo'
 require 'roo-xls'
 require 'roo-google'
+require 'libis/tools/extend/hash'
 
 module Roo
   class HeaderRowIncompleteError < Error;
   end
   class Base
 
+    # changes:
+    # - added option :skip_header to prevent #each and #parse to return the header row
+    # - added option :partial_match to allow to use headers that only partially match the query
+    # - added option :required to force the result to have at least these columns
+    # - allow option :headers to contain an array with header labels that will be forced when no header row is found
+    # - improved proper range scanning (first_row->last_row and first_column->last_column)
+
+    attr_accessor :partial_match
+
     def each(options = {})
       return to_enum(:each, options) unless block_given?
 
+      skip_headers = options.delete(:skip_headers)
+      @partial_match = options.delete(:partial_match) if options.has_key?(:partial_match)
+      required_headers = options.delete(:required_headers) if options.has_key?(:required_headers)
+
       if options.empty?
-        1.upto(last_row) do |line|
+        first_row.upto(last_row) do |line|
+          next if skip_headers && line == header_line
           yield row(line)
         end
       else
         clean_sheet_if_need(options)
-        search_or_set_header(options)
-        headers = @headers ||
-            Hash[(first_column..last_column).map do |col|
-              [cell(@header_line, col), col]
-            end]
+        @headers = search_or_set_header(options)
+        if required_headers
+          raise Roo::HeaderRowIncompleteError unless headers.keys & required_headers == required_headers
+        end
 
-        start_line = @header_line || 1
-        start_line = (@header_line || 0) + 1 if @options[:skip_headers]
+        start_line = header_line
+        start_line += 1 if skip_headers
         start_line.upto(last_row) do |line|
           yield(Hash[headers.map { |k, v| [k, cell(line, v)] }])
         end
@@ -32,54 +46,51 @@ module Roo
 
     private
 
-    def row_with(query, return_headers = false)
-      line_no = 0
+    def row_with(query)
+      line_no = first_row
       each do |row|
-        line_no += 1
         headers = query.map { |q| row.grep(q)[0] }.compact
 
         if headers.length == query.length
           @header_line = line_no
-          return return_headers ? headers : line_no
+          return headers
         elsif line_no > 100
           raise Roo::HeaderRowNotFoundError
         elsif headers.length > 0
           # partial match
           @header_line = line_no
-          raise Roo::HeaderRowIncompleteError unless @options[:force_headers]
-          return return_headers ? headers : line_no
+          raise Roo::HeaderRowIncompleteError unless partial_match
+          return headers
         end
+        line_no += 1
       end
       raise Roo::HeaderRowNotFoundError
     end
 
     def search_or_set_header(options)
+      force_headers = options.delete(:headers)
       if options[:header_search]
-        @headers = nil
-        @header_line = row_with(options[:header_search])
-      elsif [:first_row, true].include?(options[:headers])
-        @headers = Hash[row(first_row).map.with_index{ |x, i| [x, i + first_column] }]
+        row_with(options[:header_search])
+      elsif [:first_row, true].include?(force_headers)
         @header_line = first_row
       else
-        set_headers(options)
+        return set_headers(options)
       end
+      return Hash[row(header_line).map { |x| [x, header_index(x)] }]
+    rescue Roo::HeaderRowNotFoundError => e
+      # Not OK unless a list of headers is supplied
+      raise e unless force_headers.is_a?(Array)
+      # Force the headers in the order they are given, but up to the last column
+      @header_line = first_row - 1
+      return Hash[force_headers.zip(first_column..last_column)].cleanup
     end
 
-    def set_headers(hash = {})
+    def set_headers(hash)
       # try to find header row with all values or give an error
       # then create new hash by indexing strings and keeping integers for header array
-      @headers = row_with(hash.values, true)
-      @headers = Hash[hash.keys.zip(@headers.map { |x| header_index(x) })]
-    rescue Roo::HeaderRowNotFoundError => e
-      if @options[:force_headers]
-        # Finding headers failed. Force the headers in the order they are given, but up to the last column
-        @headers = {}
-        hash.keys.each.with_index { |k, i| @headers[k] = i + first_column if i + first_column <= last_column }
-        @header_line = first_row
-        @header_line -= 1 unless hash.values.any? { |v| row(1).include? v } # partial match
-      else
-        raise e
-      end
+      row_with(hash.values)
+      positions = Hash[row(header_line).map { |x| [x, header_index(x)] }]
+      Hash[positions.map { |k, v| [hash.invert[k] || k, v] }]
     end
 
   end

data/lib/libis/tools/spreadsheet.rb

@@ -19,10 +19,13 @@ module Libis
       # - required: a list of headers that need to be present. The list can be an Array containing the litteral header
       #   values expected. Alternatively, a Hash is also allowed with alternative header names as keys and litteral
       #   names as values. If a :headers keys is present in the Hash with a value of true or :first, whatever is on the
-      #   first row, will be used as header values, ignoring the rest of the Hash. A key of :header_search
+      #   first row, will be used as header values, ignoring the rest of the Hash. A key of :header_search with an array
+      #   of strings as value will search for a row that contains each of the strings in the given array. Each string is
+      #   searched by regular expression, so strings may contain wildcards.
       #   Default is empty array, meaning to use whatever is on the first row as header.
       # - optional: a list of headers that may be present, but are not required. Similar format as above. Default is
-      #   empty array
+      #   empty array.
+      # - noheader: a list of headers to force upon the sheet if no headers are present.
       # - extension: :csv, :xlsx, :xlsm, :ods, :xls, :google to help the library in deciding what format the file is in.
       #
       # The following options are only applicable to CSV input files and are ignored otherwise.
@@ -31,8 +34,6 @@ module Libis
       # - col_sep: column separator. Default is ',', but can be set to "\t" for TSV files.
       # - quote_char: character for quoting.
       #
-      # Resources are created during initialisation and should be freed by calling the #close method.
-      #
       # @param [String] file_name
       # @param [Hash] opts
       def initialize(file_name, opts = {})
@@ -45,18 +46,19 @@ module Libis
                 col_sep: ',',
                 quote_char: '"',
             ),
-            skip_headers: true,
-            force_headers: true,
         }.merge(opts)
 
         required_headers = options.delete(:required) || []
         optional_headers = options.delete(:optional) || []
+        noheader_headers = options.delete(:noheader) || []
 
         file, sheet = file_name.split('|')
         @ss = ::Roo::Spreadsheet.open(file, options)
         @ss.default_sheet = sheet if sheet
 
-        check_headers(required_headers, optional_headers)
+        @header_options = {}
+
+        check_headers(required: required_headers, optional: optional_headers, noheader: noheader_headers)
 
       end
 
@@ -66,6 +68,7 @@ module Libis
       # - :sheet - overwrites default sheet name
       # - :required - Array or Hash of required headers
       # - :optional - Array or Hash of optional headers
+      # - :noheader - Array of noheader headers
       #
       # Each iteration, a Hash will be passed with the key names as specified in the header options and the
       # corresponding cell values.
@@ -73,20 +76,39 @@ module Libis
       # @param [Hash] options
       def each(options = {}, &block)
         @ss.default_sheet = options[:sheet] if options[:sheet]
-        @ss.each(check_headers(options[:required], options[:optional]), &block)
+        @ss.each(check_headers(options), &block)
       end
 
+      # Parse sheet content.
+      #
+      # The options Hash can contain the following keys:
+      # - :sheet - overwrites default sheet name
+      # - :required - Array or Hash of required headers
+      # - :optional - Array or Hash of optional headers
+      # - :noheader - Array of noheader headers
+      #
+      # An Array will be returned with for each row a Hash with the key names as specified in the header options and the
+      # corresponding cell values.
+      #
+      # @param [Hash] options
+      # @return [Array<Hash>]
       def parse(options = {})
-        @ss.default_sheet = options[:sheet] if options[:sheet]
-        @ss.parse(check_headers(options[:required], options[:optional]))
+        @ss.default_sheet = options.delete(:sheet) if options.has_key?(:sheet)
+        @ss.parse(check_headers(options))
       end
 
+      # Return the current row and increment the current_row pointer.
       def shift
         return nil unless @current_row < @ss.last_row
         @current_row += 1
         Hash[@ss.row(@current_row).map.with_index { |v, i| [headers[i], v] }]
       end
 
+      # Set the current_row pointer back to the start
+      def restart
+        @current_row = @ss.header_line
+      end
+
       # Open and iterate over sheet content.
       #
       # @param @see #initialize
@@ -95,60 +117,77 @@ module Libis
       end
 
       def headers
-        (@ss.headers || {}).keys + @extra_headers
+        (@ss.headers || {}).keys
       end
 
       private
 
-      def check_headers(required_headers, optional_headers)
-        return @header_options unless required_headers || optional_headers
-        header_options = {}
-        required_headers ||= []
-        optional_headers ||= []
-        unless required_headers.is_a?(Hash) || required_headers.is_a?(Array)
-          raise RuntimeError, 'Required headers should be either a Hash or an Array.'
-        end
-        unless optional_headers.is_a?(Hash) || optional_headers.is_a?(Array)
-          raise RuntimeError, 'Optional headers should be either a Hash or an Array.'
-        end
-        if required_headers.empty?
-          if optional_headers.empty?
-            header_options[:headers] = :first_row
-          else
-            header_options[:header_search] =
-                (optional_headers.is_a?(Hash) ? optional_headers.values : optional_headers)
+      def check_headers(options = {})
+        if options[:required] || options[:optional] || options[:noheader]
+
+          # defaults
+          ss_options = {}
+          required_headers = options[:required] || []
+          optional_headers = options[:optional] || []
+
+          # make sure required_headers is a Hash
+          case required_headers
+            when Hash
+              # OK
+            when Array
+              required_headers = Hash[required_headers.zip(required_headers)]
+            else
+              raise RuntimeError, 'Required headers should be either a Hash or an Array.'
           end
-        else
-          header_options =
-              required_headers.is_a?(Hash) ? required_headers : Hash[required_headers.map { |x| [x] * 2 }]
-          header_options.merge!(
-              optional_headers.is_a?(Hash) ? optional_headers : Hash[optional_headers.map { |x| [x] * 2 }]
-          )
-        end
 
-        required_headers = required_headers.values if required_headers.is_a?(Hash)
-
-        @ss.each(header_options) { break }
-        @current_row = @ss.header_line
+          # make sure optional_headers is a Hash
+          case optional_headers
+            when Hash
+              # OK
+            when Array
+              optional_headers = Hash[optional_headers.zip(optional_headers)]
+            else
+              raise RuntimeError, 'Optional headers should be either a Hash or an Array.'
+          end
 
-        # checks
-        found_headers = required_headers & @ss.row([@current_row, 1].max)
-        if found_headers.empty?
-          # No headers found - check if there are enough columns to satisfy the required headers
-          if required_headers.size > (@ss.last_column - @ss.first_column) + 1
-            raise RuntimeError, 'Sheet does not contain enough columns.'
+          # make sure noheader_headers is properly intialized
+          noheader_headers = options[:noheader]
+          raise RuntimeError, 'Noheader headers should be an Array.' unless noheader_headers.is_a?(Array)
+
+          # if not set, default to both required and optional headers
+          noheader_headers = (required_headers.keys + optional_headers.keys) if noheader_headers.empty?
+
+          # force noheader_headers or just use first row
+          ss_options[:headers] = noheader_headers.empty? ? :first_row : noheader_headers
+
+          # search for whatever whas supplied
+          ss_options.merge!(required_headers).merge!(optional_headers)
+
+          # allow partial match for only required headers
+          ss_options[:partial_match] = true
+          ss_options[:required_headers] = required_headers.keys
+
+          # force a header check (may throw exceptions)
+          begin
+            @ss.each(ss_options.dup) { break }
+          rescue Roo::HeaderRowNotFoundError
+            found_headers = required_headers.keys & @ss.headers.keys
+            raise RuntimeError, "Headers not found: #{required_headers.keys - found_headers}."
+          rescue Roo::HeaderRowIncompleteError
+            if @ss.row(@ss.header_line).compact.empty?
+              raise RuntimeError, 'Sheet does not contain enough columns.'
+            else
+              found_headers = required_headers.keys & @ss.headers.keys
+              raise RuntimeError, "Headers not found: #{required_headers.keys - found_headers}."
+
+            end
           end
-        elsif found_headers.size < required_headers.size
-          # Some, but not all headers found
-          raise RuntimeError, "Headers not found: #{required_headers - found_headers}."
-        else
-          # All required headers found
-        end
 
-        @extra_headers = (required_headers.empty? && optional_headers.empty?) ? [] :
-            @ss.row(@ss.header_line).keep_if { |x| x && !header_options.values.include?(x) }
+          @current_row = @ss.header_line
+          @header_options = ss_options.merge(skip_headers: true)
+        end
 
-        @header_options = header_options.merge(Hash[@extra_headers.map { |v| [v] * 2 }])
+        @header_options.dup
       end
 
     end

data/lib/libis/tools/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Tools
-    VERSION = '0.9.44'
+    VERSION = '0.9.45'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-tools
 version: !ruby/object:Gem::Version
-  version: 0.9.44
+  version: 0.9.45
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-22 00:00:00.000000000 Z
+date: 2016-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler