dreader 0.5.0 → 1.1.0

data/lib/dreader/engine.rb

@@ -0,0 +1,495 @@
+require "roo"
+require "logger"
+require "fast_excel"
+
+require "dreader/column"
+require "dreader/options"
+require "dreader/util"
+
+module Dreader
+  #
+  # This is where the real stuff begins
+  #
+  module Engine
+    # the options we passed
+    attr_accessor :declared_options
+    # the specification of the columns to process
+    attr_accessor :declared_columns
+    # some example lines
+    attr_accessor :declared_examples
+    # the specification of the virtual columns
+    attr_accessor :declared_virtual_columns
+    # the mapping rules
+    attr_accessor :declared_mapping
+    
+    # the data we read
+    attr_reader :table
+
+    # define a DSL for options
+    # any string is processed as an option and it ends up in the
+    # @options hash
+    def options(&block)
+      options = Options.new
+      options.instance_eval(&block)
+
+      @declared_options = options.to_hash
+    end
+
+    # define a DSL for column specification
+    # - `name` is the name of the column
+    # - `block` contains two declarations, `process` and `check`, which are
+    #   used, respectively, to make a cell into the desired data and to check
+    #   whether the desired data is ok
+    def column(name, &block)
+      column = Column.new
+      column.instance_eval(&block)
+
+      @declared_columns ||= []
+
+      if name.instance_of?(Hash)
+        @declared_columns << column.to_hash.merge(
+          { name: name.keys.first, colref: name.values.first }
+        )
+      else
+        @declared_columns << column.to_hash.merge({ name: name })
+      end
+    end
+
+    # define a DSL for multiple column specification (bulk_declare)
+    #
+    # - hash is a hash in the form { symbolic_name: colref }
+    #
+    # i.bulk_declare {name: "B", age: "C"} is equivalent to:
+    # 
+    # i.column :name do
+    #   colref "B"
+    # end
+    # i.column :age do
+    #   colref "C"
+    # end
+    #
+    # i.bulk_declare {name: "B", age: "C"} do
+    #   process do |cell|
+    #     cell.strip
+    #   end
+    # end
+    #
+    # is equivalent to:
+    #
+    # i.column :name do
+    #   colref "B"
+    #   process do |cell|
+    #     cell.strip
+    #   end
+    # end
+    # i.column :age do
+    #   colref "C"
+    #   process do |cell|
+    #     cell.strip
+    #   end
+    # end
+    def columns(hash, &block)
+      hash.each_key do |key|
+        column = Column.new
+        column.colref hash[key]
+        column.instance_eval(&block) if block
+
+        @declared_columns ||= []
+        @declared_columns << column.to_hash.merge({ name: key })
+      end
+    end
+
+    def example(hash)
+      @declared_examples ||= []
+      @declared_examples << hash
+    end
+
+    # virtual columns define derived attributes
+    # the code specified in the virtual column is executed after reading
+    # a row and before applying the mapping function
+    #
+    # virtual colum declarations are executed in the order in which
+    # they are defined
+    def virtual_column(name, &block)
+      column = Column.new
+      column.instance_eval &block
+
+      @declared_virtual_columns ||= []
+      @declared_virtual_columns << column.to_hash.merge({ name: name })
+    end
+
+    # define what we do with each line we read
+    # - `block` is the code which takes as input a `row` and processes
+    #   `row` is a hash in which each spreadsheet cell is accessible under
+    #   the column names. Each cell has the following values:
+    #   :value, :error, :row_number, :col_number
+    def mapping(&block)
+      @declared_mapping = block
+    end
+
+    # read a file and store it internally
+    #
+    # @param hash, a hash, possibly overriding any of the parameters
+    #              set in the initial options.  This allows you, for
+    #              instance, to apply the same column specification to
+    #              different files and different sheets
+    #
+    # @return the data read from filename, in the form of an array of
+    #         hashes
+    def read(args = {})
+      # args override values in options (if defined)
+      # the initializer guarantees @options is at least {}
+      options = (@declared_options || {}).merge(args)
+
+      @logger = options[:logger] || Logger.new($stdout)
+      @logger.level = options[:logger_level] || Logger::WARN
+      @debug = options[:debug] == true
+
+      if !args.instance_of?(Hash)
+        @logger.error "#{__callee__}: this function takes a Hash as input"
+        raise Exception
+      end
+
+      spreadsheet = open_spreadsheet (options[:filename])
+      sheet = spreadsheet.sheet(options[:sheet] || 0)
+      first_row = options[:first_row] || 1
+      last_row = options[:last_row] || sheet.last_row
+
+      if @debug
+        # override some defaults
+        @logger.level = Logger::DEBUG
+
+        # override the number of lines read
+        n_lines = options[:n] ? options[:n].to_i : 10
+        last_row = first_row + n_lines - 1
+        
+        # apply some defaults for debugging, if not defined in the options
+        [:check_raw, :process, :check].map do |key|
+          options[key] = true unless options.key?(key)
+        end
+      end
+
+      { current: @declared_options, debug: options }.each do |k, v|
+        @logger.debug "[dreader] #{k.capitalize} configuration:"
+        v.each do |key, value|
+          @logger.debug "  #{key}: #{value}"
+        end
+      end
+      
+      @errors = []
+
+      @table = (first_row..last_row).map do |row_number|
+        r = { row_number: row_number, row_errors: [] }
+        
+        # this has side-effects on r
+        columns_on(r, row_number, sheet)
+
+        # this has side-effects on r
+        virtual_columns_on(r) if options[:virtual] || options[:mapping]
+
+        options[:mapping] ? mappings_on(r) : r
+      end
+    end
+
+    # TODO: PASS A ROW (and not row_number and sheet)
+    def columns_on(r, row_number, sheet)
+      @declared_columns.each_with_index do |colspec, index|
+        colname = colspec[:name]
+        colref = colspec[:colref]
+        cell = sheet.cell(row_number, colref)
+
+        r[colname] = {
+          row: row_number,
+          col: colspec[:colref],
+          value: cell,
+          error: false
+        }
+
+        # Repeated below
+        # @logger.debug "[dreader] Processing #{coord(row_number, colref)}"
+
+        # check raw data
+        check_data(colspec[:checks_raw], r, colname)
+
+        # process data
+        coord = coord(row_number, colspec[:colref], cell)
+        begin
+          processed = colspec[:process] ? colspec[:process].call(cell) : cell
+          @logger.debug "[dreader] #{colname} process #{coord} yields '#{processed}' (#{processed.class})"
+          r[colname][:value] = processed
+        rescue => e
+          @logger.error "[dreader] #{colname} process #{coord} raises an exception"
+          raise e
+        end
+
+        # check data after process - notice that now r contains the value
+        # processed by process
+        check_data(colspec[:checks], r, colname)
+      end
+
+      r
+    end
+
+    alias load read
+
+    # show to stdout the first `n` records we read from the file given the
+    # current configuration
+    def debug(args = {})
+      read(args.merge({ debug: true }))
+    end
+
+    # get (processed) row number
+    #
+    # - row_number is the row to get: index starts at 1.
+    #
+    # get_row(1) get the first line read, that is, the row specified
+    # by `first_row` in `options` (or in read)
+    #
+    # You need to invoke read first
+    def get_row(row_number)
+      if row_number > @table.size
+        @logger.error "[dreader] 'row_number' is out of range (did you invoke read?)"
+        exit
+      elsif row_number <= 0
+        @logger.error "[dreader] 'row_number' is zero or negative (first row is 1)."
+      else
+        @table[row_number - 1]
+      end
+    end
+
+    # return an array of hashes with all the errors we have encountered
+    # an empty array is a good news
+    attr_reader :errors
+
+    def virtual_columns
+      # execute the virtual column specification
+      @table.each { |row| virtual_columns_on(row) }
+    end
+
+    # Compute virtual columns for, with side effect on row
+    def virtual_columns_on(row)
+      @declared_virtual_columns.each do |virtualcol|
+        colname = virtualcol[:name]
+        row[colname] = { virtual: true }
+        
+        check_data(virtualcol[:checks_raw], row, colname, full_row: true)
+
+        begin
+          # add the cell to the table
+          if virtualcol[:process]
+            row[colname][:value] = virtualcol[:process].call(row)
+          end
+        rescue => e
+          r = row[:row_number]
+          @logger.error "[dreader] #{colname} process raises an exception at row #{r}"
+          raise e
+        end
+
+        # check data after process -- we also have the processed value of
+        # the virtual column
+        check_data(virtualcol[:checks], row, colname, full_row: true)
+      end
+    end
+
+    # apply the mapping code to the array it makes sense to invoke it only
+    # once.
+    #
+    # the mapping is applied only if it defined and it uses map, so that
+    # it can be used functionally
+    def mappings
+      @table.map { |row| mappings_on(row) }
+    end
+
+    def mappings_on(row)
+      @declared_mapping&.call(row)
+    end
+
+    # an alias
+    def data
+      @table
+    end
+
+    def to_s
+      @table.to_s
+    end
+
+    #
+    # headers validation functions
+    #
+    def correct_headers?(hash = {})
+      output = compare_headers
+      output.values.map { |x| x[:correct] }.all?(true)
+    end
+
+    def compare_headers(hash = {})
+      options = @declared_options.merge(hash)
+
+      spreadsheet = open_spreadsheet(options[:filename])
+      sheet = spreadsheet.sheet(options[:sheet] || 0)
+      header_row_number = options[:first_row] - 1 || 1
+
+      output_hash = {}
+      @declared_columns.map do |colspec|
+        cell = sheet.cell(row_number, colspec[:colref])
+        human_readable = colspec[:name].to_s.split("_").map(&:capitalize).join(" ")
+
+        output_hash[colspec[:colref]] = {
+          header_in_spec: colspec[:name],
+          human_readable: human_readable,
+          header_in_file: cell,
+          correct: cell == human_readable
+        }
+      end
+
+      output_hash
+    end
+
+    # generate a template from the column specification
+    # first row is a header, determined by colspec
+    # second row includes the documentation string, to document values in
+    # the columns
+    def template(hash = {})
+      options = @declared_options.merge(hash)
+      filename = options[:template_filename]
+
+      workbook = FastExcel.open(filename, constant_memory: true)
+      worksheet = workbook.add_worksheet("Template")
+      worksheet.auto_width = true
+
+      # first_row is indexed from 1 by roo and from 0 by FastExcel
+      first_row = [(options[:first_row] || 0) - 2, 0].max
+      bold = workbook.bold_format
+
+      #
+      # somehow fast excel seems to allow to set cells row by row
+      #
+
+      # here we write the first row
+      @declared_columns.each do |colspec|
+        human_readable = colspec[:name].to_s.split("_").map(&:capitalize).join(" ")
+        colref = colref_to_i(colspec[:colref])
+        worksheet.write_string(first_row, colref, human_readable, bold)
+      end
+
+      # here we create a note with the legenda
+      @declared_columns.each do |colspec|
+        colref = colref_to_i(colspec[:colref])
+        worksheet.write_string(first_row + 1, colref, colspec[:doc], nil)
+      end
+
+      # here we write some example records
+      @declared_examples.each_with_index do |example_hash, index|
+        example_hash.each do |colname, value|
+          colspec = @declared_columns.select { |x| x[:name] == colname }.first
+          if colspec
+            colref = colref_to_i(colspec[:colref])
+            worksheet.write_string(index + 3, colref, value, nil)
+          else
+            @logger.err "generate_template: #{colname} used in example is not defined"
+          end
+        end
+      end
+
+      workbook.close
+    end
+
+    private
+
+    # list of keys we support in options. We remove them when reading
+    # the CSV file
+    OPTION_KEYS = %i[
+      filename sheet first_row last_row logger logger_level
+    ]
+
+    def open_spreadsheet(filename)
+      ext = File.extname(filename)
+
+      case ext
+      when ".csv"
+        csv_options = @declared_options.except(*OPTION_KEYS)
+        Roo::CSV.new(filename, csv_options:)
+      when ".tsv"
+        csv_options = @declared_options.except(*OPTION_KEYS).merge({ col_sep: "\t" })
+        Roo::CSV.new(filename, csv_options:)
+      when ".ods"
+        Roo::OpenOffice.new(filename)
+      when ".xls"
+        Roo::Excel.new(filename)
+      when ".xlsx"
+        Roo::Excelx.new(filename)
+      else
+        raise "Unknown extension: #{ext}"
+      end
+    end
+
+    def colref_to_i(colref)
+      return colref if colref.instance_of?(Integer)
+      value = 0
+      power = 1
+      colref.to_s.reverse.split("").map do |char|
+        value = value + power * (1 + char.ord - 'A'.ord)
+        power = power * 26
+      end
+      value - 1
+    end
+
+    # performs check on data (this is the same code for check_raw and
+    # check)
+    #
+    # @params
+    # - check_spec :: the set of checks to perform (an array of hashes
+    #   in the form error_message: lambda
+    #
+    # - hash :: either the hash of a column or the hash of a row
+    #
+    # - colname :: the name of the column we are checking (if we check
+    #              a column) or the column we are computing (virtual
+    #              columns)
+    #
+    # - full_row :: if true, pass the full row to check rather than
+    #               the value of a column
+    #              
+    # - debug :: a boolean
+    def check_data(check_spec, hash, colname, full_row: false)
+      check_spec.each do |error_message, check_function|
+        # here we extract values by distinguishing whether the hash is that of
+        # column or that of a row
+        if full_row
+          value = hash
+          row = hash[:row_number]
+          col = :multiple_columns
+        else
+          value = hash[colname][:value]
+          row = hash[colname][:row]
+          col = hash[colname][:col]
+        end
+        coord = coord(row, col, value)
+
+        begin
+          pass = check_function.call(value)
+          @logger.debug "[dreader] check #{colname}/#{error_message} at #{coord} yields: '#{pass}'"
+
+          if pass != true
+            hash[colname][:error] = true
+            error = {
+              callee: __callee__,
+              cell_value: value, colname: colname,
+              row: row, col: col,
+              message: error_message,
+              content: pass
+            }
+            @errors << error
+            hash[:row_errors] << error
+          end
+        rescue => e
+          @logger.error "[dreader] check #{colname}/#{error_message} raises an exception at #{coord}"
+          raise e
+        end
+      end
+    end
+
+    def coord(row, col, value = nil)
+      ["#{row}#{col}", (value ? "(#{value})" : nil)].join(" ")
+    end
+  end
+end

data/lib/dreader/options.rb

@@ -0,0 +1,16 @@
+module Dreader
+  # service class to implement the options DSL language
+  class Options
+    def initialize
+      @attributes = {}
+    end
+
+    def method_missing(name, *args, &block)
+      @attributes[name] = args[0]
+    end
+
+    def to_hash
+      @attributes
+    end
+  end
+end

data/lib/dreader/util.rb

@@ -0,0 +1,86 @@
+module Dreader
+  # Utilities function to simplify importing data into
+  # ActiveRecords
+  class Util
+    # given a hash returned by Engine, return the same hash with
+    # keys directly bound to the content of the :value sub-key
+    #
+    # Example
+    #
+    # hash = {name: {value: "A", ...}, surname: {value: "B", ...}}
+    # simplify hash
+    # {name: "A", surname: "B"}
+    #
+    # remove all keys which are not part of the data read (row_number and
+    # row_errors)
+    def self.simplify(hash)
+      (hash.keys - %i[row_number row_errors]).map do |colname|
+        [colname, hash[colname][:value]]
+      end.to_h
+    end
+
+    # Given a "simplified" hash restructure it according to the second
+    # argument.
+    #
+    # Use it for generating hashes with nested attributes, which
+    # follows Rails conventions.
+    #
+    # @params:
+    # - hash the hash to restructure
+    # - args splat arguments which specify how to (re)structure the
+    #   values in Hash.  Each element is either a symbol or a Hash
+    #
+    # Example
+    #
+    # hash = { name: "A", surname: "B", address: "via Piave", city: "Genoa" }
+    #
+    # restructure(hash, :name, :surname)
+    # { name: "A", surname: "B" }
+    #
+    # restructure(hash, :name, address_attributes: [:address, :city])
+    # {name: "A", address_attributes: { address: "via Piave", city: "Genoa" }
+    #
+    def self.restructure(hash, *new_structure)
+      new_structure.to_h do |value|
+        if value.instance_of?(Hash)
+          [value.keys.first, self.restructure(hash, *value.values.first)]
+        else
+          [value, hash[value]]
+        end
+      end
+    end
+
+    # an alias for Hash.slice
+    # keys is an array of keys
+    def self.slice(hash, keys)
+      hash.slice(*keys)
+    end
+
+    # remove all `keys` from `hash`
+    def self.clean(hash, keys)
+      hash.reject { |key, _| keys.include?(key) }
+    end
+
+    # given a hash, return a new hash with key and whose value is
+    # the hash
+    #
+    # Example:
+    #
+    #    hash = {name: "A", size: 10}
+    #    prepend hash, :product_attributes
+    #    {product_attributes: {name: "A", size: 10}}
+    #
+    def self.prepend(hash, key)
+      { key => hash }
+    end
+
+    #
+    # Retrieve all errors related to row/col from and Array of error messages
+    #
+    def self.errors(errors_array, row, col = nil)
+      errors_array.select do |error|
+        error[:row] == row && (col.nil? || error[:col] == col)
+      end
+    end
+  end
+end

data/lib/dreader/version.rb

@@ -1,3 +1,3 @@
 module Dreader
-  VERSION = "0.5.0"
+  VERSION = "1.1.0"
 end