dreader 0.4.2 → 1.0.0

data/lib/dreader/engine.rb

@@ -0,0 +1,473 @@
+require "roo"
+require "logger"
+require "fast_excel"
+
+require "dreader/column"
+require "dreader/options"
+require "dreader/util"
+
+module Dreader
+  #
+  # This is where the real stuff begins
+  #
+  class Engine
+    # TODO: make the writer into private methods (need to be accessed only
+    # in the initializer) and demote to attr_reader
+    
+    # the options we passed
+    attr_accessor :options
+    # the specification of the columns to process
+    attr_accessor :colspec
+    # some example lines
+    attr_accessor :examples
+    # the specification of the virtual columns
+    attr_accessor :virtualcols
+    # the mapping rules
+    attr_accessor :mapping
+    
+    # the data we read
+    attr_reader :table
+
+    # variables declared in the class which need to be propagated in
+    # the instance
+    INSTANTIATE = %i[options colspec examples virtualcols]
+
+    def initialize
+      @logger = Logger.new($stdout)
+      @logger.level = Logger::WARN
+
+      # populate the instance with the variables defined in the class
+      @options = defined?(@@options) ? @@options : {}
+      @colspec = defined?(@@colspec) ? @@colspec : []
+      @examples = defined?(@@examples) ? @@examples : []
+      @virtualcols = defined?(@@virtualcols) ? @@virtualcols : []
+    end
+
+    # define a DSL for options
+    # any string is processed as an option and it ends up in the
+    # @options hash
+    def self.options(&block)
+      options = Options.new
+      options.instance_eval(&block)
+
+      @@options = options.to_hash
+    end
+
+    # define a DSL for column specification
+    # - `name` is the name of the column
+    # - `block` contains two declarations, `process` and `check`, which are
+    #   used, respectively, to make a cell into the desired data and to check
+    #   whether the desired data is ok
+    def self.column(name, &block)
+      column = Column.new
+      column.instance_eval(&block)
+
+      @@colspec ||= []
+
+      if name.instance_of?(Hash)
+        @@colspec << column.to_hash.merge(
+          { name: name.keys.first, colref: name.values.first }
+        )
+      else
+        @@colspec << column.to_hash.merge({ name: name })
+      end
+    end
+
+    # define a DSL for multiple column specification (bulk_declare)
+    #
+    # - hash is a hash in the form { symbolic_name: colref }
+    #
+    # i.bulk_declare {name: "B", age: "C"} is equivalent to:
+    # 
+    # i.column :name do
+    #   colref "B"
+    # end
+    # i.column :age do
+    #   colref "C"
+    # end
+    #
+    # i.bulk_declare {name: "B", age: "C"} do
+    #   process do |cell|
+    #     cell.strip
+    #   end
+    # end
+    #
+    # is equivalent to:
+    #
+    # i.column :name do
+    #   colref "B"
+    #   process do |cell|
+    #     cell.strip
+    #   end
+    # end
+    # i.column :age do
+    #   colref "C"
+    #   process do |cell|
+    #     cell.strip
+    #   end
+    # end
+    def self.columns(hash, &block)
+      hash.each_key do |key|
+        column = Column.new
+        column.colref hash[key]
+        column.instance_eval(&block) if block
+
+        @@colspec ||= []
+        @@colspec << column.to_hash.merge({ name: key })
+      end
+    end
+
+    def self.example(hash)
+      @@examples ||= []
+      @@examples << hash
+    end
+
+    # virtual columns define derived attributes
+    # the code specified in the virtual column is executed after reading
+    # a row and before applying the mapping function
+    #
+    # virtual colum declarations are executed in the order in which
+    # they are defined
+    def self.virtual_column(name, &block)
+      column = Column.new
+      column.instance_eval &block
+
+      @@virtualcols ||= []
+      @@virtualcols << column.to_hash.merge({ name: name })
+    end
+
+    # define what we do with each line we read
+    # - `block` is the code which takes as input a `row` and processes
+    #   `row` is a hash in which each spreadsheet cell is accessible under
+    #   the column names. Each cell has the following values:
+    #   :value, :error, :row_number, :col_number
+    def self.mapping(&block)
+      @@mapping = block
+    end
+
+    # read a file and store it internally
+    #
+    # @param hash, a hash, possibly overriding any of the parameters
+    #              set in the initial options.  This allows you, for
+    #              instance, to apply the same column specification to
+    #              different files and different sheets
+    #
+    # @return the data read from filename, in the form of an array of
+    #         hashes
+    def read(args = {})
+      if !args.instance_of?(Hash)
+        @logger.error "#{__callee__}: this function takes a Hash as input"
+        raise Exception
+      end
+
+      options = (@options || {}).merge(args)
+
+      @logger = options[:logger] if options[:logger]
+      @logger.level = options[:logger_level] if options[:logger_level]
+      @debug = options[:debug] == true
+
+      spreadsheet = Dreader::Engine.open_spreadsheet (options[:filename])
+      sheet = spreadsheet.sheet(options[:sheet] || 0)
+      first_row = options[:first_row] || 1
+      last_row = options[:last_row] || sheet.last_row
+
+      if @debug
+        # override some defaults
+        @logger.level = Logger::DEBUG
+
+        # override the number of lines read
+        options[:n] ||= 10 unless options[:n]
+        last_row = options[:first_row] + options[:n].to_i - 1
+      
+        # apply some defaults for debugging, if not defined in the options
+        [:check_raw, :process, :check].map do |key|
+          options[key] = true unless options.key?(key)
+        end
+      end
+
+      { current: @options, debug: options}.each do |k, v|
+        @logger.debug "#{k.capitalize} configuration:"
+        v.each do |key, value|
+          @logger.debug "  #{key}: #{value}"
+        end
+      end
+      
+      @table = []
+      @errors = []
+
+      (first_row..last_row).each do |row_number|
+        r = { row_number: row_number, row_errors: [] }
+        
+        @colspec.each_with_index do |colspec, index|
+          colref = colspec[:colref]
+          cell = sheet.cell(row_number, colref)
+          colname = colspec[:name]
+
+          r[colname] = {
+            row: row_number,
+            col: colspec[:colref],
+            value: cell,
+            error: false
+          }
+          @logger.debug "#{__callee__} [ROW DECL] Processing row: #{row_number}, col: #{colref}"
+
+          # check raw data
+          check_data(colspec[:checks_raw], r, colname)
+
+          # process data
+          coord = coord(row_number, colspec[:colref], cell)
+          begin
+            processed = colspec[:process] ? colspec[:process].call(cell) : cell
+            @logger.debug "process on #{colname} at #{coord} yields: '#{processed}' (#{processed.class})"
+            r[colname][:value] = processed
+          rescue => e
+            @logger.error "#{__callee__}: process on :#{colname} raised an exception at #{coord}"
+            raise e
+          end
+
+          # check data after process - notice that now r contains the value
+          # processed by process
+          check_data(colspec[:checks], r, colname)
+        end
+
+        @table << r
+      end
+
+      @table
+    end
+
+    alias load read
+
+    # show to stdout the first `n` records we read from the file given the
+    # current configuration
+    def debug(args = {})
+      read(args.merge({ debug: true }))
+    end
+
+    # get (processed) row number
+    #
+    # - row_number is the row to get: index starts at 1.
+    #
+    # get_row(1) get the first line read, that is, the row specified
+    # by `first_row` in `options` (or in read)
+    #
+    # You need to invoke read first
+    def get_row(row_number)
+      if row_number > @table.size
+        @logger.error "#{__callee__}: 'row_number' is out of range (did you invoke read?)"
+        exit
+      elsif row_number <= 0
+        @logger.error "#{__callee__}: 'row_number' is zero or negative (first row is 1)."
+      else
+        @table[row_number - 1]
+      end
+    end
+
+    # return an array of hashes with all the errors we have encounterd
+    # an empty array is a good news
+    attr_reader :errors
+
+    def virtual_columns(hash = {})
+      # execute the virtual column specification
+      @table.each do |r|
+        @virtualcols.each do |virtualcol|
+          colname = virtualcol[:name]
+          r[colname] = { virtual: true }
+          
+          check_data(virtualcol[:checks_raw], r, colname, full_row: true)
+
+          begin
+            # add the cell to the table
+            if virtualcol[:process]
+              r[colname][:value] = virtualcol[:process].call(r)
+            end
+          rescue => e
+            row = r[:row_number]
+            @logger.error "#{__callee__}: process for virtual column :#{colname} raised an exception at row #{row}"
+            raise e
+          end
+
+          # check data after process -- we also have the processed value of the virtual column
+          check_data(virtualcol[:checks], r, colname, full_row: true)
+        end
+      end
+    end
+
+    # apply the mapping code to the array
+    # it makes sense to invoke it only once
+    #
+    # the mapping is applied only if it defined
+    def process
+      @table.each { |row| @mapping&.call(row) }
+    end
+
+    def to_s
+      @table.to_s
+    end
+
+    #
+    # headers validation functions
+    #
+    def correct_headers?(hash = {})
+      output = compare_headers
+      output.values.map { |x| x[:correct] }.all?(true)
+    end
+
+    def compare_headers(hash = {})
+      options = @options.merge(hash)
+
+      spreadsheet = Dreader::Engine.open_spreadsheet(options[:filename])
+      sheet = spreadsheet.sheet(options[:sheet] || 0)
+      header_row_number = options[:first_row] - 1 || 1
+
+      output_hash = {}
+      @colspec.map do |colspec|
+        cell = sheet.cell(row_number, colspec[:colref])
+        human_readable = colspec[:name].to_s.split("_").map(&:capitalize).join(" ")
+
+        output_hash[colspec[:colref]] = {
+          header_in_spec: colspec[:name],
+          human_readable: human_readable,
+          header_in_file: cell,
+          correct: cell == human_readable
+        }
+      end
+
+      output_hash
+    end
+
+    # generate a template from the column specification
+    # first row is a header, determined by colspec
+    # second row includes the documentation string, to document values in
+    # the columns
+    def template(hash = {})
+      options = @options.merge(hash)
+      filename = options[:template_filename]
+
+      workbook = FastExcel.open(filename, constant_memory: true)
+      worksheet = workbook.add_worksheet("Template")
+      worksheet.auto_width = true
+
+      # first_row is indexed from 1 by roo and from 0 by FastExcel
+      first_row = [(options[:first_row] || 0) - 2, 0].max
+      bold = workbook.bold_format
+
+      #
+      # somehow fast excel seems to allow to set cells row by row
+      #
+
+      # here we write the first row
+      @colspec.each do |colspec|
+        human_readable = colspec[:name].to_s.split("_").map(&:capitalize).join(" ")
+        colref = colref_to_i(colspec[:colref])
+        worksheet.write_string(first_row, colref, human_readable, bold)
+      end
+
+      # here we create a note with the legenda
+      @colspec.each do |colspec|
+        colref = colref_to_i(colspec[:colref])
+        worksheet.write_string(first_row + 1, colref, colspec[:doc], nil)
+      end
+
+      # here we write some example records
+      @examples.each_with_index do |example_hash, index|
+        example_hash.each do |colname, value|
+          colspec = @colspec.select { |x| x[:name] == colname }.first
+          if colspec
+            colref = colref_to_i(colspec[:colref])
+            worksheet.write_string(index + 3, colref, value, nil)
+          else
+            @logger.err "generate_template: #{colname} used in example is not defined"
+          end
+        end
+      end
+
+      workbook.close
+    end
+
+    private
+
+    def self.open_spreadsheet(filename)
+      ext = File.extname(filename)
+
+      case ext
+      when ".csv" then Roo::CSV.new(filename)
+      when ".tsv" then Roo::CSV.new(filename, csv_options: { col_sep: "\t" })
+      when ".ods" then Roo::OpenOffice.new(filename)
+      when ".xls" then Roo::Excel.new(filename)
+      when ".xlsx" then Roo::Excelx.new(filename)
+      else raise "Unknown extension: #{ext}"
+      end
+    end
+
+    def colref_to_i(colref)
+      return colref if colref.instance_of?(Integer)
+      value = 0
+      power = 1
+      colref.to_s.reverse.split("").map do |char|
+        value = value + power * (1 + char.ord - 'A'.ord)
+        power = power * 26
+      end
+      value - 1
+    end
+
+    # performs check on data (this is the same code for check_raw and
+    # check)
+    #
+    # @params
+    # - check_spec :: the set of checks to perform (an array of hashes
+    #   in the form error_message: lambda
+    #
+    # - hash :: either the hash of a column or the hash of a row
+    #
+    # - colname :: the name of the column we are checking (if we check
+    #              a column) or the column we are computing (virtual
+    #              columns)
+    #
+    # - full_row :: if true, pass the full row to check rather than
+    #               the value of a column
+    #              
+    # - debug :: a boolean
+    def check_data(check_spec, hash, colname, full_row: false)
+      check_spec.each do |error_message, check_function|
+        # here we extract values by distinguishing whether the hash is that of
+        # column or that of a row
+        if full_row
+          value = hash
+          row = hash[:row_number]
+          col = :multiple_columns
+        else
+          value = hash[colname][:value]
+          row = hash[colname][:row]
+          col = hash[colname][:col]
+        end
+        coord = coord(row, col, value)
+
+        begin
+          pass = check_function.call(value)
+          @logger.debug "check for #{colname}/#{error_message} at #{coord} yields: '#{pass}'"
+
+          if pass != true
+            hash[colname][:error] = true
+            error = {
+              callee: __callee__,
+              cell_value: value, colname: colname,
+              row: row, col: col,
+              message: error_message,
+              content: pass
+            }
+            @errors << error
+            hash[:row_errors] << error
+          end
+        rescue => e
+          @logger.error "#{__callee__} for #{colname}/#{error_message} raised an exception at #{coord}"
+          raise e
+        end
+      end
+    end
+
+    def coord(row, col, cell)
+      "row: #{row}, col: #{col}, value: #{cell}"
+    end
+  end
+end

data/lib/dreader/options.rb

@@ -0,0 +1,16 @@
+module Dreader
+  # service class to implement the options DSL language
+  class Options
+    def initialize
+      @attributes = {}
+    end
+
+    def method_missing(name, *args, &block)
+      @attributes[name] = args[0]
+    end
+
+    def to_hash
+      @attributes
+    end
+  end
+end

data/lib/dreader/util.rb

@@ -0,0 +1,71 @@
+module Dreader
+  # Utilities function to simplify importing data into
+  # ActiveRecords
+  class Util
+    # given a hash returned by Engine, return the same hash with
+    # keys directly bound to the content of the :value sub-key
+    #
+    # Example
+    #
+    # hash = {name: {value: "A", ...}, surname: {value: "B", ...}}
+    # simplify hash
+    # {name: "A", surname: "B"}
+    #
+    # remove all keys which are not part of the data read (row_number and
+    # row_errors)
+    def self.simplify(hash)
+      (hash.keys - %i[row_number row_errors]).map do |colname|
+        [colname, hash[colname][:value]]
+      end.to_h
+    end
+
+    # given a hash returned by Engine, keep the "kept" keys in the top
+    # of the hierarchy and move the "moved_key" below the
+    # "subordinate_key"
+    #
+    # Example
+    #
+    # hash = { name: "A", surname: "B", address: "via XX Settembre", city: "Genoa" }
+    # restructure hash, [:name, :surname], :address_attributes, [:address, :city]
+    # {name: "A", surname: "B", address_attributes: {address: "via XX Settembre", city: "Genoa"}}
+    #
+    def self.restructure(hash, kept, subordinate_key, moved_keys)
+      head = hash.slice kept
+      subordinate = prepend subordinate_key, hash.slice(moved_keys)
+      head.merge subordinate
+    end
+
+    # an alias for Hash.slice
+    # keys is an array of keys
+    def self.slice(hash, keys)
+      hash.slice(*keys)
+    end
+
+    # remove all `keys` from `hash`
+    def self.clean(hash, keys)
+      hash.reject { |key, _| keys.include?(key) }
+    end
+
+    # given a hash, return a new hash with key and whose value is
+    # the hash
+    #
+    # Example:
+    #
+    #    hash = {name: "A", size: 10}
+    #    prepend hash, :product_attributes
+    #    {product_attributes: {name: "A", size: 10}}
+    #
+    def self.prepend(hash, key)
+      { key => hash }
+    end
+
+    #
+    # Retrieve all errors related to row/col from and Array of error messages
+    #
+    def self.errors(errors_array, row, col = nil)
+      errors_array.select do |error|
+        error[:row] == row && (col.nil? || error[:col] == col)
+      end
+    end
+  end
+end

data/lib/dreader/version.rb

@@ -1,3 +1,3 @@
 module Dreader
-  VERSION = "0.4.2"
+  VERSION = "1.0.0"
 end