dreader 1.0.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c2efcf5261515d6957bc11526a5e63956bf09c3fd35cf0fb73805e541fa59ef
-  data.tar.gz: 4d6a80ea2112fbe2cbb024819801ca3853dd6046047069770361a4c32d84bcbe
+  metadata.gz: 524e55af5bb94cae3f407a1602069549783e935798a638361f3c98e922ffc54d
+  data.tar.gz: 2599e048324ccd233e3fa4a0261e134ced0a3347d27af1e978e635639b6284a8
 SHA512:
-  metadata.gz: 7060427113fcf0d8bd33d6db69d42554cd8cf00cbc4b1e4701192c7f846bbfd4c1e4cc904d50277c3b3b6896178ba743dbfb1944da79aaa020b1599cd6f3f856
-  data.tar.gz: 9875ecb098b1085033de36da91917186ba5687028694b0073da71a2ebe4185c594fe31aaf536c2cb5ac7b96bdd7a1f3031fd943b05c9baa4959d0d15e2a5a2f8
+  metadata.gz: e8a78531d96ef35f9272a38daa5327620026dd66c98529710901d4c5994b9e5369308612cc79d1e0998d46dbb9dd6d26c448ddc4265dd536f1e61a4fc50fb885
+  data.tar.gz: de71caed5d3df79d0d456b080a72a0edc035ef4e46906aac3f94295b07d57b956554a9d9e07b4b6195c5c09df0badfa9202122d7b6e0568cf89569b6a2277d28

data/{CHANGELOG.ORG → CHANGELOG.org}

@@ -1,5 +1,57 @@
 #+TITLE: Changelog
 
+* Version 1.1.1 - <2023-10-16 Mon>
+** Adds option :extension
+
+   - Adds options =extension= to the class options and to the =open_spreadsheet=
+     function, to be able to determine the type of a file with no extension
+
+* Version 1.1.0
+** Fixes an issue with visibility of variables
+
+   Version 1.1.0 makes Engine a module and requires to use extend
+
+   This allows to isolate declarations in different variables.
+
+   
+** Renames process to mappings
+
+** Renames the variables in a more consistent way
+
+   #+begin_example ruby
+    attr_accessor :declared_options
+    # the specification of the columns to process
+    attr_accessor :declared_columns
+    # some example lines
+    attr_accessor :declared_examples
+    # the specification of the virtual columns
+    attr_accessor :declared_virtual_columns
+    # the mapping rules
+    attr_accessor :declared_mapping
+   #+end_example
+
+** Declares =data= as a synonym of =table=
+** Adds options to do everything in one pass
+
+   By passing the options
+
+   - =virtual=
+   - =mapping=
+
+   to =read= you can read and process the data in one step.
+
+   See the README for more details.
+   
+** Revises the logging messages
+** Refactor some code to make it more readable
+** Refactors the restructure function to make it more flexible
+
+   Now refactor takes as input symbols and hashes and reshapes
+   according to the specification.
+
+   See the README for an example.
+   
+
 * Version 1.0.0
 ** Changes the DSL to allow declaration in a class
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dreader (0.6.3)
+    dreader (1.2.0)
       fast_excel
       roo
 
@@ -13,11 +13,11 @@ GEM
       reline (>= 0.3.1)
     fast_excel (0.4.1)
       ffi (> 1.9, < 2)
-    ffi (1.15.5)
+    ffi (1.16.2)
     io-console (0.6.0)
     irb (1.7.2)
       reline (>= 0.3.6)
-    nokogiri (1.15.3-x86_64-linux)
+    nokogiri (1.15.4-x86_64-linux)
       racc (~> 1.4)
     racc (1.7.1)
     rake (10.5.0)

data/README.org

@@ -63,7 +63,9 @@ Print name and age of people from the following data:
 #+BEGIN_EXAMPLE ruby
   require 'dreader'
 
-  class Reader < Dreader::Engine
+  class Reader
+    extend Dreader::Engine
+
     options do
       # we start reading from row 2
       first_row 2
@@ -104,14 +106,14 @@ Print name and age of people from the following data:
     end
   end
 
-  reader = Reader.new
+  reader = Reader
 
   # read the file
   reader.read filename: "Birthdays.ods"
   # compute the virtual columns
   reader.virtual_columns
   # run the mapping declaration
-  reader.process
+  reader.mappings
 
   #
   # Here we can do further processing on the data
@@ -130,33 +132,36 @@ Print name and age of people from the following data:
 To write an import function with Dreader:
 
 - Declare which is the input file and where we can find data (Sheet
-  and first row)
-- Declare the content of columns and how to check raw data, parse data,
+  and first row) (This can also be specified in each call.)
+- Declare the content of columns and, then, how to check raw data, parse data,
   and check parsed data
 - Add virtual columns, that is, columns computed from other values
   in the row
-- Specify how to process data each line. This is where you do the actual work
+- Specify how to map line. This is where you do the actual work
   (for instance, if you process a file line by line) or put together data for
   processing after the file has been fully read --- see the next step.
 
-Dreader has now collected and shaped the data according to your instructions
-and collected errors in the process.  We are now ready to do the actual
-processing:
+Dreader now knows ho to collect, shape, and tranform (map) data according to
+your instructions.  We are now ready to do the actual work.  This consists of
+the following steps, various of which can be performed together:
 
-- Do the processing
+- Read the file
+- Do the parsing/transformations
+- Compute the virtual columns
+- Do the mappings
 
 Each step is described in more details in the following sections.
 
 *** Declare which is the input file and where we can find data
 
-Require =dreader= and declare a class which inherits from =Dreader::Engine=:
-
+Require =dreader= and declare a class which extends =Dreader::Engine=:
 
 #+BEGIN_EXAMPLE ruby
   require 'dreader'
 
-  class Reader < Dreader::Engine
-  [...]
+  class Reader
+    extend Dreader::Engine
+    [...]
   end
 #+END_EXAMPLE
 
@@ -165,6 +170,7 @@ In the class specify parsing option, using the following syntax:
 #+BEGIN_EXAMPLE ruby
   options do
     filename 'example.ods'
+    extension ".ods"
 
     sheet 'Sheet 1'
 
@@ -180,10 +186,17 @@ In the class specify parsing option, using the following syntax:
 
 where:
 
-- (optional) =filename= is the file to read. If not specified, you will
-  have to supply a filename when loading the file (see =read=, below).
-  The extension determines the file type. *Use =.tsv= for tab-separated
-  files.*
+- (optional) =filename= is the file to read. If not specified, you will have
+  to supply a filename when loading the file (see =read=, below).  *Use
+  =.tsv= for tab-separated files.* 
+- (optional) =extension= overrides or specify the extension of =filename=.
+  Takes as input the extension preceded by a "." (e.g., ".xlsx").  Notice that
+  **value of this option is not appended to filename** (see =read= below).
+  Filename must thus be a valid reference to a file in the file system. This
+  option is useful in one of these two circumstances:
+  1. When =filename= has no extension
+  2. When you want to override the extension of the filename, e.g., to force
+     reading a "file.csv" as a tab separated file
 - (optional) =first_row= is the first line to read (use =2= if your file
   has a header)
 - (optional) =last_row= is the last line to read. If not specified, we
@@ -192,19 +205,20 @@ where:
   contain "garbage" after the records.
 - (optional) =sheet= is the sheet name or number to read from. If not
   specified, the first (default) sheet is used
+- (optional) =debug= specifies that we are debugging
+- (optional) =logger= specifies the logger
+- (optional) =logger_level= specifies the logger level
 
-#+BEGIN_NOTES
 You can override some of the defaults by passing a hash as argument to
 the =read= function. For instance:
 
 #+BEGIN_EXAMPLE ruby
-  i.read filename: another_filepath
+  Reader.read filename: another_filepath
 #+END_EXAMPLE
 
 will read data from =another_filepath=, rather than from the filename
 specified in the options. This might be useful, for instance, if the
 same specification has to be used for different files.
-#+END_NOTES
 
 
 *** Declare the content of columns and how to parse them
@@ -216,12 +230,12 @@ There are two notations:
 
 #+BEGIN_EXAMPLE ruby
   # First notation, colref is put in the block
-  i.column :name do
+  column :name do
     colref 'A'
   end
 
   # Second notation, a hash is passed in the name
-  i.column({ name: 'A' }) do
+  column({ name: 'A' }) do
   end
 #+END_EXAMPLE
 
@@ -242,7 +256,7 @@ The =column= declaration can contain Ruby blocks:
   =process= is valid.  *Check must return true if there are no errors.*
 
 #+begin_example
-  i.column({ name: 'A' }) do
+  column({ name: 'A' }) do
     check_raw do |cell|
       !cell.nil?
     end
@@ -256,7 +270,7 @@ The =column= declaration can contain Ruby blocks:
 #+end_quote
 
 #+begin_example
-  i.column({ name: 'A' }) do
+  column({ name: 'A' }) do
     check_raw :must_be_non_nil do |cell|
       !cell.nil?
     end
@@ -278,7 +292,7 @@ The =column= declaration can contain Ruby blocks:
 #+end_quote
 
 #+begin_example
-  i.column({ name: 'A' }) do
+  column({ name: 'A' }) do
     check_raw do |cell|
       # Here cell is like in the input file
     end
@@ -435,17 +449,28 @@ key.
 *** Process data
 
 If =mapping= does not work for your data processing activities (e.g., you need
-to make elaborations on data which span different rows), you can add your own
-code after the =process= directive.
+to make elaborations on data which span different rows), you can add your perform
+your elaborations on the data transformed by =mappings=.
 
 A typical scenario works as follows:
 
-1. Instantiate the class: ~i = Reader.new~
-
-1. Use =i.read= or =i.load= (synonyms), to read all data.
+1. Reference the class =i = Reader= and use =i.read= or =i.load=
+   (synonyms), to read all data.  
 
 #+BEGIN_EXAMPLE ruby
+  i = Reader
+
+  # read uses the options if defined and takes the same arguments as options
+  # examples:
+  # i.read
+  # i.read filename: "example.ods"
+  # i.read filename: "example.ods", extension: ".ods"
+  # i.read filename: "example", extension: ".ods" 
+  # (the line above opens the file "example" as an Open Document Spreasdheet)
   i.read
+  
+  # alternately
+  Reader.read
 #+END_EXAMPLE
 
 2. Use =errors= to see whether any of the check functions failed:
@@ -465,20 +490,38 @@ A typical scenario works as follows:
 
 (Optionally: check again for errors.)
 
-4. Use the =process= function to execute the =mapping=
-directive on each line read from the file.
+4. Use the =mappings= function to execute the =mapping= directive on each line
+   read from the file.
 
 #+BEGIN_EXAMPLE ruby
-  i.process
+  i.mappings
 #+END_EXAMPLE
 
 (Optionally: check again for errors.)
 
-5. Add your own code to process data. Use the =table= function to access data.
+5. Add your own code to process the data returned after =mappings=, which you
+   can access with =i.table= or =i.data= (synonyms).
 
-Look in the examples directory for further details and a couple of
-working examples.
+Look in the examples directory for further details and a couple of working
+examples.
 
+*** Improving performances
+
+While debugging your specification executing =read=, =virtual_columns=, and
+=mappings= in distinct steps is a good idea.  When you go in production, you
+might want to reduce the number of passes you perform on the data.
+
+You can pass the option =virtual: true= to =read= to compute virtual
+columns while you are reading data.
+
+You can pass the option =mapping: true= to =read= to compute virtual
+columns and perform the mapping while you are reading data.  Notice that:
+
+- =mapping= implies =virtual=, that is, if you pass =mapping: true= the read
+  function will also compute virtual columns
+- =mapping= alters the content of =@table= and **subsequent calls to
+  =virtual_column= and =mapping= will fail.** You have reset by invoking
+  =read= again.
 
 *** Managing Errors
 
@@ -529,22 +572,23 @@ end
 
 You can check for errors in two different ways:
 
-The first is in the =mapping= directive, where can check whether some checks for
-the =row= failed, by:
+The first is in the =mapping= directive, where can check whether some checks
+for the =row= failed, by:
 
 1. checking from the =:error= boolean key associated to each column, that is:
 
    =row[<column_name>][:error]=
 
-2. looking at the value of the =:row_errors= key, which contains all error messages
-   generated for the row:
+2. looking at the value of the =:row_errors= key, which contains all error
+   messages generated for the row:
 
    =row[:row_errors]=
 
-3. After the processing, by using the method =errors=, which lists all the errors.
+3. After the processing, by using the method =errors=, which lists all the
+   errors.
 
-The utility function =Dreader::Util.errors= takes as input the errors generated by
-Dreader and extract those of a specific row and, optionally column:
+The utility function =Dreader::Util.errors= takes as input the errors generated
+by Dreader and extract those of a specific row and, optionally column:
 
 #+begin_example ruby
   # get all the errors at line 2
@@ -644,39 +688,39 @@ Thus, for instance, given the example above returns:
 
 * Simplifying the hash with the data read
 
-The =Dreader::Util= class provides some functions to simplify the
-hashes built by =dreader=.  This is useful to simplify the code you
-write and to genereate hashes you can pass, for instance, to
-ActiveRecord creators.
+The =Dreader::Util= class provides some functions to simplify the hashes built
+by =dreader=.  This is useful to simplify the code you write and to genereate
+hashes you can pass, for instance, to ActiveRecord creators.
 
 ** Simplify removes everything but the values
 
-=Dreader::Util.simplify hash= removes all information but the value
-and making the value accessible directly from the name of the column.
+=Dreader::Util.simplify(hash)= removes all information but the value and makes
+the value accessible directly from the name of the column.
 
 #+BEGIN_EXAMPLE ruby
   i.table[0]
-  { name: { value: "John", row_number: 1, col_number: 1, errors: nil },
-    age:  { value: 30, row_number: 1, col_number: 2, errors: nil } }
+  {
+    name: { value: "John", row_number: 1, col_number: 1, errors: nil },
+    age:  { value: 30, row_number: 1, col_number: 2, errors: nil }
+  }
 
   Dreader::Util.simplify i.table[0]
   { name: "John", age: 30 }
 #+END_EXAMPLE
 
-*As an additional bonus, it removes the keys =row_number= and =row_errors=,
-which are not part of the data read, in the first place.*
-
 ** Slice and Clean select columns
 
-=Dreader::Util.slice hash, keys= and =Dreader::Util.clean hash, keys=,
-where =keys= is an arrays of keys, are respectively used to select or
-remove some keys from the hash returned by Dreader.  (Notice that the
-Ruby Hash class already provides similar methods.)
+=Dreader::Util.slice(hash, keys)= and =Dreader::Util.clean(hash, keys)=, where
+=keys= is an arrays of keys, are respectively used to select or remove some
+keys from the hash returned by Dreader.  (Notice that the Ruby Hash class
+already provides similar methods.)
 
 #+BEGIN_EXAMPLE ruby
   i.table[0]
-  { name: { value: "John", row_number: 1, col_number: 1, errors: nil },
-    age:  { value: 30, row_number: 1, col_number: 2, errors: nil }}
+  {
+    name: { value: "John", row_number: 1, col_number: 1, errors: nil },
+    age:  { value: 30, row_number: 1, col_number: 2, errors: nil }
+  }
 
   Dreader::Util.slice i.table[0], :name
   { name: { value: "John", row_number: 1, col_number: 1, errors: nil}
@@ -685,8 +729,8 @@ Ruby Hash class already provides similar methods.)
   { age:  { value: 30, row_number: 1, col_number: 2, errors: nil }
 #+END_EXAMPLE
 
-The methods =slice= and =clean= are more useful when used in
-conjuction with =simplify=:
+The methods =slice= and =clean= are more useful when used in conjuction with
+=simplify=:
 
 #+BEGIN_EXAMPLE ruby
   hash = Dreader::Util.simplify i.table[0]
@@ -704,21 +748,23 @@ create an =ActiveRecord= object.
 
 ** Better Integration with ActiveRecord
 
-Finally, the =Dreader::Util.restructure= method helps building hashes
-to create [[http://api.rubyonrails.org/classes/ActiveModel/Model.html][ActiveModel]] objects with nested attributes:
+Finally, the =Dreader::Util.restructure= method helps building hashes to create
+[[http://api.rubyonrails.org/classes/ActiveModel/Model.html][ActiveModel]] objects with nested attributes.
+
+**The starting point is a simplified row.**
 
 #+BEGIN_EXAMPLE ruby
-  hash = {name: "John", surname: "Doe", address: "Unknown", city: "NY" }
+  hash = { name: "John", surname: "Doe", address: "Unknown", city: "NY"  }
      
-  Dreader::Util.restructure hash, [:name, :surname], :address_attributes, [:address, :city]
-  {name: "John", surname: "Doe", address_attributes: {address: "Unknonw", city: "NY"}}
+  Dreader::Util.restructure hash, [:name, :surname, :address_attributes, [:address, :city]]
+  { name: "John", surname: "Doe", address_attributes: { address: "Unknown", city: "NY" } }
 #+END_EXAMPLE
 
 
 * Debugging your specification
 
-The =debug= function prints the current configuration, reads some
-records from the input file(s), and shows the records read:
+The =debug= function prints the current configuration, reads some records from
+the input file(s), and shows the records read:
 
 #+BEGIN_EXAMPLE ruby
   i.debug
@@ -735,8 +781,8 @@ read:
   i.debug process: false, check: false
 #+END_EXAMPLE
 
-Notice that =check= implies =process=, since =check= is invoked on the
-output of the =process= directive.`
+Notice that =check= implies =process=, since =check= is invoked on the output
+of the =process= directive.`
 
 If you prefer, in alternative to =debug= you can also use configuration
 variables (but then you need to change the configuration according to the
@@ -751,7 +797,7 @@ environment):
 
 * Changelog
 
-See [[file:CHANGELOG.ORG][CHANGELOG]].
+See [[file:CHANGELOG.org][CHANGELOG]].
 
 * Known Limitations
 
@@ -759,9 +805,6 @@ At the moment:
 
 - it is not possible to specify column references using header names
   (like Roo does).
-- it is not possible to pass options to the file readers. As a
-  consequence tab-separated files must have the =.tsv= extension or
-  they will not be parsed correctly
 - some more testing wouldn't hurt.
 
 * Known Bugs

data/examples/age/age.rb

@@ -1,47 +1,47 @@
-require 'dreader'
+require "dreader"
 
-class Reader < Dreader::Engine
+class Reader
+  extend Dreader::Engine
 
-options do
-  first_row 2
-  debug true
-end
+  options do
+    first_row 2
+    debug true
+  end
 
-column :name do
-  doc "A is the name string"
-  colref 'A'
-end
+  column :name do
+    doc "A is the name string"
+    colref 'A'
+  end
 
-column :birthdate do
-  doc "Birthdate contains a full date (i.e., including the year)"
-  colref 'B'
+  column :birthdate do
+    doc "Birthdate contains a full date (i.e., including the year)"
+    colref 'B'
 
-  process do |c|
-    Date.parse(c)
+    process do |c|
+      Date.parse(c)
+    end
   end
-end
 
-virtual_column :age do
-  process do |row|
-    birthdate = row[:birthdate][:value]
-    birthday = Date.new(Date.today.year, birthdate.month, birthdate.day)
-    today = Date.today
+  virtual_column :age do
+    process do |row|
+      birthdate = row[:birthdate][:value]
+      birthday = Date.new(Date.today.year, birthdate.month, birthdate.day)
+      today = Date.today
 
-    [0, today.year - birthdate.year - (birthday < today ? 1 : 0)].max
+      [0, today.year - birthdate.year - (birthday < today ? 1 : 0)].max
+    end
   end
-end
 
-mapping do |row|
-  r = Dreader::Util.simplify(row)
-  puts "#{r[:name]} is #{r[:age]} years old (born on #{r[:birthdate]})"
-end
+  mapping do |row|
+    r = Dreader::Util.simplify(row)
+    puts "#{r[:name]} is #{r[:age]} years old (born on #{r[:birthdate]})"
+  end
 end
 
-i = Reader.new
-
+i = Reader
 i.read filename: "Birthdays.ods"
 i.virtual_columns
-i.process
+i.mappings
 
 #
 # Here we can do further processing on the data

data/examples/age_with_multiple_checks/age_with_multiple_checks.rb

@@ -1,6 +1,8 @@
 require 'dreader'
 
-class Reader < Dreader::Engine
+class Reader
+  extend  Dreader::Engine
+
   options { first_row 2; debug true }
 
   #
@@ -54,9 +56,9 @@ class Reader < Dreader::Engine
   end
 end
 
-i = Reader.new
+i = Reader
 
 i.read filename: "Birthdays.ods"
 i.virtual_columns
-i.process
+i.mappings
 

data/examples/local_vars/local_vars.rb

@@ -0,0 +1,28 @@
+#
+# This demonstrates that variables are local
+# 
+
+require "dreader"
+
+class OneReader
+  extend Dreader::Engine
+
+  options do
+    first_row 2
+    debug true
+  end
+end
+
+class AnotherReader
+  extend Dreader::Engine
+
+  options do
+    filename "filename"
+  end
+end
+
+r1 = OneReader
+r2 = AnotherReader
+
+puts r1.declared_options
+puts r2.declared_options

data/examples/wikipedia_big_us_cities/big_us_cities.rb

@@ -2,7 +2,9 @@
 
 require 'dreader'
 
-class Processor < Dreader::Engine
+class Processor
+  extend Dreader::Engine
+
   options do
     first_row 2
     filename "cities_by_state.ods"
@@ -25,7 +27,7 @@ class Processor < Dreader::Engine
   end
 end
 
-processor = Processor.new
+processor = Processor
 
 printf "Loading the spreadsheet..."
 processor.load
@@ -43,8 +45,8 @@ else
 end
 puts "done!"
 
-puts "Processing the spreadsheet..."
-processor.process
+puts "Applying mapping rules to the spreadsheet..."
+processor.mappings
 puts "... done"
 
 

data/examples/wikipedia_us_cities/us_cities.rb

@@ -13,7 +13,9 @@ class City
   end
 end
 
-class Importer < Dreader::Engine
+class Importer
+  extend Dreader::Engine
+
   # read from us_cities.tsv, lines from 2 to 10 (included)
   options do
     filename "us_cities.tsv"
@@ -50,7 +52,7 @@ class Importer < Dreader::Engine
 end
 
 cities = []
-importer = Importer.new
+importer = Importer
 
 importer.mapping do |row|
   # remove all additional information stored in each cell
@@ -81,7 +83,7 @@ importer.debug process: false, check: false
 # load and process
 importer.load
 cities = []
-importer.process
+importer.mappings
 
 # output everything to see whether it works
 puts "First ten cities in the US (source Wikipedia)"

data/examples/wikipedia_us_cities/us_cities_bulk_declare.rb

@@ -13,7 +13,9 @@ class City
   end
 end
 
-class Importer < Dreader::Engine
+class Importer
+  extend Dreader::Engine
+
   # read from us_cities.tsv, lines from 2 to 10 (included)
   options do
     filename "us_cities.tsv"
@@ -42,7 +44,7 @@ class Importer < Dreader::Engine
 end
 
 cities = []
-importer = Importer.new
+importer = Importer
 
 importer.mapping do |row|
   # remove all additional information stored in each cell
@@ -73,7 +75,7 @@ importer.debug process: false, check: false
 # load and process
 importer.load
 cities = []
-importer.process
+importer.mappings
 
 # output everything to see whether it works
 puts "First ten cities in the US (source Wikipedia)"

data/lib/dreader/engine.rb

@@ -10,47 +10,29 @@ module Dreader
   #
   # This is where the real stuff begins
   #
-  class Engine
-    # TODO: make the writer into private methods (need to be accessed only
-    # in the initializer) and demote to attr_reader
-    
+  module Engine
     # the options we passed
-    attr_accessor :options
+    attr_accessor :declared_options
     # the specification of the columns to process
-    attr_accessor :colspec
+    attr_accessor :declared_columns
     # some example lines
-    attr_accessor :examples
+    attr_accessor :declared_examples
     # the specification of the virtual columns
-    attr_accessor :virtualcols
+    attr_accessor :declared_virtual_columns
     # the mapping rules
-    attr_accessor :mapping
+    attr_accessor :declared_mapping
     
     # the data we read
     attr_reader :table
 
-    # variables declared in the class which need to be propagated in
-    # the instance
-    INSTANTIATE = %i[options colspec examples virtualcols]
-
-    def initialize
-      @logger = Logger.new($stdout)
-      @logger.level = Logger::WARN
-
-      # populate the instance with the variables defined in the class
-      @options = defined?(@@options) ? @@options : {}
-      @colspec = defined?(@@colspec) ? @@colspec : []
-      @examples = defined?(@@examples) ? @@examples : []
-      @virtualcols = defined?(@@virtualcols) ? @@virtualcols : []
-    end
-
     # define a DSL for options
     # any string is processed as an option and it ends up in the
     # @options hash
-    def self.options(&block)
+    def options(&block)
       options = Options.new
       options.instance_eval(&block)
 
-      @@options = options.to_hash
+      @declared_options = options.to_hash
     end
 
     # define a DSL for column specification
@@ -58,18 +40,18 @@ module Dreader
     # - `block` contains two declarations, `process` and `check`, which are
     #   used, respectively, to make a cell into the desired data and to check
     #   whether the desired data is ok
-    def self.column(name, &block)
+    def column(name, &block)
       column = Column.new
       column.instance_eval(&block)
 
-      @@colspec ||= []
+      @declared_columns ||= []
 
       if name.instance_of?(Hash)
-        @@colspec << column.to_hash.merge(
+        @declared_columns << column.to_hash.merge(
           { name: name.keys.first, colref: name.values.first }
         )
       else
-        @@colspec << column.to_hash.merge({ name: name })
+        @declared_columns << column.to_hash.merge({ name: name })
       end
     end
 
@@ -106,20 +88,20 @@ module Dreader
     #     cell.strip
     #   end
     # end
-    def self.columns(hash, &block)
+    def columns(hash, &block)
       hash.each_key do |key|
         column = Column.new
         column.colref hash[key]
         column.instance_eval(&block) if block
 
-        @@colspec ||= []
-        @@colspec << column.to_hash.merge({ name: key })
+        @declared_columns ||= []
+        @declared_columns << column.to_hash.merge({ name: key })
       end
     end
 
-    def self.example(hash)
-      @@examples ||= []
-      @@examples << hash
+    def example(hash)
+      @declared_examples ||= []
+      @declared_examples << hash
     end
 
     # virtual columns define derived attributes
@@ -128,12 +110,12 @@ module Dreader
     #
     # virtual colum declarations are executed in the order in which
     # they are defined
-    def self.virtual_column(name, &block)
+    def virtual_column(name, &block)
       column = Column.new
       column.instance_eval &block
 
-      @@virtualcols ||= []
-      @@virtualcols << column.to_hash.merge({ name: name })
+      @declared_virtual_columns ||= []
+      @declared_virtual_columns << column.to_hash.merge({ name: name })
     end
 
     # define what we do with each line we read
@@ -141,8 +123,8 @@ module Dreader
     #   `row` is a hash in which each spreadsheet cell is accessible under
     #   the column names. Each cell has the following values:
     #   :value, :error, :row_number, :col_number
-    def self.mapping(&block)
-      @@mapping = block
+    def mapping(&block)
+      @declared_mapping = block
     end
 
     # read a file and store it internally
@@ -155,18 +137,20 @@ module Dreader
     # @return the data read from filename, in the form of an array of
     #         hashes
     def read(args = {})
+      # args override values in options (if defined)
+      # the initializer guarantees @options is at least {}
+      options = (@declared_options || {}).merge(args)
+
+      @logger = options[:logger] || Logger.new($stdout)
+      @logger.level = options[:logger_level] || Logger::WARN
+      @debug = options[:debug] == true
+
       if !args.instance_of?(Hash)
         @logger.error "#{__callee__}: this function takes a Hash as input"
         raise Exception
       end
 
-      options = (@options || {}).merge(args)
-
-      @logger = options[:logger] if options[:logger]
-      @logger.level = options[:logger_level] if options[:logger_level]
-      @debug = options[:debug] == true
-
-      spreadsheet = Dreader::Engine.open_spreadsheet (options[:filename])
+      spreadsheet = open_spreadsheet(options)
       sheet = spreadsheet.sheet(options[:sheet] || 0)
       first_row = options[:first_row] || 1
       last_row = options[:last_row] || sheet.last_row
@@ -176,64 +160,74 @@ module Dreader
         @logger.level = Logger::DEBUG
 
         # override the number of lines read
-        options[:n] ||= 10 unless options[:n]
-        last_row = options[:first_row] + options[:n].to_i - 1
-      
+        n_lines = options[:n] ? options[:n].to_i : 10
+        last_row = first_row + n_lines - 1
+        
         # apply some defaults for debugging, if not defined in the options
         [:check_raw, :process, :check].map do |key|
           options[key] = true unless options.key?(key)
         end
       end
 
-      { current: @options, debug: options}.each do |k, v|
-        @logger.debug "#{k.capitalize} configuration:"
+      { current: @declared_options, debug: options }.each do |k, v|
+        @logger.debug "[dreader] #{k.capitalize} configuration:"
         v.each do |key, value|
           @logger.debug "  #{key}: #{value}"
         end
       end
       
-      @table = []
       @errors = []
 
-      (first_row..last_row).each do |row_number|
+      @table = (first_row..last_row).map do |row_number|
         r = { row_number: row_number, row_errors: [] }
         
-        @colspec.each_with_index do |colspec, index|
-          colref = colspec[:colref]
-          cell = sheet.cell(row_number, colref)
-          colname = colspec[:name]
-
-          r[colname] = {
-            row: row_number,
-            col: colspec[:colref],
-            value: cell,
-            error: false
-          }
-          @logger.debug "#{__callee__} [ROW DECL] Processing row: #{row_number}, col: #{colref}"
-
-          # check raw data
-          check_data(colspec[:checks_raw], r, colname)
-
-          # process data
-          coord = coord(row_number, colspec[:colref], cell)
-          begin
-            processed = colspec[:process] ? colspec[:process].call(cell) : cell
-            @logger.debug "process on #{colname} at #{coord} yields: '#{processed}' (#{processed.class})"
-            r[colname][:value] = processed
-          rescue => e
-            @logger.error "#{__callee__}: process on :#{colname} raised an exception at #{coord}"
-            raise e
-          end
+        # this has side-effects on r
+        columns_on(r, row_number, sheet)
+
+        # this has side-effects on r
+        virtual_columns_on(r) if options[:virtual] || options[:mapping]
+
+        options[:mapping] ? mappings_on(r) : r
+      end
+    end
+
+    # TODO: PASS A ROW (and not row_number and sheet)
+    def columns_on(r, row_number, sheet)
+      @declared_columns.each_with_index do |colspec, index|
+        colname = colspec[:name]
+        colref = colspec[:colref]
+        cell = sheet.cell(row_number, colref)
+
+        r[colname] = {
+          row: row_number,
+          col: colspec[:colref],
+          value: cell,
+          error: false
+        }
+
+        # Repeated below
+        # @logger.debug "[dreader] Processing #{coord(row_number, colref)}"
 
-          # check data after process - notice that now r contains the value
-          # processed by process
-          check_data(colspec[:checks], r, colname)
+        # check raw data
+        check_data(colspec[:checks_raw], r, colname)
+
+        # process data
+        coord = coord(row_number, colspec[:colref], cell)
+        begin
+          processed = colspec[:process] ? colspec[:process].call(cell) : cell
+          @logger.debug "[dreader] #{colname} process #{coord} yields '#{processed}' (#{processed.class})"
+          r[colname][:value] = processed
+        rescue => e
+          @logger.error "[dreader] #{colname} process #{coord} raises an exception"
+          raise e
         end
 
-        @table << r
+        # check data after process - notice that now r contains the value
+        # processed by process
+        check_data(colspec[:checks], r, colname)
       end
 
-      @table
+      r
     end
 
     alias load read
@@ -254,51 +248,65 @@ module Dreader
     # You need to invoke read first
     def get_row(row_number)
       if row_number > @table.size
-        @logger.error "#{__callee__}: 'row_number' is out of range (did you invoke read?)"
+        @logger.error "[dreader] 'row_number' is out of range (did you invoke read?)"
         exit
       elsif row_number <= 0
-        @logger.error "#{__callee__}: 'row_number' is zero or negative (first row is 1)."
+        @logger.error "[dreader] 'row_number' is zero or negative (first row is 1)."
       else
         @table[row_number - 1]
       end
     end
 
-    # return an array of hashes with all the errors we have encounterd
+    # return an array of hashes with all the errors we have encountered
     # an empty array is a good news
     attr_reader :errors
 
-    def virtual_columns(hash = {})
+    def virtual_columns
       # execute the virtual column specification
-      @table.each do |r|
-        @virtualcols.each do |virtualcol|
-          colname = virtualcol[:name]
-          r[colname] = { virtual: true }
-          
-          check_data(virtualcol[:checks_raw], r, colname, full_row: true)
-
-          begin
-            # add the cell to the table
-            if virtualcol[:process]
-              r[colname][:value] = virtualcol[:process].call(r)
-            end
-          rescue => e
-            row = r[:row_number]
-            @logger.error "#{__callee__}: process for virtual column :#{colname} raised an exception at row #{row}"
-            raise e
-          end
+      @table.each { |row| virtual_columns_on(row) }
+    end
+
+    # Compute virtual columns for, with side effect on row
+    def virtual_columns_on(row)
+      @declared_virtual_columns.each do |virtualcol|
+        colname = virtualcol[:name]
+        row[colname] = { virtual: true }
+        
+        check_data(virtualcol[:checks_raw], row, colname, full_row: true)
 
-          # check data after process -- we also have the processed value of the virtual column
-          check_data(virtualcol[:checks], r, colname, full_row: true)
+        begin
+          # add the cell to the table
+          if virtualcol[:process]
+            row[colname][:value] = virtualcol[:process].call(row)
+          end
+        rescue => e
+          r = row[:row_number]
+          @logger.error "[dreader] #{colname} process raises an exception at row #{r}"
+          raise e
         end
+
+        # check data after process -- we also have the processed value of
+        # the virtual column
+        check_data(virtualcol[:checks], row, colname, full_row: true)
       end
     end
 
-    # apply the mapping code to the array
-    # it makes sense to invoke it only once
+    # apply the mapping code to the array it makes sense to invoke it only
+    # once.
     #
-    # the mapping is applied only if it defined
-    def process
-      @table.each { |row| @mapping&.call(row) }
+    # the mapping is applied only if it defined and it uses map, so that
+    # it can be used functionally
+    def mappings
+      @table.map { |row| mappings_on(row) }
+    end
+
+    def mappings_on(row)
+      @declared_mapping&.call(row)
+    end
+
+    # an alias
+    def data
+      @table
     end
 
     def to_s
@@ -314,14 +322,14 @@ module Dreader
     end
 
     def compare_headers(hash = {})
-      options = @options.merge(hash)
+      options = @declared_options.merge(hash)
 
-      spreadsheet = Dreader::Engine.open_spreadsheet(options[:filename])
+      spreadsheet = open_spreadsheet(options)
       sheet = spreadsheet.sheet(options[:sheet] || 0)
       header_row_number = options[:first_row] - 1 || 1
 
       output_hash = {}
-      @colspec.map do |colspec|
+      @declared_columns.map do |colspec|
         cell = sheet.cell(row_number, colspec[:colref])
         human_readable = colspec[:name].to_s.split("_").map(&:capitalize).join(" ")
 
@@ -341,7 +349,7 @@ module Dreader
     # second row includes the documentation string, to document values in
     # the columns
     def template(hash = {})
-      options = @options.merge(hash)
+      options = @declared_options.merge(hash)
       filename = options[:template_filename]
 
       workbook = FastExcel.open(filename, constant_memory: true)
@@ -357,22 +365,22 @@ module Dreader
       #
 
       # here we write the first row
-      @colspec.each do |colspec|
+      @declared_columns.each do |colspec|
         human_readable = colspec[:name].to_s.split("_").map(&:capitalize).join(" ")
         colref = colref_to_i(colspec[:colref])
         worksheet.write_string(first_row, colref, human_readable, bold)
       end
 
       # here we create a note with the legenda
-      @colspec.each do |colspec|
+      @declared_columns.each do |colspec|
         colref = colref_to_i(colspec[:colref])
         worksheet.write_string(first_row + 1, colref, colspec[:doc], nil)
       end
 
       # here we write some example records
-      @examples.each_with_index do |example_hash, index|
+      @declared_examples.each_with_index do |example_hash, index|
         example_hash.each do |colname, value|
-          colspec = @colspec.select { |x| x[:name] == colname }.first
+          colspec = @declared_columns.select { |x| x[:name] == colname }.first
           if colspec
             colref = colref_to_i(colspec[:colref])
             worksheet.write_string(index + 3, colref, value, nil)
@@ -387,16 +395,31 @@ module Dreader
 
     private
 
-    def self.open_spreadsheet(filename)
-      ext = File.extname(filename)
+    # list of keys we support in options. We remove them when reading
+    # the CSV file
+    OPTION_KEYS = %i[
+      filename sheet first_row last_row logger logger_level
+    ]
+
+    def open_spreadsheet(options)
+      filename = options[:filename]
+      ext = options[:extension] || File.extname(filename)
 
       case ext
-      when ".csv" then Roo::CSV.new(filename)
-      when ".tsv" then Roo::CSV.new(filename, csv_options: { col_sep: "\t" })
-      when ".ods" then Roo::OpenOffice.new(filename)
-      when ".xls" then Roo::Excel.new(filename)
-      when ".xlsx" then Roo::Excelx.new(filename)
-      else raise "Unknown extension: #{ext}"
+      when ".csv"
+        csv_options = @declared_options.except(*OPTION_KEYS)
+        Roo::CSV.new(filename, csv_options:)
+      when ".tsv"
+        csv_options = @declared_options.except(*OPTION_KEYS).merge({ col_sep: "\t" })
+        Roo::CSV.new(filename, csv_options:)
+      when ".ods"
+        Roo::OpenOffice.new(filename)
+      when ".xls"
+        Roo::Excel.new(filename)
+      when ".xlsx"
+        Roo::Excelx.new(filename)
+      else
+        raise "Unknown extension: #{ext}"
       end
     end
 
@@ -445,7 +468,7 @@ module Dreader
 
         begin
           pass = check_function.call(value)
-          @logger.debug "check for #{colname}/#{error_message} at #{coord} yields: '#{pass}'"
+          @logger.debug "[dreader] check #{colname}/#{error_message} at #{coord} yields: '#{pass}'"
 
           if pass != true
             hash[colname][:error] = true
@@ -460,14 +483,14 @@ module Dreader
             hash[:row_errors] << error
           end
         rescue => e
-          @logger.error "#{__callee__} for #{colname}/#{error_message} raised an exception at #{coord}"
+          @logger.error "[dreader] check #{colname}/#{error_message} raises an exception at #{coord}"
           raise e
         end
       end
     end
 
-    def coord(row, col, cell)
-      "row: #{row}, col: #{col}, value: #{cell}"
+    def coord(row, col, value = nil)
+      ["#{row}#{col}", (value ? "(#{value})" : nil)].join(" ")
     end
   end
 end

data/lib/dreader/util.rb

@@ -19,20 +19,35 @@ module Dreader
       end.to_h
     end
 
-    # given a hash returned by Engine, keep the "kept" keys in the top
-    # of the hierarchy and move the "moved_key" below the
-    # "subordinate_key"
+    # Given a "simplified" hash restructure it according to the second
+    # argument.
+    #
+    # Use it for generating hashes with nested attributes, which
+    # follows Rails conventions.
+    #
+    # @params:
+    # - hash the hash to restructure
+    # - args splat arguments which specify how to (re)structure the
+    #   values in Hash.  Each element is either a symbol or a Hash
     #
     # Example
     #
-    # hash = { name: "A", surname: "B", address: "via XX Settembre", city: "Genoa" }
-    # restructure hash, [:name, :surname], :address_attributes, [:address, :city]
-    # {name: "A", surname: "B", address_attributes: {address: "via XX Settembre", city: "Genoa"}}
+    # hash = { name: "A", surname: "B", address: "via Piave", city: "Genoa" }
+    #
+    # restructure(hash, :name, :surname)
+    # { name: "A", surname: "B" }
     #
-    def self.restructure(hash, kept, subordinate_key, moved_keys)
-      head = hash.slice kept
-      subordinate = prepend subordinate_key, hash.slice(moved_keys)
-      head.merge subordinate
+    # restructure(hash, :name, address_attributes: [:address, :city])
+    # {name: "A", address_attributes: { address: "via Piave", city: "Genoa" }
+    #
+    def self.restructure(hash, *new_structure)
+      new_structure.to_h do |value|
+        if value.instance_of?(Hash)
+          [value.keys.first, self.restructure(hash, *value.values.first)]
+        else
+          [value, hash[value]]
+        end
+      end
     end
 
     # an alias for Hash.slice

data/lib/dreader/version.rb

@@ -1,3 +1,3 @@
 module Dreader
-  VERSION = "1.0.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dreader
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-28 00:00:00.000000000 Z
+date: 2023-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: roo
@@ -97,7 +97,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- CHANGELOG.ORG
+- CHANGELOG.org
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -110,6 +110,7 @@ files:
 - examples/age/age.rb
 - examples/age_with_multiple_checks/Birthdays.ods
 - examples/age_with_multiple_checks/age_with_multiple_checks.rb
+- examples/local_vars/local_vars.rb
 - examples/template/template_generation.rb
 - examples/wikipedia_big_us_cities/big_us_cities.rb
 - examples/wikipedia_big_us_cities/cities_by_state.ods
@@ -141,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Process and import data from cvs and spreadsheets