dreader 0.5.0 → 1.1.0

data/dreader.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.name          = "dreader"
   spec.version       = Dreader::VERSION
   spec.authors       = ["Adolfo Villafiorita"]
-  spec.email         = ["adolfo.villafiorita@ict4g.net"]
+  spec.email         = ["adolfo@shair.tech"]
 
   spec.summary       = %q{Process and import data from cvs and spreadsheets}
   spec.description   = %q{Use this gem to specify the structure of some tabular data
@@ -19,7 +19,7 @@ Rails application, but the gem can used in any Ruby application.
 
 The gem should be relatively easy to use, despite its name. (Dread
 stands for *d*ata *r*eader)}
-  spec.homepage      = "https://ict4g.net/gitea/adolfo/dreader"
+  spec.homepage      = "https://redmine.shair.tech/projects/dreader"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
@@ -29,8 +29,10 @@ stands for *d*ata *r*eader)}
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "roo"
+  spec.add_runtime_dependency "fast_excel"
+
+  spec.add_development_dependency "debug", ">= 1.0.0"
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
-
-  spec.add_runtime_dependency "roo"
 end

data/examples/age/age.rb

@@ -1,39 +1,55 @@
-require 'dreader'
+require "dreader"
 
-i = Dreader::Engine.new
+class Reader
+  extend Dreader::Engine
 
-i.options do
-  first_row 2
-end
+  options do
+    first_row 2
+    debug true
+  end
 
-i.column :name do
-  colref 'A'
-end
+  column :name do
+    doc "A is the name string"
+    colref 'A'
+  end
 
-i.column :birthdate do
-  colref 'B'
+  column :birthdate do
+    doc "Birthdate contains a full date (i.e., including the year)"
+    colref 'B'
 
-  process do |c|
-    Date.parse(c)
+    process do |c|
+      Date.parse(c)
+    end
   end
-end
 
-i.virtual_column :age do
-  process do |row|
-    birthdate = row[:birthdate][:value]
-    birthday = Date.new(Date.today.year, birthdate.month, birthdate.day)
-    today = Date.today
+  virtual_column :age do
+    process do |row|
+      birthdate = row[:birthdate][:value]
+      birthday = Date.new(Date.today.year, birthdate.month, birthdate.day)
+      today = Date.today
 
-    [0, today.year - birthdate.year - (birthday < today ? 1 : 0)].max
+      [0, today.year - birthdate.year - (birthday < today ? 1 : 0)].max
+    end
   end
-end
 
-i.mapping do |row|
-  r = Dreader::Util.simplify(row)
-  puts "#{r[:name]} is #{r[:age]} years old (born on #{r[:birthdate]})"
+  mapping do |row|
+    r = Dreader::Util.simplify(row)
+    puts "#{r[:name]} is #{r[:age]} years old (born on #{r[:birthdate]})"
+  end
 end
 
+i = Reader
 i.read filename: "Birthdays.ods"
 i.virtual_columns
-i.process
-
+i.mappings
+
+#
+# Here we can do further processing on the data
+#
+File.open("ages.txt", "w") do |file|
+  i.table.each do |row|
+    unless row[:row_errors].any?
+      file.puts "#{row[:name][:value]} #{row[:age][:value]}"
+    end
+  end
+end

data/examples/age_with_multiple_checks/age_with_multiple_checks.rb

@@ -0,0 +1,64 @@
+require 'dreader'
+
+class Reader
+  extend  Dreader::Engine
+
+  options { first_row 2; debug true }
+
+  #
+  # Here we intentionally fail on good data
+  #
+  column :name do
+    doc "A is the name string"
+
+    colref 'A'
+
+    check_raw :name_too_short do |cell|
+      cell&.size < 7
+    end
+
+    check_raw :name_too_long do |cell|
+      cell&.size > 10
+    end
+  end
+
+  column :birthdate do
+    doc "Birthdate contains a full date (i.e., including the year)"
+    colref 'B'
+
+    process do |c|
+      Date.parse(c)
+    end
+  end
+
+  virtual_column :age do
+    process do |row|
+      birthdate = row[:birthdate][:value]
+      birthday = Date.new(Date.today.year, birthdate.month, birthdate.day)
+      today = Date.today
+
+      [0, today.year - birthdate.year - (birthday < today ? 1 : 0)].max
+    end
+  end
+
+  mapping do |row|
+    errors_for_row = row[:row_errors]
+
+    if errors_for_row.any?
+      puts "ATTENTION: Record at row #{row[:row_number]} has the following errors:"
+      errors_for_row.each do |error|
+        puts "  " + error[:message].to_s
+      end
+    else
+      r = Dreader::Util.simplify(row)
+      puts "#{r[:name]} is #{r[:age]} years old (born on #{r[:birthdate]})"
+    end
+  end
+end
+
+i = Reader
+
+i.read filename: "Birthdays.ods"
+i.virtual_columns
+i.mappings
+

data/examples/local_vars/local_vars.rb

@@ -0,0 +1,28 @@
+#
+# This demonstrates that variables are local
+# 
+
+require "dreader"
+
+class OneReader
+  extend Dreader::Engine
+
+  options do
+    first_row 2
+    debug true
+  end
+end
+
+class AnotherReader
+  extend Dreader::Engine
+
+  options do
+    filename "filename"
+  end
+end
+
+r1 = OneReader
+r2 = AnotherReader
+
+puts r1.declared_options
+puts r2.declared_options

data/examples/template/template_generation.rb

@@ -0,0 +1,37 @@
+require 'dreader'
+
+i = Dreader::Engine.new
+
+i.options do
+  first_row 2
+end
+
+i.column :name do
+  doc "A is the name string"
+  colref 'A'
+end
+
+i.column :birthdate do
+  doc "Birthdate contains a full date (i.e., including the year)"
+  colref 'B'
+
+  process do |c|
+    Date.parse(c)
+  end
+end
+
+i.column :place_of_birth do
+  doc "This is a string, a city (the interesting part is that it is in column E"
+  colref 'E'
+end
+
+i.example({ name: "Galileo Galilei", birthdate: "15/02/1564", place_of_birth: "Pisa" })
+i.example({ name: "Cyrus McCormick", birthdate: "15/02/1809" })
+i.example({ name: "Charles Lewis Tiffany", birthdate: "15/02/1812" })
+i.example({ name: "Ernest Shackleton", birthdate: "15/02/1874" })
+i.example({ name: "Yelena Bonner", birthdate: "15/02/1923" })
+i.example({ name: "Matt Groening", birthdate: "15/02/1954", place_of_birth: "USA" })
+
+# only xlsx at the moment
+i.generate_template template_filename: "birthdays.xlsx"
+

data/examples/wikipedia_big_us_cities/big_us_cities.rb

@@ -2,29 +2,33 @@
 
 require 'dreader'
 
-processor = Dreader::Engine.new
+class Processor
+  extend Dreader::Engine
 
-processor.options do
-  first_row 2
-  filename "cities_by_state.ods"
-end
+  options do
+    first_row 2
+    filename "cities_by_state.ods"
+  end
 
-processor.column :state do |col|
-  col.colref 'A'
-end
+  column :state do |col|
+    col.colref 'A'
+  end
 
-processor.column :cities do |col|
-  col.colref 'B'
-  col.check do |data|
-    data.class == Integer
+  column :cities do |col|
+    col.colref 'B'
+    col.check do |data|
+      data.class == Integer
+    end
   end
-end
 
-processor.mapping do |row|
-  hash = Dreader::Util.simplify row
-  puts "  states: #{hash[:state]} number of cities: #{hash[:cities]}"
+  mapping do |row|
+    hash = Dreader::Util.simplify row
+    puts "  states: #{hash[:state]} number of cities: #{hash[:cities]}"
+  end
 end
 
+processor = Processor
+
 printf "Loading the spreadsheet..."
 processor.load
 puts "done!"
@@ -41,8 +45,8 @@ else
 end
 puts "done!"
 
-puts "Processing the spreadsheet..."
-processor.process
+puts "Applying mapping rules to the spreadsheet..."
+processor.mappings
 puts "... done"
-    
-  
+
+

data/examples/wikipedia_us_cities/us_cities.rb

@@ -13,43 +13,46 @@ class City
   end
 end
 
-importer = Dreader::Engine.new
-
-# read from us_cities.tsv, lines from 2 to 10 (included)
-importer.options do
-  filename "us_cities.tsv"
-  first_row 2
-  last_row  10
-end
+class Importer
+  extend Dreader::Engine
+
+  # read from us_cities.tsv, lines from 2 to 10 (included)
+  options do
+    filename "us_cities.tsv"
+    first_row 2
+    last_row  10
+  end
 
-# these are the columns for which we only need to specify column and name
-[[:city, 2], [:state, 3], [:latlon, 11]].each do |val|
-  # val[0] -> :city
-  # val[1] -> 2
-  importer.column val[0] do |col|
-    col.colref val[1]
-    col.process do |val|
-      val.strip
+  # these are the columns for which we only need to specify column and name
+  [[:city, 2], [:state, 3], [:latlon, 11]].each do |val|
+    # val[0] -> :city
+    # val[1] -> 2
+    column val[0] do |col|
+      col.colref val[1]
+      col.process do |val|
+        val.strip
+      end
     end
   end
-end
 
-# the population column requires more work
-importer.column :population do |col|
-  col.colref 4
+  # the population column requires more work
+  column :population do |col|
+    col.colref 4
 
-  # make "3,000" into 3000 (int)
-  col.process do |value|
-    value.gsub(",", "").to_i
-  end
+    # make "3,000" into 3000 (int)
+    col.process do |value|
+      value.gsub(",", "").to_i
+    end
 
-  col.check do |value|
-    value > 0
-  end
+    col.check do |value|
+      value > 0
+    end
 
+  end
 end
 
 cities = []
+importer = Importer
 
 importer.mapping do |row|
   # remove all additional information stored in each cell
@@ -80,7 +83,7 @@ importer.debug process: false, check: false
 # load and process
 importer.load
 cities = []
-importer.process
+importer.mappings
 
 # output everything to see whether it works
 puts "First ten cities in the US (source Wikipedia)"

data/examples/wikipedia_us_cities/us_cities_bulk_declare.rb

@@ -13,36 +13,38 @@ class City
   end
 end
 
-importer = Dreader::Engine.new
-
-# read from us_cities.tsv, lines from 2 to 10 (included)
-importer.options do
-  filename "us_cities.tsv"
-  first_row 2
-  last_row  10
-end
+class Importer
+  extend Dreader::Engine
+
+  # read from us_cities.tsv, lines from 2 to 10 (included)
+  options do
+    filename "us_cities.tsv"
+    first_row 2
+    last_row  10
+  end
 
-# these are the columns for which we only need to specify column and name
-importer.bulk_declare ({city: 2, state: 3, latlon: 11}) do
-  process { |val| val.strip }
-end
+  # these are the columns for which we only need to specify column and name
+  columns ({city: 2, state: 3, latlon: 11}) do
+    process { |val| val.strip }
+  end
 
-# the population column requires more work
-importer.column :population do |col|
-  col.colref 4
+  # the population column requires more work
+  column :population do |col|
+    col.colref 4
 
-  # make "3,000" into 3000 (int)
-  col.process do |value|
-    value.gsub(",", "").to_i
-  end
+    # make "3,000" into 3000 (int)
+    col.process do |value|
+      value.gsub(",", "").to_i
+    end
 
-  col.check do |value|
-    value > 0
+    col.check do |value|
+      value > 0
+    end
   end
-
 end
 
 cities = []
+importer = Importer
 
 importer.mapping do |row|
   # remove all additional information stored in each cell
@@ -73,7 +75,7 @@ importer.debug process: false, check: false
 # load and process
 importer.load
 cities = []
-importer.process
+importer.mappings
 
 # output everything to see whether it works
 puts "First ten cities in the US (source Wikipedia)"

data/lib/dreader/column.rb

@@ -0,0 +1,39 @@
+module Dreader
+  # service class to implement the column DSL language
+  class Column
+    def initialize
+      @checks_raw = {}
+      @checks = {}
+    end
+
+    def doc(doc)
+      @doc = doc
+    end
+
+    def colref(colref)
+      @colref = colref
+    end
+
+    def check_raw(message = :unnamed_check, &block)
+      @checks_raw[message] = block
+    end
+
+    def process(&block)
+      @process = block
+    end
+
+    def check(message = :unnamed_check, &block)
+      @checks[message] = block
+    end
+
+    def to_hash
+      {
+        checks_raw: @checks_raw,
+        process: @process,
+        checks: @checks,
+        colref: @colref,
+        doc: @doc
+      }
+    end
+  end
+end