sycsvpro 0.0.9 → 0.1.0

data/lib/sycsvpro.rb

@@ -12,3 +12,4 @@ require 'sycsvpro/script_creator.rb'
 require 'sycsvpro/script_list.rb'
 require 'sycsvpro/inserter.rb'
 require 'sycsvpro/sorter.rb'
+require 'sycsvpro/aggregator.rb'

data/lib/sycsvpro/aggregator.rb

@@ -0,0 +1,112 @@
+require_relative 'row_filter'
+require_relative 'column_filter'
+require_relative 'dsl'
+
+# Operating csv files
+module Sycsvpro
+
+  # An Aggregator counts specified row values and adds a sum to the end of the row
+  class Aggregator
+
+    include Dsl
+
+    # infile contains the data that is operated on
+    attr_reader :infile
+    # outfile is the file where the result is written to
+    attr_reader :outfile
+    # file doesn't contain a header
+    attr_reader :headerless
+    # filter that is used for rows
+    attr_reader :row_filter
+    # filter that is used for columns
+    attr_reader :col_filter
+    # values that are aggregated
+    attr_reader :key_values
+    # header of the out file
+    attr_reader :heading
+    # Title of the sum row
+    attr_reader :sum_row_title
+    # row where to add the sums of the columns
+    attr_reader :sum_row
+    # Title of the sum column
+    attr_reader :sum_col_title
+    # column where to add the sum of the row sum
+    attr_reader :sum_col
+    # sums of the column values
+    attr_reader :sums
+    
+    # Creates a new aggregator. Takes as attributes infile, outfile, key, rows, cols, date-format 
+    # and indicator whether to add a sum row
+    def initialize(options={})
+      @infile     = options[:infile]
+      @outfile    = options[:outfile]
+      @headerless = options[:headerless] || false
+      @row_filter = RowFilter.new(options[:rows])
+      @col_filter = ColumnFilter.new(options[:cols], df: options[:df])
+      @key_values = Hash.new(0)
+      @heading    = []
+      @sums       = Hash.new(0)
+      init_sum_scheme(options[:sum])
+    end
+
+    # Executes the aggregator
+    def execute
+      process_aggregation
+      write_result
+    end
+
+    # Process the aggregation of the key values
+    def process_aggregation
+      File.new(infile).each_with_index do |line, index|
+        result = col_filter.process(row_filter.process(line.chomp, row: index))
+        unless result.nil? or result.empty?
+          if heading.empty? and not headerless
+            heading << result.split(';')
+            next
+          else
+            @sum_col = [result.split(';').size, sum_col].max 
+          end
+          key_values[result]  += 1
+          sums[sum_col_title] += 1
+        end
+      end
+      heading.flatten!
+      heading[sum_col] = sum_col_title
+    end
+
+    # Writes the aggration results
+    def write_result
+      sum_line = [sum_row_title]
+      (heading.size - 2).times { sum_line << "" }
+      sum_line << sums[sum_col_title]
+      row = 0;
+      File.open(outfile, 'w') do |out|
+        out.puts sum_line.join(';') if row == sum_row ; row += 1
+        out.puts heading.join(';')
+        key_values.each do |k, v|
+          out.puts sum_line.join(';') if row == sum_row ; row += 1
+          out.puts [k, v].join(';')
+        end
+      end
+    end
+
+    private
+
+      # Initializes the sum row title an positions as well as the sum column title and position
+      def init_sum_scheme(sum_scheme)
+        row_scheme, col_scheme = sum_scheme.split(',') unless sum_scheme.nil?
+
+        unless row_scheme.nil?
+          @sum_row_title, @sum_row = row_scheme.split(':') unless row_scheme.empty?
+        end
+        
+        @sum_row.nil? ? @sum_row = 0 : @sum_row = @sum_row.to_i
+        @sum_row_title = 'Total' if @sum_row_title.nil? 
+
+        col_scheme.nil? ? @sum_col_title = 'Total' : @sum_col_title = col_scheme
+        @sum_col = 0
+      end
+
+  end
+
+end

data/lib/sycsvpro/calculator.rb

@@ -24,16 +24,20 @@ module Sycsvpro
     attr_reader :header
     # filter that is used for columns
     attr_reader :columns
+    # if true add a sum row at the bottom of the out file
+    attr_reader :add_sum_row
 
     # Creates a new Calculator. Options expects :infile, :outfile, :rows and :columns. Optionally
     # a header can be provided. The header can be supplemented with additional column names that
     # are generated due to a arithmetic operation that creates new columns
     def initialize(options={})
-      @infile     = options[:infile]
-      @outfile    = options[:outfile]
-      @row_filter = RowFilter.new(options[:rows])
-      @header     = Header.new(options[:header])
-      @formulae   = {}
+      @infile      = options[:infile]
+      @outfile     = options[:outfile]
+      @row_filter  = RowFilter.new(options[:rows])
+      @header      = Header.new(options[:header])
+      @sum_row     = []
+      @add_sum_row = options[:sum] || false
+      @formulae    = {}
       create_calculator(options[:cols])
     end
 
@@ -64,7 +68,19 @@ module Sycsvpro
             @columns[col.to_i] = eval(formula)
           end
           out.puts @columns.join(';')
+
+          @columns.each_with_index do |column, index|
+            if @sum_row[index]
+              @sum_row[index] += to_number column
+            else
+              @sum_row[index] =  to_number column
+            end
+          end if add_sum_row
+
         end
+
+        out.puts @sum_row.join(';') if add_sum_row
+
       end
     end
 

data/lib/sycsvpro/counter.rb

@@ -5,7 +5,7 @@ require_relative 'dsl'
 # Operating csv files
 module Sycsvpro
 
-  # Creates a new counter that counts values and uses the values as column names and uses the count
+  # Counter counts values and uses the values as column names and uses the count
   # as the column value
   class Counter
 
@@ -15,8 +15,10 @@ module Sycsvpro
     attr_reader :infile
     # outfile is the file where the result is written to
     attr_reader :outfile
-    # values are assigned to the key column
-    attr_reader :key_column
+    # values are assigned to the key columns
+    attr_reader :key_columns
+    # key columns headers
+    attr_reader :key_titles
     # filter that is used for rows
     attr_reader :row_filter
     # filter that is used for columns
@@ -26,9 +28,11 @@ module Sycsvpro
     # header of the out file
     attr_reader :heading
     # Title of the sum row
-    attr_reader :sum_title
-    # row where to add the sums of the columns of the sum columns
+    attr_reader :sum_row_title
+    # row where to add the sums of the columns
     attr_reader :sum_row
+    # Title of the sum column
+    attr_reader :sum_col_title
     # sums of the column values
     attr_reader :sums
     
@@ -37,59 +41,101 @@ module Sycsvpro
     def initialize(options={})
       @infile     = options[:infile]
       @outfile    = options[:outfile]
-      @key_column = options[:key].to_i
+      init_key_columns(options[:key])
       @row_filter = RowFilter.new(options[:rows])
       @col_filter = ColumnFilter.new(options[:cols], df: options[:df])
       @key_values = {}
       @heading    = []
-      @sum_title, @sum_row = options[:sum].split(':') unless options[:sum].nil?
-      @sum_row    = @sum_row.to_i unless @sum_row.nil?
+      init_sum_scheme(options[:sum])
       @sums       = Hash.new(0)
     end
 
     # Executes the counter
     def execute
-      process_file
+      process_count
       write_result
     end
 
     # Processes the counting on the in file
-    def process_file
+    def process_count
       File.new(infile).each_with_index do |line, index|
         result = col_filter.process(row_filter.process(line.chomp, row: index))
         unless result.nil? or result.empty?
-          key = unstring(line).split(';')[key_column]
-          key_value = key_values[key] || key_values[key] = { name: key, elements: Hash.new(0) }
+          key = unstring(line).split(';').values_at(*key_columns)
+          key_value = key_values[key] || key_values[key] = { name: key, 
+                                                             elements: Hash.new(0), 
+                                                             sum: 0 }
           result.chomp.split(';').each do |column|
             heading << column if heading.index(column).nil?
             key_value[:elements][column] += 1
+            key_value[:sum] += 1
             sums[column] += 1
           end
         end
       end
+      unless sum_col_title.nil?
+        heading << sum_col_title
+        sums[sum_col_title] = sums.values.inject(:+)
+      end
     end
 
-    # Writes the results
+   # Writes the count results
     def write_result
-      sum_line = [sum_title]
+      sum_line = [sum_row_title] + [''] * (key_titles.size - 1)
       heading.sort.each do |h|
         sum_line << sums[h]
       end
       row = 0;
       File.open(outfile, 'w') do |out|
         out.puts sum_line.join(';') if row == sum_row ; row += 1
-        out.puts (["key"] + heading.sort).join(';')
+        out.puts (key_titles + heading.sort).join(';')
         key_values.each do |k,v|
           out.puts sum_line.join(';') if row == sum_row ; row += 1
           line = [k]
           heading.sort.each do |h|
-            line << v[:elements][h]
+            line << v[:elements][h] unless h == sum_col_title
           end
+          line << v[:sum] unless sum_col_title.nil?
           out.puts line.join(';')
         end
       end
     end
 
+    private
+
+      # Initializes the sum row title an positions as well as the cum column title
+      def init_sum_scheme(sum_scheme)
+
+        return if sum_scheme.nil?
+
+        re = /(\w+):(\d+)|(\w+)/
+
+        sum_scheme.scan(re).each do |part|
+          if part.compact.size == 2
+            @sum_row_title = part[0]
+            @sum_row       = part[1].to_i
+          else
+            @sum_col_title = part[2]
+          end
+        end
+
+      end
+
+      # Initialize the key columns and headers
+      def init_key_columns(key_scheme)
+
+        @key_titles  = []
+        @key_columns = []
+
+        keys = key_scheme.scan(/(\d+):(\w+)/)
+
+        keys.each do |key|
+          @key_titles  << key[1]
+          @key_columns << key[0].to_i
+        end
+
+      end
+
   end
 
 end

data/lib/sycsvpro/filter.rb

@@ -30,14 +30,16 @@ module Sycsvpro
 
     # Creates the filters based on the given patterns
     def method_missing(id, *args, &block)
-      return equal($1, args, block)              if id =~ /^(\d+)$/
-      return equal_type($1, $2, args, block)     if id =~ /^(s|n|d):(\d+)$/
-      return range($1, $2, args, block)          if id =~ /^(\d+)-(\d+)$/
-      return range_type($1, $2, $3, args, block) if id =~ /^(s|n|d):(\d+)-(\d+)$/
-      return regex($1, args, block)              if id =~ /^\/(.*)\/$/
-      return col_regex($1, $2, args, block)      if id =~ /^(\d+):\/(.*)\/$/
-      return date($1, $2, $3, args, block)       if id =~ /^(\d+):(<|=|>)(\d+.\d+.\d+)/
-      return date_range($1, $2, $3, args, block) if id =~ /^(\d+):(\d+.\d+.\d+.)-(\d+.\d+.\d+)$/
+      return equal($1, args, block)                if id =~ /^(\d+)$/
+      return equal_type($1, $2, args, block)       if id =~ /^(s|n|d):(\d+)$/
+      return range($1, $2, args, block)            if id =~ /^(\d+)-(\d+)$/
+      return range_type($1, $2, $3, args, block)   if id =~ /^(s|n|d):(\d+)-(\d+)$/
+      return regex($1, args, block)                if id =~ /^\/(.*)\/$/
+      return col_regex($1, $2, args, block)        if id =~ /^(\d+):\/(.*)\/$/
+      return date($1, $2, $3, args, block)         if id =~ /^(\d+):(<|=|>)(\d+.\d+.\d+)/
+      return date_range($1, $2, $3, args, block)   if id =~ /^(\d+):(\d+.\d+.\d+.)-(\d+.\d+.\d+)$/
+      return number($1, $2, $3, args, block)       if id =~ /^(\d+):(<|=|>)(\d+)/
+      return number_range($1, $2, $3, args, block) if id =~ /^(\d):(\d+)-(\d+)/
       super
     end
 
@@ -54,7 +56,8 @@ module Sycsvpro
         match = false
         begin
           match = eval(parameters[:operation].gsub('[value]', value))
-        rescue
+        rescue Exception => e
+
         end
         yield column, match
       end
@@ -124,6 +127,19 @@ module Sycsvpro
         pivot["#{start_date}-#{end_date}"] = { col: col, operation: operation }
       end
 
+      # Adds a number filter
+      def number(col, comparator, number, args, block)
+        comparator = '==' if comparator == '='
+        operation = "[value] #{comparator} #{number}"
+        pivot["#{comparator}#{number}"] = { col: col, operation: operation }
+      end
+
+      # Adds a number range filter
+      def number_range(col, start_number, end_number, arg, block)
+        operation = " #{start_number} <= [value] && [value] <= #{end_number}"
+        pivot["#{start_number}-#{end_number}"] = { col: col, operation: operation }
+      end
+
   end
 
 end

data/lib/sycsvpro/version.rb

@@ -1,5 +1,5 @@
 # Operating csv files
 module Sycsvpro
   # Version number of sycsvpro
-  VERSION = '0.0.9'
+  VERSION = '0.1.0'
 end

data/spec/sycsvpro/aggregator_spec.rb

@@ -0,0 +1,55 @@
+require 'sycsvpro/aggregator'
+
+module Sycsvpro
+
+  describe Aggregator do
+
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/in.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+
+    it "should aggregate single column values" do
+      aggregator = Aggregator.new(infile: @in_file, outfile: @out_file, rows: "1-10",
+                                  cols: "0", sum: "Total:1,Machines", headerless: true)
+
+      aggregator.execute
+
+      result = [ ";Machines",
+                 "Total;7",
+                 "Fink;2",
+                 "Haas;1",
+                 "Gent;1",
+                 "Rank;1",
+                 "Klig;1",
+                 "fink;1" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+    end
+
+    it "should aggregate multiple column values" do
+      aggregator = Aggregator.new(infile: @in_file, outfile: @out_file, rows: "0-10",
+                            cols: "0,1", sum: "Total:1,Machines", headerless: false)
+  
+      aggregator.execute
+
+      result = [ "customer;contract-number;Machines",
+                 "Total;;7",
+                 "Fink;1234;2",
+                 "Haas;3322;1",
+                 "Gent;4323;1",
+                 "Rank;3232;1",
+                 "Klig;4432;1",
+                 "fink;1234;1" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+    
+  end
+
+end

data/spec/sycsvpro/calculator_spec.rb

@@ -40,6 +40,28 @@ module Sycsvpro
         expect(line.chomp).to eq result[index]
       end
     end
+
+    it "should sum specified rows" do
+      header = "*,drives,motors"
+      rows = "1-8"
+      cols = "5:c3+c4,6:c3*2"
+      sums = "1,3-5"
+      calculator = Calculator.new(infile: @in_file, outfile: @out_file, 
+                                  header: header, rows: rows, cols: cols, sum: true)
+      calculator.execute
+
+      result = ["customer;machines;controls;contracts;visits;drives;motors",
+                "Fink;2;2;1;1;2;2",
+                "Haas;3;3;1;1.0;2.0;2",
+                "Gent;4;4;1;1;2;2",
+                "Rank;5;5;1;1;2;2",
+                "0;14;14;4;4.0;8;8"]
+
+      File.new(@out_file, 'r').each_with_index do |line, index|
+        expect(line.chomp).to eq result[index]
+      end
+    end
+
   end
 
 end