sycsvpro 0.1.3 → 0.1.4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sycsvpro (0.1.3)
+    sycsvpro (0.1.4)
       gli (= 2.9.0)
       timeleap (~> 0.0.1)
 

data/README.md

@@ -16,6 +16,7 @@ Processing of csv files. *sycsvpro* offers following functions
 * create or edit a Ruby script
 * list scripts available optionally with methods (since version 0.0.7)
 * execute a Ruby script file that operates a csv file
+* create a table from a source file with dynamically create columns (since version 0.1.4)
 
 To get help type
 
@@ -308,6 +309,26 @@ command: `sycsvpro execute script.rb method infile param1 param2`
 * Add `clean_up` to *Dsl* that takes files to be deleted after the script has
 run: `clean_up(%w{file1 file2})`
 
+Version 0.1.4
+-------------
+* A new Table class is available with following features
+    * Create dynamic headline columns based on source table data
+    * Associate values to multi keys
+    * Create values based on arithmetic operations of source table data
+  Example
+    `sycsvpro -f in.csv -o out.csv table -h "c4,c5,c0=~/\\.(\\d{4})/" \
+                                         -k "c4,c5" \
+                                         -c "c0=~/\\.(\\d{4})/:+n1"`
+  h:: the header is created from the source table header of column 4 and 5.
+      Another header column is created dynamicall based on the year part of a
+      date in column 0
+  k:: the key is based on source table of column 4 and 5
+  c:: the column operation is in the form HeaderName:Operation. In this case the
+      HeaderName is dynamically determined based on column 0 and added the value
+      of column 1 to this column that is associated to the key
+
+  c4, n4, d4 are string, number and date values respectively
+
 Installation
 ============
 [![Gem Version](https://badge.fury.io/rb/sycsvpro.png)](http://badge.fury.io/rb/sycsvpro)

data/bin/sycsvpro

@@ -303,6 +303,51 @@ command :aggregate do |c|
 
 end
 
+desc 'Creates a table from a source file'
+
+command :table do |c|
+
+  c.desc 'Rows to consider'
+  c.arg_name '1,2,10-30,45-EOF,REGEXP'
+  c.flag [:r, :row], :must_match => row_regex
+
+  c.desc 'Header can be defined by Words (Year), references to source header (c1) and dynamically created header values (c1+c2,c0=~/\\.(\\d{4})/)'
+  c.arg_name "COL_A,c6,c2+c4,c0=~/\\.(\\d{4})/"
+  c.flag [:h, :header]
+
+  c.desc 'Key to that the other columns are associated to. A key can be created dynamically'
+  c.arg_name "c0=~/\\.(\\d{4})/,c6"
+  c.flag [:k, :key]
+
+  c.desc 'Columns to be associated to the key. Columns are identified by the column name. The operation to create the column value is separated by a colon (:) from the column name'
+  c.arg_name "c0=~/\\.(\\d{4})/:+n1,Value:+n2"
+  c.flag [:c, :col]
+
+  c.desc 'Format of date values'
+  c.arg_name '%d.%m.%Y|%m/%d/%Y|...'
+  c.flag [:df]
+
+  c.desc 'Format of number values'
+  c.arg_name 'DE|EN'
+  c.default_value 'EN'
+  c.flag [:nf]
+
+  c.action do |global_options,options,args|
+    print "Table..."
+    table = Sycsvpro::Table.new(infile:  global_options[:f], 
+                                outfile: global_options[:o], 
+                                df:      options[:df],
+                                nf:      options[:nf],
+                                rows:    options[:r], 
+                                header:  options[:h],
+                                key:     options[:k],
+                                cols:    options[:c]) 
+    table.execute
+    puts "done"
+  end
+
+end
+
 desc 'Sort rows based on column values'
 command :sort do |c|
   c.desc 'Rows to consider'

data/lib/sycsvpro/header.rb

@@ -21,12 +21,56 @@ module Sycsvpro
       end
     end
 
+    def method_missing(id, *args, &block)
+      return @row_cols[$1.to_i] if id =~ /^c(\d+)$/
+      super
+    end
+
     # Returns the header
-    def process(line)
+    def process(line, values = true)
       return "" if @header_cols.empty?
-      @header_cols[0] = unstring(line).split(';')
-      @header_cols.flatten.join(';')
+      header_patterns = {}
+      @row_cols = unstring(line).split(';')
+      if @header_cols[0] == '*'
+        @header_cols[0] = @row_cols
+      else
+        @header_cols.each_with_index do |h,i|
+          if h =~ /^c\d+(?:[=~]{,2}).*$/ 
+            if col = eval(h)
+              last_eval = $1
+              unless @header_cols.index(last_eval) || @header_cols.index(col)
+                if values
+                  @header_cols[i] = (h =~ /^c\d+=~/) ? last_eval : col
+                  header_patterns[i+1] = h if h =~ /^c\d+[=~+-]{1,2}/
+                else
+                  @header_cols[i] = col if h =~ /^c\d+$/
+                end
+              end
+            end
+          else
+            @header_cols[i] = h
+          end
+        end
+      end
+      header_patterns.each { |i,h| @header_cols.insert(i,h) }
+      to_s
+    end
+
+    # Returns @header_cols without pattern
+    def clear_header_cols
+      @header_cols.flatten.select { |col| col !~ /^c\d+[=~+]{1,2}/ }
     end
+
+    # Returns the index of the column
+    def column_of(value)
+      clear_header_cols.index(value)
+    end
+
+    # Returns the header
+    def to_s
+      clear_header_cols.join(';')
+    end
+
   end
 
 end

data/lib/sycsvpro/table.rb

@@ -0,0 +1,174 @@
+require_relative 'row_filter'
+require_relative 'header'
+require_relative 'dsl'
+require 'date'
+
+module Sycsvpro
+
+  class Table
+
+    include Dsl
+
+    # infile contains the data that is operated on
+    attr_reader :infile
+    # outfile is the file where the result is written to
+    attr_reader :outfile
+    # filter that is used for rows
+    attr_reader :row_filter
+    # date format for date operations
+    attr_reader :date_format
+    # header of the outfile
+    attr_reader :header
+    # rows of the created table
+    attr_reader :rows
+
+    # Creates a new Table. Options expects :infile, :outfile, :rows and 
+    # :columns. Optionally a header can be provided. The header can be 
+    # supplemented with additional column names that are generated due to a 
+    # arithmetic operation that creates new columns
+    # :call-seq:
+    #   Sycsvpro::Table.new(infile:  "in.csv",
+    #                       outfile: "out.csv",
+    #                       df:      "%d.%m.%Y",
+    #                       rows:    "1,2,BEGINn3>20END",
+    #                       header:  "Year,c6,c1",
+    #                       key:     "c0=~/\\.(\\d{4})/,c6",
+    #                       cols:    "Value:+n1,c2+c3:+n1",
+    #                       nf:      "DE").execute
+    def initialize(options = {})
+      @infile      = options[:infile]
+      @outfile     = options[:outfile]
+      @date_format = options[:df] || "%Y-%m-%d"
+      @row_filter  = RowFilter.new(options[:rows], df: options[:df])
+      @header      = Header.new(options[:header])
+      @keys        = options[:key].split(',')
+      @cols        = options[:cols].split(',')
+      @number_format = options[:nf] || 'EN'
+      @rows        = {}
+    end
+
+    # Retrieves the values from a row as the result of a arithmetic operation
+    # with #eval
+    def method_missing(id, *args, &block)
+      return @columns[$1.to_i]            if id =~ /c(\d+)/
+      return to_number(@columns[$1.to_i]) if id =~ /n(\d+)/
+      return to_date(@columns[$1.to_i])   if id =~ /d(\d+)/
+      super
+    end
+
+    # Executes the table and writes the result to the _outfile_
+    def execute
+      create_table_data
+      write_to_file
+    end
+
+    # Create the table
+    def create_table_data
+      processed_header = false
+
+      File.open(infile).each_with_index do |line, index|
+        line = line.chomp
+
+        next if line.empty?
+        
+        line = unstring(line).chomp
+
+        header.process line, processed_header
+
+        unless processed_header
+          processed_header = true
+          next
+        end
+
+        next if row_filter.process(line, row: index).nil?
+        
+        @columns = line.split(';')
+
+        create_row(create_key, line)
+      end
+
+    end
+
+    # Write table to _outfile_
+    def write_to_file
+      File.open(outfile, 'w') do |out|
+        out.puts header.to_s
+        rows.each do |key, row|
+          line = [] << row[:key]
+          header.clear_header_cols.each_with_index do |col, index|
+            next if index < row[:key].size
+            line << row[:cols][col]
+          end
+          out.puts line.flatten.join(';')
+        end
+      end
+    end
+
+    # Creates a key from the provided key pattern
+    def create_key
+      key = []
+      @keys.each { |k| key << evaluate(k, "") }
+      key
+    end
+
+    # Creates a table row based on the column pattern
+    # Examples of column patterns
+    # * Value:+n1             Adds content of column 1 to Value column
+    # * Value:+n1,c2+c3:+n1   Creates a dynamic column and adds column 1 value
+    # * c0=~/\\.(\\d{4})/:+n1 Creates dynamic column from regex and adds
+    #                         column 1 value
+    def create_row(key, line)
+      row = rows[key] || rows[key] = { key: key, cols: Hash.new(0) }  
+      @cols.each do |col|
+        column, formula = col.split(':')
+        column = evaluate(column) if column =~ /^c\d+[=~+]/
+        row[:cols][column] = eval("#{row[:cols][column]}#{formula}")
+      end
+    end
+
+    private
+
+      # Casts a string to an integer or float depending whether the value has a 
+      # decimal point
+      def to_number(value)
+        value = convert_to_en(value)
+        return value.to_i unless value =~ /\./
+        return value.to_f if     value =~ /\./ 
+      end
+
+      # Casts a string to a date
+      def to_date(value)
+        if value.nil? or value.strip.empty?
+          nil
+        else
+          Date.strptime(value, date_format)
+        end
+      end
+
+      # Localize the number to EN
+      def convert_to_en(value)
+        if @number_format == 'DE'
+          value.gsub('.', '_').gsub(',', '.')
+        else
+          value
+        end
+      end
+
+      # Evaluate a formula
+      # Example invokation
+      #     evaluate("n1+n2", 0)
+      #     evaluate("c1+c2", "failed")
+      #     evaluate("c0=~/\\.(\\d{4})/", "0")
+      def evaluate(formula, fail_result = 0)
+        if value = eval(formula)
+          last_match = $1
+          (formula =~ /^c\d+=~/) ? last_match : value
+        else
+          fail_result
+        end   
+      end
+
+  end
+
+end
+

data/lib/sycsvpro/version.rb

@@ -1,5 +1,5 @@
 # Operating csv files
 module Sycsvpro
   # Version number of sycsvpro
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/lib/sycsvpro.rb

@@ -13,3 +13,4 @@ require 'sycsvpro/script_list.rb'
 require 'sycsvpro/inserter.rb'
 require 'sycsvpro/sorter.rb'
 require 'sycsvpro/aggregator.rb'
+require 'sycsvpro/table.rb'

data/spec/sycsvpro/header_spec.rb

@@ -0,0 +1,79 @@
+require 'sycsvpro/header'
+
+module Sycsvpro
+
+  describe Header do
+
+    it "should create a header from '*,A,B'" do
+      header = Header.new("*,A,B")
+
+      header.process("a;b;c").should eq 'a;b;c;A;B'
+    end
+
+    it "should create a header form 'A,c6,c1'" do
+      header = Header.new("A,c6,c1")
+
+      header.process("a0;a1;a2;a3;a4;a5;a6").should eq "A;a6;a1"
+      header.process("x0;x1;x2;x3;x4;x5;x6").should eq "A;a6;a1"
+    end
+
+    it "should not create columns on arithmetic operation" do
+      header = Header.new("A,c1,c2+c3")
+
+      header.process("h0;h1;h2;h3;h4;h5", false).should eq "A;h1"
+      header.process("a0;a1;a2;a3;a4;a5").should eq "A;h1;a2a3"
+      header.process("b0;b1;b2;b3;b4;b5").should eq "A;h1;a2a3;b2b3"
+      header.process("c0;a1;a2;c3;a4;a5").should eq "A;h1;a2a3;b2b3;a2c3"
+
+    end
+
+    it "should create a header from 'A,c1,c2+c3'" do
+      header = Header.new("A,c1,c2+c3")
+
+      header.process("a0;a1;a2;a3;a4;a5").should eq "A;a1;a2a3"
+      header.process("b0;b1;b2;b3;b4;b5").should eq "A;a1;a2a3;b2b3"
+      header.process("c0;a1;a2;c3;a4;a5").should eq "A;a1;a2a3;b2b3;a2c3"
+    end
+
+    it "should create a header form 'A,c1,c2+'-'+c3'" do
+      header = Header.new("A,c1,c2+'-'+c3")
+
+      header.process("a0;a1;a2;a3;a4;a5").should eq "A;a1;a2-a3"
+      header.process("b0;b1;b2;b3;b4;b5").should eq "A;a1;a2-a3;b2-b3"
+      header.process("c0;a1;a2;a3;c4;c5").should eq "A;a1;a2-a3;b2-b3"
+    end
+
+    it "should create a header from 'c4,A,c0=~/\.(\d{4})/,c1,B'" do
+      header = Header.new("c4,A,c0=~/\\.(\\d{4})/,c1,B")
+
+      header.process("a0;a1;a2;a3;a4;a5").should eq "a4;A;a1;B"
+      header.process("1.1.2012;b1;b2;b3;b4;b5").should eq "a4;A;2012;a1;B"
+      header.process("3.4.2013;c1;c2;c3;c4;c5").should eq "a4;A;2012;2013;a1;B"
+      header.process("5.5.2012;d1;d2;d3;d4;d5").should eq "a4;A;2012;2013;a1;B"
+    end
+
+    it "should return the header" do
+      header = Header.new("c4,A,c0=~/\\.(\\d{4})/,c1,B")
+
+      header.process("a0;a1;a2;a3;a4;a5").should eq "a4;A;a1;B"
+      header.process("1.1.2012;b1;b2;b3;b4;b5").should eq "a4;A;2012;a1;B"
+      header.process("3.4.2013;c1;c2;c3;c4;c5").should eq "a4;A;2012;2013;a1;B"
+      header.process("5.5.2012;d1;d2;d3;d4;d5").should eq "a4;A;2012;2013;a1;B"
+
+      header.to_s.should eq "a4;A;2012;2013;a1;B"
+    end
+
+    it "should return the index of the coloum" do
+      header = Header.new("c4,A,c0=~/\\.(\\d{4})/,c1,B")
+
+      header.process("a0;a1;a2;a3;a4;a5").should eq "a4;A;a1;B"
+      header.column_of("a1").should eq 2
+      header.process("1.1.2012;b1;b2;b3;b4;b5").should eq "a4;A;2012;a1;B"
+      header.column_of("a1").should eq 3
+      header.process("3.4.2013;c1;c2;c3;c4;c5").should eq "a4;A;2012;2013;a1;B"
+      header.column_of("B").should eq 5
+     end
+
+  end
+
+end

data/spec/sycsvpro/table_spec.rb

@@ -0,0 +1,65 @@
+require 'sycsvpro/table'
+
+module Sycsvpro
+
+  describe Table do
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/table.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+
+    it "should create headings from String and column values" do
+      Sycsvpro::Table.new(infile: @in_file,
+                          outfile: @out_file,
+                          header:  "Year,c6,c1",
+                          key:     "c0=~/\\.(\\d{4})/,c6",
+                          cols:    "Value:+n1").execute
+
+      result = [ "Year;Country;Value", 
+                 "2013;AT;53.7", 
+                 "2014;DE;21.0",
+                 "2014;AT;20.5" ] 
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+    end
+
+    it "should create headings from operation" do
+      Sycsvpro::Table.new(infile: @in_file,
+                          outfile: @out_file,
+                          header:  "Year,c6,c1,c2+c3",
+                          key:     "c0=~/\\.(\\d{4})/,c6",
+                          cols:    "Value:+n1,c2+c3:+n1").execute
+
+      result = [ "Year;Country;Value;A1;B2;B4", 
+                 "2013;AT;53.7;20.5;0;33.2", 
+                 "2014;DE;21.0;0;21.0;0",
+                 "2014;AT;20.5;20.5;0;0" ] 
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+
+    it "should create key from operation" do
+      Sycsvpro::Table.new(infile: @in_file,
+                          outfile: @out_file,
+                          header:  "c4,c5,c0=~/\\.(\\d{4})/",
+                          key:     "c4,c5",
+                          cols:    "c0=~/\\.(\\d{4})/:+n1").execute
+
+      result = [ "Customer Name;Customer-ID;2013;2014", 
+                 "Hank;133;20.5;20.5",
+                 "Hans;234;0;21.0",
+                 "Jack;432;33.2;0" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sycsvpro
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-21 00:00:00.000000000 Z
+date: 2014-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -148,6 +148,7 @@ files:
 - lib/sycsvpro/script_creator.rb
 - lib/sycsvpro/script_list.rb
 - lib/sycsvpro/sorter.rb
+- lib/sycsvpro/table.rb
 - lib/sycsvpro/version.rb
 - spec/sycsvpro/aggregator_spec.rb
 - spec/sycsvpro/allocator_spec.rb
@@ -161,12 +162,14 @@ files:
 - spec/sycsvpro/files/profile.rb
 - spec/sycsvpro/files/script.rb
 - spec/sycsvpro/files/unsert.ins
+- spec/sycsvpro/header_spec.rb
 - spec/sycsvpro/inserter_spec.rb
 - spec/sycsvpro/mapper_spec.rb
 - spec/sycsvpro/profiler_spec.rb
 - spec/sycsvpro/row_filter_spec.rb
 - spec/sycsvpro/script_list_spec.rb
 - spec/sycsvpro/sorter_spec.rb
+- spec/sycsvpro/table_spec.rb
 - sycsvpro.gemspec
 - sycsvpro.rdoc
 homepage: https://github.com/sugaryourcoffee/syc-svpro