sycsvpro 0.1.2 → 0.1.3

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    sycsvpro (0.1.2)
+    sycsvpro (0.1.3)
       gli (= 2.9.0)
+      timeleap (~> 0.0.1)
 
 GEM
   remote: https://rubygems.org/
@@ -39,6 +40,7 @@ GEM
     rspec-expectations (2.14.4)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.4)
+    timeleap (0.0.1)
 
 PLATFORMS
   ruby

data/README.md

@@ -280,21 +280,34 @@ Release notes
 
 Version 0.1.2
 -------------
-* Now it is possible to have , in the filter as non separating values. You can
-now define filter like 1-2,4,/[56789]{2,}/,10
+* Now it is possible to have comma ',' in the filter as non separating values. 
+  You can now define a filter like 1-2,4,/[56789]{2,}/,10
 * Filtering rows on boolean expression based on values contained in columns.
   The boolean expression has to be enclosed between BEGIN and END
+
   Example:
-    -r BEGINs0=='Ruby'&&n1<1||d2==Date.new(2014,6,17)END
-    s0 - string in column 0
-    n1 - number in column 1
-    d2 - date   in column 2
-* ``list`` shows the directory of the script file and has the flag *all* to
-show all scripts, that is _insert files_ and _Ruby files_
+    + ``-r BEGINs0=='Ruby'&&n1<1||d2==Date.new(2014,6,17)END``
+    + s0 - string in column 0
+    + n1 - number in column 1
+    + d2 - date   in column 2
+* ``list`` shows the directory of the script file (`dir: true`) and has the 
+flag *all* to show all scripts, that is _insert files_ and _Ruby files_
 * When counting columns with *count* the column headers are sorted
 alphabetically. No it is possible to set ``sort: false`` to keep the column
 headers in the sequence they are specified
 
+Version 0.1.3
+-------------
+* In counter `sort: false` didn't work with column filters like `cols: "1,2"`.
+Now all filters work
+* Sorter now accepts a start row where to start sorting. Rows before the start
+row are added on top of the sorted file
+* `sycsvpro -f infile analyze` now lists the columns with sample data
+* Add `params` method to *Dsl* that retrieves the params provided in the execute
+command: `sycsvpro execute script.rb method infile param1 param2`
+* Add `clean_up` to *Dsl* that takes files to be deleted after the script has
+run: `clean_up(%w{file1 file2})`
+
 Installation
 ============
 [![Gem Version](https://badge.fury.io/rb/sycsvpro.png)](http://badge.fury.io/rb/sycsvpro)

data/README.rdoc

@@ -4,7 +4,8 @@ Author:: Pierre Sugar (mailto:pierre@sugaryourcoffee.de)
 Copyright:: Copyright (c) 2014 by Pierre Sugar
 License:: Distributed uder the MIT license, see LICENSE in the source distro
 
-The application provides an interface for analyzing, cleaning and operating on csv files
+The application provides an interface for analyzing, cleaning and operating on 
+csv files
 
 {<img src="https://badge.fury.io/rb/sycsvpro.png" alt="Gem Version" />}[http://badge.fury.io/rb/sycsvpro]
 
@@ -41,5 +42,5 @@ Test files are in
 
     spec/sycsvpro/files
 
-:include:sycsvpro.rdoc
+:include: sycsvpro.rdoc
 

data/bin/sycsvpro

@@ -53,11 +53,13 @@ command :analyze do |c|
     puts "Analysis of #{global_options[:f]}"
     puts "#{result.col_count} columns: #{result.cols}"
     puts "#{result.row_count} rows"
+    puts; puts "Row sample data:"
+    puts "#{result.sample_row}"
+    puts; puts "Column index: Column name | Column sample value"
+    sample_cols = result.sample_row.split(';')
     result.cols.each_with_index do |col, index|
-      puts "#{index}: #{col}"
+      puts "#{index}: #{col} | #{sample_cols[index]}"
     end
-    puts "Row sample data:"
-    puts "#{result.sample_row}"
   end
 end
 
@@ -316,6 +318,10 @@ command :sort do |c|
   c.default_value '%Y-%m-%d'
   c.flag [:df]
 
+  c.desc 'First row to sort, row numbers are 0 based'
+  c.arg_name 'ROW_NUMBER'
+  c.flag [:start]
+
   c.desc 'File doesn\'t contain a header'
   c.switch [:h, :headerless]
 
@@ -323,9 +329,14 @@ command :sort do |c|
   c.switch [:d, :desc]
 
   c.action do |global_options,options,args|
-    sorter = Sycsvpro::Sorter.new(infile: global_options[:f], outfile: global_options[:o],
-                                  rows: options[:r], cols: options[:c], df: options[:df],
-                                  headerless: options[:h], desc: options[:d])
+    sorter = Sycsvpro::Sorter.new(infile: global_options[:f], 
+                                  outfile: global_options[:o],
+                                  rows: options[:r], 
+                                  cols: options[:c], 
+                                  df: options[:df],
+                                  start: options[:start],
+                                  headerless: options[:h], 
+                                  desc: options[:d])
     print 'Sorting...'
     sorter.execute
     print 'done'

data/lib/sycsvpro/aggregator.rb

@@ -5,7 +5,8 @@ require_relative 'dsl'
 # Operating csv files
 module Sycsvpro
 
-  # An Aggregator counts specified row values and adds a sum to the end of the row
+  # An Aggregator counts specified row values and adds a sum to the end of 
+  # the row
   class Aggregator
 
     include Dsl
@@ -35,8 +36,16 @@ module Sycsvpro
     # sums of the column values
     attr_reader :sums
     
-    # Creates a new aggregator. Takes as attributes infile, outfile, key, rows, cols, date-format 
-    # and indicator whether to add a sum row
+    # Creates a new aggregator. Takes as attributes infile, outfile, key, rows, 
+    # cols, date-format and indicator whether to add a sum row
+    # :call-seq:
+    #   Sycsvpro::Aggregator.new(infile:     "in.csv",
+    #                            outfile:    "out.csv",
+    #                            headerless: false,
+    #                            rows:       "1,2-4,/\S/"
+    #                            cols:       "0,5",
+    #                            df:         "%d.%m.%Y",
+    #                            sum:        "Total:1,Items").execute
     def initialize(options={})
       @infile     = options[:infile]
       @outfile    = options[:outfile]
@@ -55,7 +64,8 @@ module Sycsvpro
       write_result
     end
 
-    # Process the aggregation of the key values
+    # Process the aggregation of the key values. The result will be written to
+    # _outfile_
     def process_aggregation
       File.new(infile).each_with_index do |line, index|
         result = col_filter.process(row_filter.process(line.chomp, row: index))
@@ -92,7 +102,8 @@ module Sycsvpro
 
     private
 
-      # Initializes the sum row title an positions as well as the sum column title and position
+      # Initializes the sum row title an positions as well as the sum column 
+      # title and position
       def init_sum_scheme(sum_scheme)
         row_scheme, col_scheme = sum_scheme.split(',') unless sum_scheme.nil?
 

data/lib/sycsvpro/calculator.rb

@@ -30,9 +30,18 @@ module Sycsvpro
     # if true add a sum row at the bottom of the out file
     attr_reader :add_sum_row
 
-    # Creates a new Calculator. Options expects :infile, :outfile, :rows and :columns. Optionally
-    # a header can be provided. The header can be supplemented with additional column names that
-    # are generated due to a arithmetic operation that creates new columns
+    # Creates a new Calculator. Options expects :infile, :outfile, :rows and 
+    # :columns. Optionally a header can be provided. The header can be 
+    # supplemented with additional column names that are generated due to a 
+    # arithmetic operation that creates new columns
+    # :call-seq:
+    #   Sycsvpro::Calculator.new(infile:  "in.csv",
+    #                            outfile: "out.csv",
+    #                            df:      "%d.%m.%Y",
+    #                            rows:    "1,2,BEGINn3>20END",
+    #                            header:  "*,Count",
+    #                            cols:    "4:Count=c1+c2*2",
+    #                            sum:     true).execute
     def initialize(options={})
       @infile      = options[:infile]
       @outfile     = options[:outfile]
@@ -46,12 +55,14 @@ module Sycsvpro
     end
 
     # Retrieves the values from a row as the result of a arithmetic operation
+    # with #eval
     def method_missing(id, *args, &block)
       return to_number(columns[$1.to_i]) if id =~ /c(\d+)/
       return to_date(columns[$1.to_i])   if id =~ /d(\d+)/
+      super
     end
 
-    # Executes the calculator
+    # Executes the calculator and writes the result to the _outfile_
     def execute
       processed_header = false
 
@@ -75,6 +86,8 @@ module Sycsvpro
           out.puts @columns.join(';')
 
           @columns.each_with_index do |column, index|
+            column = 0 unless column.to_s =~ /^[\d\.,]*$/
+
             if @sum_row[index]
               @sum_row[index] += to_number column
             else
@@ -107,7 +120,8 @@ module Sycsvpro
         end
       end
 
-      # Casts a string to an integer or float depending whether the value has a decimal point
+      # Casts a string to an integer or float depending whether the value has a 
+      # decimal point
       def to_number(value)
         return value.to_i unless value =~ /\./
         return value.to_f if     value =~ /\./ 

data/lib/sycsvpro/counter.rb

@@ -76,16 +76,13 @@ module Sycsvpro
           end
         end
       end
-      unless sum_col_title.nil?
-        heading << sum_col_title
-        sums[sum_col_title] = sums.values.inject(:+)
-      end
     end
 
-   # Writes the count results
+    # Writes the count results
     def write_result
       sum_line = [sum_row_title] + [''] * (key_titles.size - 1)
-      headline = heading_sort ? heading.sort : col_filter.pivot.keys
+      headline = heading_sort ? heading.sort : original_pivot_sequence_heading
+      headline << add_sum_col unless sum_col_title.nil?
       headline.each do |h|
         sum_line << sums[h]
       end
@@ -107,7 +104,8 @@ module Sycsvpro
 
     private
 
-      # Initializes the sum row title an positions as well as the cum column title
+      # Initializes the sum row title an positions as well as the sum column 
+      # title
       def init_sum_scheme(sum_scheme)
 
         return if sum_scheme.nil?
@@ -140,6 +138,18 @@ module Sycsvpro
 
       end
 
+      # Arrange heading in the original sequence regarding conditional column
+      # filters
+      def original_pivot_sequence_heading
+        (heading.sort - col_filter.pivot.keys << col_filter.pivot.keys).flatten
+      end
+
+      # Add a sum column to the end of the heading
+      def add_sum_col
+        sums[sum_col_title] = sums.values.inject(:+)
+        sum_col_title
+      end
+
   end
 
 end

data/lib/sycsvpro/dsl.rb

@@ -3,6 +3,49 @@ require_relative 'row_filter'
 # Methods to be used in customer specific script files
 module Dsl
 
+  # read arguments provided at invocation
+  # :call-seq:
+  #   params => infile, Result, other_params
+  #
+  # Result methods are #cols, #col_count, #row_count, #sample_row
+  def params
+
+    script = ARGV.shift
+    method = ARGV.shift
+    infile = ARGV.shift
+
+    if infile.nil?
+      STDERR.puts "You must provide an input file"
+      exit -1
+    elsif !File.exists? infile
+      STDERR.puts "#{infile} does not exist. You must provide a valid input file"
+      exit -1
+    end
+
+    if ARGV.empty?
+      print "#{method}(#{infile})"
+    else
+      print "#{method}(#{infile}, #{ARGV.join(', ')})"
+    end
+
+    puts; print "Analyzing #{infile}..."
+
+    result = Sycsvpro::Analyzer.new(infile).result
+    puts; print "> #{result.col_count} cols | #{result.row_count} rows"
+
+    [infile, result, ARGV].flatten
+
+  end
+
+  # Delete obsolete files
+  # :call-seq:
+  #   clean_up(%w{ file1 file2 }) -> nil
+  def clean_up(files)
+    puts; print "Cleaning up directory..."
+
+    files.each { |file| File.delete(file) }
+  end
+  
   # Retrieves rows and columns from the file and returns them to the block provided by the caller
   def rows(options={})
     infile     = File.expand_path(options[:infile])

data/lib/sycsvpro/filter.rb

@@ -66,11 +66,7 @@ module Sycsvpro
       return false if boolean_filter.empty? or values.empty?
       expression = boolean_filter
       columns = expression.scan(/(([nsd])(\d+))([<!=~>]{1,2})(.*?)(?:[\|&]{2}|$)/)
-#      STDERR.puts "expr = #{expression.inspect}"
-#      STDERR.puts "vals = #{values.inspect}"
-#      STDERR.puts "cols = #{columns.inspect}"
       columns.each do |c|
-#        STDERR.puts "val = #{values[c[2].to_i].inspect}"
         value = case c[1]
         when 'n'
           values[c[2].to_i].empty? ? '0' : values[c[2].to_i]
@@ -93,9 +89,7 @@ module Sycsvpro
           end 
         end
         expression = expression.gsub(c[0], value)
-#        STDERR.puts "val2 = #{value}"
       end
-#      STDERR.puts "exp = #{expression.inspect}"
       eval(expression)
     end
 
@@ -124,7 +118,6 @@ module Sycsvpro
       # Creates a filter based on the provided rows and columns select criteria
       def create_filter(values)
         values.scan(/(?<=,|^)(BEGIN.*?END|\/.*?\/|.*?)(?=,|$)/i).flatten.each do |value|
-#          STDERR.puts "value = #{value}"
           send(value)
         end unless values.nil?
       end

data/lib/sycsvpro/sorter.rb

@@ -20,17 +20,33 @@ module Sycsvpro
     attr_reader :col_type_filter
     # sorted rows
     attr_reader :sorted_rows
-    # file doesn't contain a header
+    # file doesn't contain a header. If not headerless then empty rows from
+    # beginning of file are discarted and first non empty row is considered as
+    # header. Subsequent rows will be sorted and added in the resulting file
+    # after the header
     attr_reader :headerless
+    # First row to sort. Will skip rows 0 to start - 1 and add them to top of
+    # file. Rows from start on will be sorted.
+    attr_reader :start
     # sort order descending or ascending
     attr_reader :desc
 
-    # Creates a Sorter and takes as options infile, outfile, rows, cols including types and a
-    # date format for the date columns to sort (optional).
+    # Creates a Sorter and takes as options infile, outfile, rows, cols 
+    # including types and a date format for the date columns to sort (optional).
+    # :call-seq:
+    #   Sycsvrpo::Sorter.new(infile:     "infile.csv",
+    #                        outfile:    "outfile.csv",
+    #                        rows:       "1,2-5,12-30",
+    #                        cols:       "n:1,s:3",
+    #                        headerless: true,
+    #                        df:         "%d.%m.%Y",
+    #                        start:      "2").execute
+    # The sorted infile will saved to outfile
     def initialize(options={})
       @infile          = options[:infile]
       @outfile         = options[:outfile]
       @headerless      = options[:headerless] || false
+      @start           = options[:start]
       @desc            = options[:desc] || false
       @row_filter      = RowFilter.new(options[:rows], df: options[:df])
       @col_type_filter = ColumnTypeFilter.new(options[:cols], df: options[:df])
@@ -41,9 +57,15 @@ module Sycsvpro
     def execute
       rows = File.readlines(infile)
 
+      skipped_rows = []
+
       unless headerless
-        header = ""
-        header = rows.shift while header.chomp.strip.empty?
+        skipped_rows[0] = ""
+        skipped_rows[0] = rows.shift while skipped_rows[0].chomp.strip.empty?
+      end
+
+      if start
+        (0...start.to_i).each { |row| skipped_rows << rows.shift }  
       end
 
       rows.each_with_index do |line, index|
@@ -53,9 +75,7 @@ module Sycsvpro
       end
 
       File.open(outfile, 'w') do |out|
-        unless headerless
-          out.puts header
-        end
+        skipped_rows.each { |row| out.puts unstring(row) }
 
         if desc
           sorted_rows.compact.sort.reverse.each do |row|

data/lib/sycsvpro/version.rb

@@ -1,5 +1,5 @@
 # Operating csv files
 module Sycsvpro
   # Version number of sycsvpro
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

data/spec/sycsvpro/sorter_spec.rb

@@ -6,6 +6,7 @@ module Sycsvpro
 
     before do
       @in_file   = File.join(File.dirname(__FILE__), "files/in.csv")
+      @in_start  = File.join(File.dirname(__FILE__), "files/in_start.csv")
       @in_2_file = File.join(File.dirname(__FILE__), "files/in2.csv")
       @in_3_file = File.join(File.dirname(__FILE__), "files/in3.csv")
       @out_file  = File.join(File.dirname(__FILE__), "files/out.csv")
@@ -80,6 +81,55 @@ module Sycsvpro
       end
     end
 
+    it "should start sorting at specified row with headerless false" do
+      rows = "0-7"
+      cols = "s:3-5,s:0"
+      df   = "%d.%m.%Y"
+
+      sorter = Sorter.new(infile: @in_start, outfile: @out_file, rows: rows,
+                          cols: cols, df: df, start: "1")
+
+      sorter.execute
+
+      result = [ "customer;contract-number;expires-on;machine;product1;product2",
+                 "Total;0;0;0;0;0",
+                 "Fink;1234;20.12.2015;f1;con123;dri222",
+                 "Fink;1234;30.12.2016;f2;con333;dri321",
+                 "fink;1234;;f3;con332;dri321",
+                 "Gent;4323;1.3.2014;g1;con123;dri111",
+                 "Haas;3322;1.10.2011;h1;con332;dri111",
+                 "Klig;4432;;k1;con332;dri222",
+                 "Rank;3232;1.5.2013;r1;con332;dri321" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+    end
+
+    it "should start sorting at specified row with header true" do
+      rows = "0-7"
+      cols = "s:3-5,s:0"
+      df   = "%d.%m.%Y"
+
+      sorter = Sorter.new(infile: @in_start, outfile: @out_file, rows: rows,
+                          cols: cols, df: df, headerless: true, start: "2")
+
+      sorter.execute
+
+      result = [ "customer;contract-number;expires-on;machine;product1;product2",
+                 "Total;0;0;0;0;0",
+                 "Fink;1234;20.12.2015;f1;con123;dri222",
+                 "Fink;1234;30.12.2016;f2;con333;dri321",
+                 "fink;1234;;f3;con332;dri321",
+                 "Gent;4323;1.3.2014;g1;con123;dri111",
+                 "Haas;3322;1.10.2011;h1;con332;dri111",
+                 "Klig;4432;;k1;con332;dri222",
+                 "Rank;3232;1.5.2013;r1;con332;dri321" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+    end
 
     it "should sort a date column" do
       rows = "1-7"

data/sycsvpro.gemspec

@@ -20,5 +20,6 @@ spec = Gem::Specification.new do |s|
   s.add_development_dependency('rdoc')
   s.add_development_dependency('aruba')
   s.add_development_dependency('rspec')
+  s.add_runtime_dependency('timeleap', '~>0.0.1')
   s.add_runtime_dependency('gli','2.9.0')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sycsvpro
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-17 00:00:00.000000000 Z
+date: 2014-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -75,6 +75,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timeleap
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.1
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -112,70 +128,6 @@ files:
 - features/step_definitions/sycsvpro_steps.rb
 - features/support/env.rb
 - features/sycsvpro.feature
-- html/Dsl.html
-- html/Object.html
-- html/README_rdoc.html
-- html/Sycsvpro.html
-- html/Sycsvpro/Aggregator.html
-- html/Sycsvpro/Allocator.html
-- html/Sycsvpro/Analyzer.html
-- html/Sycsvpro/Calculator.html
-- html/Sycsvpro/Collector.html
-- html/Sycsvpro/ColumnFilter.html
-- html/Sycsvpro/ColumnTypeFilter.html
-- html/Sycsvpro/Counter.html
-- html/Sycsvpro/Extractor.html
-- html/Sycsvpro/Filter.html
-- html/Sycsvpro/Header.html
-- html/Sycsvpro/Inserter.html
-- html/Sycsvpro/Mapper.html
-- html/Sycsvpro/Profiler.html
-- html/Sycsvpro/RowFilter.html
-- html/Sycsvpro/ScriptCreator.html
-- html/Sycsvpro/ScriptList.html
-- html/Sycsvpro/Sorter.html
-- html/created.rid
-- html/fonts.css
-- html/fonts/Lato-Light.ttf
-- html/fonts/Lato-LightItalic.ttf
-- html/fonts/Lato-Regular.ttf
-- html/fonts/Lato-RegularItalic.ttf
-- html/fonts/SourceCodePro-Bold.ttf
-- html/fonts/SourceCodePro-Regular.ttf
-- html/images/add.png
-- html/images/arrow_up.png
-- html/images/brick.png
-- html/images/brick_link.png
-- html/images/bug.png
-- html/images/bullet_black.png
-- html/images/bullet_toggle_minus.png
-- html/images/bullet_toggle_plus.png
-- html/images/date.png
-- html/images/delete.png
-- html/images/find.png
-- html/images/loadingAnimation.gif
-- html/images/macFFBgHack.png
-- html/images/package.png
-- html/images/page_green.png
-- html/images/page_white_text.png
-- html/images/page_white_width.png
-- html/images/plugin.png
-- html/images/ruby.png
-- html/images/tag_blue.png
-- html/images/tag_green.png
-- html/images/transparent.png
-- html/images/wrench.png
-- html/images/wrench_orange.png
-- html/images/zoom.png
-- html/index.html
-- html/js/darkfish.js
-- html/js/jquery.js
-- html/js/navigation.js
-- html/js/search.js
-- html/js/search_index.js
-- html/js/searcher.js
-- html/rdoc.css
-- html/table_of_contents.html
 - lib/sycsvpro.rb
 - lib/sycsvpro/aggregator.rb
 - lib/sycsvpro/allocator.rb