sycsvpro 0.0.1

data/lib/sycsvpro/filter.rb

@@ -0,0 +1,98 @@
+require 'date'
+
+# Operating csv files
+module Sycsvpro
+
+  # Creates a new filter that can be extended by sub-classes. A sub-class needs to override the
+  # process method
+  class Filter
+
+    # When date are used as filters the date format has to be provided
+    attr_reader :date_format
+    # Filter for rows and columns
+    attr_reader :filter
+    # Pattern that is used as a filter
+    attr_reader :pattern
+    # Comparison that is used as a filter
+    attr_reader :pivot
+
+    # Creates a new filter
+    def initialize(values, options={})
+      @date_format = options[:df] || "%Y-%m-%d"
+      @filter  = []
+      @pattern = []
+      @pivot   = {}
+      create_filter(values)
+    end
+
+    # Creates the filters based on the given patterns
+    def method_missing(id, *args, &block)
+      return equal($1, args, block)              if id =~ /^(\d+)$/
+      return range($1, $2, args, block)          if id =~ /^(\d+)-(\d+)$/
+      return regex($1, args, block)              if id =~ /^\/(.*)\/$/
+      return col_regex($1, $2, args, block)      if id =~ /^(\d+):\/(.*)\/$/
+      return date($1, $2, $3, args, block)       if id =~ /^(\d+):(<|=|>)(\d+.\d+.\d+)/
+      return date_range($1, $2, $3, args, block) if id =~ /^(\d+):(\d+.\d+.\d+.)-(\d+.\d+.\d+)$/
+      super
+    end
+
+    # Processes the filter. Needs to be overridden by the sub-class
+    def process(object, options={})
+      raise 'Needs to be overridden by sub class'
+    end
+
+    # Yields the column value and whether the filter matches the column
+    def pivot_each_column(values=[])
+      pivot.each do |column, parameters|
+        yield column, eval(parameters[:operation].gsub('[value]', values[parameters[:col].to_i]))
+      end
+    end
+
+    private
+
+      # Creates a filter based on the provided rows and columns
+      def create_filter(values)
+        values.split(',').each { |f| send(f) } unless values.nil?
+      end
+
+      # Adds a single value to the filter
+      def equal(value, args, block)
+        filter << value.to_i unless filter.index(value.to_i) 
+      end
+
+      # Adds a range to the filter
+      def range(start_value, end_value, args, block)
+        filter << (start_value.to_i..end_value.to_i).to_a
+      end
+
+      # Adds a regex to the pattern filter
+      def regex(value, args, block)
+        pattern << value unless pattern.index(value)
+      end
+
+      # Adds a comparisson filter 
+      def col_regex(col, r, args, block)
+        operation = "'[value]' =~ Regexp.new('#{r}')"
+        pivot[r] = { col: col, operation: operation } 
+      end
+
+      # Adds a date filter
+      def date(col, comparator, date, args, block)
+        comparator = '==' if comparator == '='
+        operation = "Date.strptime(\"[value]\", \"#{date_format}\") #{comparator} " +
+                    "Date.strptime(\"#{date}\", \"#{date_format}\")"
+        pivot["#{comparator}#{date}"] = { col: col, operation: operation }
+      end
+
+      # Adds a date range filter
+      def date_range(col, start_date, end_date, args, block)
+        operation = "   Date.strptime(\"#{start_date}\",  \"#{date_format}\") "    +
+                    "<= Date.strptime(\"[value]\",        \"#{date_format}\") && " +
+                    "   Date.strptime(\"[value]\",        \"#{date_format}\") "    +
+                    "<= Date.strptime(\"#{end_date}\",    \"#{date_format}\")"
+        pivot["#{start_date}-#{end_date}"] = { col: col, operation: operation }
+      end
+
+  end
+
+end

data/lib/sycsvpro/header.rb

@@ -0,0 +1,29 @@
+require_relative 'filter'
+
+# Operating csv files
+module Sycsvpro
+
+  # Creates a header
+  class Header < Filter
+
+    # Header columns
+    attr_reader :header_cols
+
+    # Create a new header
+    def initialize(header)
+      unless header.nil? or header.empty?
+        @header_cols = header.split(',')
+      else
+        @header_cols = []
+      end
+    end
+
+    # Returns the header
+    def process(line)
+      return "" if @header_cols.empty?
+      @header_cols[0] = line.split(';')
+      @header_cols.flatten.join(';')
+    end
+  end
+
+end

data/lib/sycsvpro/mapper.rb

@@ -0,0 +1,53 @@
+# Operating csv files
+module Sycsvpro
+
+  # Map values to new values described in a mapping file
+  class Mapper
+
+    # infile contains the data that is operated on
+    attr_reader :infile
+    # outfile is the file where the result is written to
+    attr_reader :outfile
+    # file that contains the mappings from existing column values to new values
+    attr_reader :mapper
+    # filter that is used for rows
+    attr_reader :row_filter
+    # filter that is used for columns
+    attr_reader :col_filter
+
+    # Creates new mapper
+    def initialize(options={})
+      @infile = options[:infile]
+      @outfile = options[:outfile]
+      @row_filter = RowFilter.new(options[:row_filter])
+      @col_filter = ColumnFilter.new(options[:col_filter])
+      @mapper = {}
+      init_mapper(options[:mapping])
+    end
+
+    # Executes the mapper
+    def execute
+      File.open(outfile, 'w') do |out|
+        File.new(infile, 'r').each_with_index do |line, index|
+          result = col_filter.process(row_filter.process(line, row: index))
+          next if result.chomp.empty? or result.nil?
+          mapper.each do |from, to|
+            result = result.chomp.gsub(/(?<=^|;)#{from}(?=;|$)/, to)
+          end
+          out.puts result
+        end
+      end
+    end
+
+    private
+
+      # Initializes the mappings
+      def init_mapper(file)
+        File.new(file, 'r').each_line do |line|
+          from, to = line.chomp.split(':')
+          mapper[from] = to
+        end
+      end
+  end
+
+end

data/lib/sycsvpro/profiler.rb

@@ -0,0 +1,26 @@
+require_relative 'dsl'
+
+# Operating csv files
+module Sycsvpro
+
+  # A profiler takes a Ruby script and executes the provided method in the script
+  class Profiler
+
+    include Dsl
+
+    # Ruby script file
+    attr_reader :pro_file
+
+    # Creates a new profiler
+    def initialize(pro_file)
+      require pro_file
+    end
+
+    # Executes the provided method in the Ruby script
+    def execute(method)
+      send(method)
+    end
+
+  end
+
+end

data/lib/sycsvpro/row_filter.rb

@@ -0,0 +1,20 @@
+require_relative 'filter'
+
+# Operating csv files
+module Sycsvpro
+
+  # Filters rows based on provided patterns
+  class RowFilter < Filter
+    
+    # Processes the filter on the given row
+    def process(object, options={})
+      filtered = (!filter.flatten.uniq.index(options[:row]).nil? or filter.empty?)
+      pattern.each do |p|
+        filtered = (filtered or !(object =~ Regexp.new(p)).nil?)
+      end
+      filtered ? object : nil
+    end
+    
+  end
+
+end

data/lib/sycsvpro/version.rb

@@ -0,0 +1,5 @@
+# Operating csv files
+module Sycsvpro
+  # Version number of sycsvpro
+  VERSION = '0.0.1'
+end

data/lib/sycsvpro.rb

@@ -0,0 +1,9 @@
+require 'sycsvpro/version.rb'
+require 'sycsvpro/analyzer.rb'
+require 'sycsvpro/extractor.rb'
+require 'sycsvpro/profiler.rb'
+require 'sycsvpro/counter.rb'
+require 'sycsvpro/collector.rb'
+require 'sycsvpro/mapper.rb'
+require 'sycsvpro/header.rb'
+require 'sycsvpro/calculator.rb'

data/spec/sycsvpro/analyze_spec.rb

@@ -0,0 +1,23 @@
+require 'sycsvpro/analyzer.rb'
+
+module Sycsvpro
+
+  describe Analyzer do
+
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/in.csv")
+    end
+
+    it "should analyze infile" do
+      analyzer = Analyzer.new(@in_file)
+      result = analyzer.result
+      result.cols.should =~ ['customer', 'contract-number', 'expires-on', 'machine', 
+                             'product1', 'product2']
+      result.col_count.should eq 6
+      result.row_count.should eq 5
+      result.sample_row.should eq "Fink;1234;20.12.2015;f1;con123;dri222"
+    end
+
+  end
+
+end

data/spec/sycsvpro/calculator_spec.rb

@@ -0,0 +1,45 @@
+require 'sycsvpro/calculator.rb'
+
+module Sycsvpro
+
+  describe Calculator do
+
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/machines.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/machines_out.csv")
+    end
+
+    it "should operate on existing row" do
+      rows = "2-8"
+      cols = "3:*3,4:*4+1"
+      calculator = Calculator.new(infile: @in_file, outfile: @out_file, rows: rows, cols: cols)
+      calculator.execute
+
+      result = ["Fink;2;2;3;5", "Haas;3;3;3;5.0", "Gent;4;4;3;5", "Rank;5;5;3;5"]
+
+      File.new(@out_file, 'r').each_with_index do |line, index|
+        expect(line.chomp).to eq result[index]
+      end 
+    end
+
+    it "should add additional rows" do
+      header = "*,drives,motors"
+      rows = "1-8"
+      cols = "5:c3+c4,6:c3*2"
+      calculator = Calculator.new(infile: @in_file, outfile: @out_file, 
+                                  header: header, rows: rows, cols: cols)
+      calculator.execute
+
+      result = ["customer;machines;controls;contracts;visits;drives;motors",
+                "Fink;2;2;1;1;2;2",
+                "Haas;3;3;1;1.0;2.0;2",
+                "Gent;4;4;1;1;2;2",
+                "Rank;5;5;1;1;2;2"]
+
+      File.new(@out_file, 'r').each_with_index do |line, index|
+        expect(line.chomp).to eq result[index]
+      end
+    end
+  end
+
+end

data/spec/sycsvpro/collector_spec.rb

@@ -0,0 +1,27 @@
+require 'sycsvpro/collector.rb'
+
+module Sycsvpro
+
+  describe Collector do
+
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/in.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+
+    it "should collect and categorize values" do
+      collector = Collector.new(infile: @in_file, outfile: @out_file, 
+                                cols: "customer:0+products:4,5", rows: "1-20")
+      collector.execute
+
+      result = ['[customer]', 'Fink', 'Gent', 'Haas', 'Rank',
+                '[products]', 'con123', 'con332', 'con333', 
+                              'dri111', 'dri222', 'dri321']
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+    end
+
+  end
+
+end

data/spec/sycsvpro/counter_spec.rb

@@ -0,0 +1,51 @@
+require 'sycsvpro/counter.rb'
+
+module Sycsvpro
+
+  describe Counter do
+
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/in.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+
+    it "should count columns" do
+      counter = Counter.new(infile: @in_file, outfile: @out_file, rows: "1-10", cols: "4,5", 
+                            key: "0")
+
+      counter.execute
+
+      result = [ "customer;con123;con332;con333;dri111;dri222;dri321",
+                 "Fink;1;0;1;0;1;1",
+                 "Haas;0;1;0;1;0;0",
+                 "Gent;1;0;0;1;0;0",
+                 "Rank;0;1;0;0;0;1" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+
+    it "should count date coloms" do
+      counter = Counter.new(infile: @in_file, outfile: @out_file, rows: "1-10", 
+                            cols: "2:<1.1.2013,2:1.1.2013-31.12.2014,2:>31.12.2014", key: "0",
+                            df: "%d.%m.%Y")
+
+      counter.execute
+
+      result = [ "customer;1.1.2013-31.12.2014;<1.1.2013;>31.12.2014",
+                 "Fink;0;0;2",
+                 "Haas;0;1;0",
+                 "Gent;1;0;0",
+                 "Rank;1;0;0" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+
+  end
+
+end

data/spec/sycsvpro/extractor_spec.rb

@@ -0,0 +1,27 @@
+require 'sycsvpro/extractor.rb'
+
+module Sycsvpro
+
+  describe Extractor do
+
+    before do
+      @in_file = File.join(File.dirname(__FILE__), "files/in.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+
+    it "should extract rows and columns" do
+      extractor = Extractor.new(infile: @in_file, outfile: @out_file, rows: "2-4", cols: "1,3")
+
+      extractor.execute
+
+      result = ["3322;h1", "4323;g1", "3342;f2"]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+
+  end
+
+end

data/spec/sycsvpro/files/mappings

@@ -0,0 +1,6 @@
+con123:control123
+con332:control332
+con333:control333
+dri222:drive222
+dri111:drive111
+dri321:drive321

data/spec/sycsvpro/files/profile.rb

@@ -0,0 +1,42 @@
+# This is an example of a sycsvpro DSL to process a CSV file
+
+def calc
+
+  customers = {}
+  heading = []
+
+  rows infile: "./spec/sycsvpro/files/in.csv", 
+       row_filter: "1-20",
+       key_column: 0, 
+       machine_column: 3, 
+       data_columns: [4,5] do |key, machine, columns|
+    customer = customers[key] || customers[key] = { name: key, products: Hash.new(0) }
+    columns.each do |column|
+      heading << column if heading.index(column).nil?
+      customer[:products][column] += 1
+    end
+  end
+  
+#  puts heading.sort.join('-')
+
+=begin
+  customers.each do |k,v|
+    print k
+    heading.sort.each do |h|
+      print " #{v[:products][h]} "
+    end
+    puts
+  end
+=end
+
+  write_to "./spec/sycsvpro/files/out.csv" do |out|
+    out.puts (["customer"] + heading.sort).join(';')
+    customers.each do |k,v|
+      line = [k]
+      heading.sort.each do |h|
+        line << v[:products][h]
+      end
+      out.puts line.join(';')
+    end
+  end
+end

data/spec/sycsvpro/mapper_spec.rb

@@ -0,0 +1,33 @@
+require 'sycsvpro/mapper.rb'
+
+module Sycsvpro
+
+  describe Mapper do
+
+    before do
+      @in_file  = File.join(File.dirname(__FILE__), "files/in.csv")
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+      @mappings = File.join(File.dirname(__FILE__), "files/mappings")
+    end
+
+    it "should map values to new values" do
+      mapper = Mapper.new(infile: @in_file, outfile: @out_file, mapping: @mappings)
+
+      mapper.execute
+
+      result = [ "customer;contract-number;expires-on;machine;product1;product2",
+                 "Fink;1234;20.12.2015;f1;control123;drive222",
+                 "Haas;3322;1.10.2011;h1;control332;drive111",
+                 "Gent;4323;1.3.2014;g1;control123;drive111",
+                 "Fink;3342;30.12.2016;f2;control333;drive321",
+                 "Rank;3232;1.5.2013;r1;control332;drive321" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+
+  end
+
+end

data/spec/sycsvpro/profiler_spec.rb

@@ -0,0 +1,32 @@
+require 'sycsvpro/profiler.rb'
+
+module Sycsvpro
+
+  describe Profiler do
+
+    before do
+      @profile = File.join(File.dirname(__FILE__), "files/profile.rb")
+      @method  = "calc"
+      @out_file = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+
+    it "should execute the profile file" do
+      profiler = Profiler.new(@profile)
+
+      profiler.execute(@method)
+
+      result = [ "customer;con123;con332;con333;dri111;dri222;dri321",
+                 "Fink;1;0;1;0;1;1",
+                 "Haas;0;1;0;1;0;0",
+                 "Gent;1;0;0;1;0;0",
+                 "Rank;0;1;0;0;0;1" ]
+
+      File.open(@out_file).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+      end
+
+    end
+
+  end
+
+end

data/sycsvpro.gemspec

@@ -0,0 +1,24 @@
+# Ensure we require the local version and not one we might have installed already
+require File.join([File.dirname(__FILE__),'lib','sycsvpro','version.rb'])
+spec = Gem::Specification.new do |s| 
+  s.name = 'sycsvpro'
+  s.version = Sycsvpro::VERSION
+  s.author = 'Pierre Sugar'
+  s.email = 'pierre@sugaryourcoffee.de'
+  s.homepage = 'https://github.com/sugaryourcoffee/syc-svpro'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'Processing of csv files'
+  s.files = `git ls-files`.split("
+")
+  s.require_paths << 'lib'
+  s.has_rdoc = true
+  s.extra_rdoc_files = ['README.rdoc','sycsvpro.rdoc']
+  s.rdoc_options << '--title' << 'sycsvpro' << '--main' << 'README.rdoc' << '-ri'
+  s.bindir = 'bin'
+  s.executables << 'sycsvpro'
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rdoc')
+  s.add_development_dependency('aruba')
+  s.add_development_dependency('rspec')
+  s.add_runtime_dependency('gli','2.9.0')
+end

data/sycsvpro.rdoc

@@ -0,0 +1,29 @@
+= sycsvpro
+
+NAME
+    sycsvpro - Processing CSV files
+
+SYNOPSIS
+    sycsvpro [global options] command [command options] [arguments...]
+
+VERSION
+    0.0.1
+
+GLOBAL OPTIONS
+    -f, --file=FILE    - CSV file to operate on (default: none)
+    --help             - Show this message
+    -o, --out=OUT_FILE - CSV file to write the result to (default: none)
+    --version          - Display the program version
+
+COMMANDS
+    analyze - Analyze the CSV file regarding columns, rows and content
+    calc    - Process math operations on columns
+    collect - Collect values of specified rows and columns from the file and group them in
+              categories
+    count   - Counts the occurences of column values. Uses column values as headings with count as
+              values. Columns with a condition will be added as new columns and the condition will
+              be set as column name
+    execute - Executes the code provided in a file
+    extract - Extract specified rows and columns from the file
+    help    - Shows a list of commands or help for one command
+    map     - Map values in columns to new values