sycsvpro 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c635bce6d7a54564795bf018de60a62a8c1efec2
+  data.tar.gz: 6ac41a7ed53e1305632ec183b2f9d10a0d536fac
+SHA512:
+  metadata.gz: 7584ba89eb1ba64a658aa610f7d4972a47df95a6acb8d5b8ad90b8b994cf7af43464c480d79a296052c50f990b69115386ac1a447ddba055e370d44e767047ed
+  data.tar.gz: 3ae8b59e1a79b4c9decb7de253b32d8df9b3dc4b463b856d6f7e8353169b5336777e2cbfe92bcf7fb7ddaa24d38f1aa52c840924403ba43dae1ec08c64af3469

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+# Test files
+*.csv

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,51 @@
+PATH
+  remote: .
+  specs:
+    sycsvpro (0.0.1)
+      gli (= 2.9.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aruba (0.5.4)
+      childprocess (>= 0.3.6)
+      cucumber (>= 1.1.1)
+      rspec-expectations (>= 2.7.0)
+    builder (3.2.2)
+    childprocess (0.4.0)
+      ffi (~> 1.0, >= 1.0.11)
+    cucumber (1.3.10)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.0.2)
+    diff-lcs (1.2.5)
+    ffi (1.9.3)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    gli (2.9.0)
+    json (1.8.1)
+    multi_json (1.8.4)
+    multi_test (0.0.3)
+    rake (10.1.1)
+    rdoc (4.1.1)
+      json (~> 1.4)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba
+  rake
+  rdoc
+  rspec
+  sycsvpro!

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Pierre Sugar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,188 @@
+syc-svpro
+=========
+
+Processing of csv files. *sycsvpro* offers following functions
+
+* analyze csv file
+* extract rows and columns from a file
+* collect values of rows and assign them to categories
+* map column values to new values
+* count values in columns and use the value as column name
+* arithmetic operations on values of columns
+* execute a ruby script file that operates a csv file
+
+To get help type
+
+    $ sycsvpro -h
+    
+In the following examples we assume the following file
+
+```
+customer;machine;control;drive;motor;date;contract
+hello;h1;con123;dri120;mot100;1.12.3013;1
+hello;h2;con123;dri130;mot110;1.12.3013;1
+indix;i1;con456;dri130;mot090;1.12.3013;1
+chiro;c1;con333;dri110;mot100;1.12.3013;1
+chiro;c2;con331;dri100;mot130;1.12.3013;1
+```
+
+Analyze
+-------
+
+Analyze the content of the provided file *in.csv*
+
+    $ sycsvpro -f in.csv analyze
+    Analysis of in.csv
+    7 columns: ["customer", "machine", "control", "drive", "motor", "date", "contract"]
+    10393 rows
+    0: customer
+    1: machine
+    2: control
+    3: drive
+    4: motor
+    5: date
+    6: contract
+    Row sample data:
+    hello;h1;con123;dri120;mot100;16.02.2014;1
+
+Extract
+-------
+
+Extract row 1,2 and 10-20 as well as columns 4 and 6-7
+
+    $ sycsvpro -f in.csv -o out.csv extract -r 1,2,10-20 -c 4,6-7
+
+Collect
+-------
+
+Collect all product rows (2, 3 and 4) to the category product
+
+    $ sycsvpro -f in.csv -o out.csv collect -r 2-20 -c products:2-4
+    $ cat out.csv
+    [products]
+    con123
+    con331
+    con333
+    con456
+    dri100
+    dri110
+    dri120
+    dri130
+    mot090
+    mot100
+    mot110
+    mot130
+
+Map
+---
+
+Map the product names to new names
+
+The mapping file (mapping) uses the result from the collect command above
+
+    con123:control123
+    con331:control331
+    con333:control333
+    con456:control456
+    dri100:drive100
+    dri110:drive110
+    dri120:drive120
+    dri130:drive130
+    mot090:motor090
+    mot100:motor100
+    mot110:motor110
+    mot130:motor130
+
+    $ sycsvpro -f in.csv -o out.csv map mapping -c 2-4
+
+Count
+-----
+
+Count all customers (key column) in rows 2 to 20 that have machines that start with *h* and have contract valid beginning after 1.1.2000
+
+    $ sycsvpro -f in.csv -o out.csv count -r 2-20 -k 0 -c 1:/^h/,5:">1.1.2000" --df "%d.%m.%Y"
+
+The result in file out.csv is
+
+    $ cat out.csv
+    customer;>1.1.2000;^h
+    hello;2;2
+    indix;1;0
+    chiro;2;0
+
+Calc
+----
+
+Process arithmetic operations on the contract count and create a target column
+
+    $ sycsvpro -f in.csv -o out.csv calc -r 2-20 -h *,target -c 6:*2,7:target=c6*10
+
+    $ cat out.csv
+    customer;machine;control;drive;motor;date;contract;target
+    hello;h1;con123;dri120;mot100;1.12.3013;2;20
+    hello;h2;con123;dri130;mot110;1.12.3013;2;20
+    indix;i1;con456;dri130;mot090;1.12.3013;2;20
+    chiro;c1;con333;dri110;mot100;1.12.3013;2;20
+    chiro;c2;con331;dri100;mot130;1.12.3013;2;20
+
+Execute
+-------
+
+Execute takes a Ruby script file as an argument and processes the script. The following command executes the script *script.rb* and invokes the method *calc*
+
+    $ sycsvpro execute ./script.rb calc
+
+Below is an example script file that is ultimately doing the same as the count command
+
+    $ sycsvpro -f in.csv -o out.csv count -r 1-20 -k 0 -c 4,5
+
+```
+def calc
+
+  customers = {}
+  heading = []
+
+  rows infile: "./spec/sycsvpro/files/in.csv", 
+       row_filter: "1-20",
+       key_column: 0, 
+       machine_column: 3, 
+       data_columns: [4,5] do |key, machine, columns|
+    customer = customers[key] || customers[key] = { name: key, products: Hash.new(0) }
+    columns.each do |column|
+      heading << column if heading.index(column).nil?
+      customer[:products][column] += 1
+    end
+  end
+  
+  write_to "./spec/sycsvpro/files/out.csv" do |out|
+    out.puts (["customer"] + heading.sort).join(';')
+    customers.each do |k,v|
+      line = [k]
+      heading.sort.each do |h|
+        line << v[:products][h]
+      end
+      out.puts line.join(';')
+    end
+  end
+end
+```
+
+*rows* and *write_to* are convenience methods provided by sycsvpro that can be used in script files to operate on files.
+
+*rows* will return values at the specified columns in the order they are provided in the call to
+rows. The columns to be returned in the block have to end with _column_ or _columns_ dependent if a value or an array should be returned. You can find the *rows* and *write_to* methods at _lib/sycsvpro/dsl.rb_.
+
+Working with sycsvpro
+=====================
+
+sycsvpro emerged from my daily work when cleaning and anaylzing data. If you want to dig deeper I would recommend [R](http://www.r-project.org/).
+
+A work flow could be as follows
+
+* Analyze the file `analyze`
+* Clean the data `map`
+* Extract rows and columns of interest `extract`
+* Count values `count`
+* Do arithmetic operations on the values `calc`
+
+When I have analyzed the data I use _Microsoft Excel_ or _LibreOffice Calc_ to create nice graphs. To create more sophisiticated analysis *R* is the right tool to use.

data/README.rdoc

@@ -0,0 +1,44 @@
+= sycsvpro
+
+Author:: Pierre Sugar (mailto:pierre@sugaryourcoffee.de)
+Copyright:: Copyright (c) 2014 by Pierre Sugar
+License:: Distributed uder the MIT license, see LICENSE in the source distro
+
+The application provides an interface for analyzing, cleaning and operating on csv files
+
+
+== Install
+
+Install:
+
+    gem install sycsvpro
+
+== Use
+
+    sycsvpro --help
+
+== Developing for `sycsvpro`
+
+First install bundler
+
+    gem install bundler
+
+Get the development dependencies
+
+    bundle install
+
+The code is in
+
+    lib/sycsvpro
+    bin
+
+Tests are in
+
+    spec/sycsvpro
+
+Test files are in
+
+    spec/sycsvpro/files
+
+:include:sycsvpro.rdoc
+

data/Rakefile

@@ -0,0 +1,44 @@
+require 'rake/clean'
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'cucumber'
+require 'cucumber/rake/task'
+Rake::RDocTask.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+  rd.title = 'Your application title'
+end
+
+spec = eval(File.read('sycsvpro.gemspec'))
+
+Gem::PackageTask.new(spec) do |pkg|
+end
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+desc 'Run features'
+Cucumber::Rake::Task.new(:features) do |t|
+  opts = "features --format html -o #{CUKE_RESULTS} --format progress -x"
+  opts += " --tags #{ENV['TAGS']}" if ENV['TAGS']
+  t.cucumber_opts =  opts
+  t.fork = false
+end
+
+desc 'Run features tagged as work-in-progress (@wip)'
+Cucumber::Rake::Task.new('features:wip') do |t|
+  tag_opts = ' --tags ~@pending'
+  tag_opts = ' --tags @wip'
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty -x -s#{tag_opts}"
+  t.fork = false
+end
+
+task :cucumber => :features
+task 'cucumber:wip' => 'features:wip'
+task :wip => 'features:wip'
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+end
+
+task :default => [:test,:features]

data/bin/sycsvpro

@@ -0,0 +1,208 @@
+#!/usr/bin/env ruby
+require 'gli'
+begin # XXX: Remove this begin/rescue before distributing your app
+require 'sycsvpro'
+rescue LoadError
+  STDERR.puts "In development, you need to use `bundle exec bin/sycsvpro` to run your app"
+  STDERR.puts "At install-time, RubyGems will make sure lib, etc. are in the load path"
+  STDERR.puts "Feel free to remove this message from bin/sycsvpro now"
+  exit 64
+end
+
+include GLI::App
+
+program_desc 'Processing CSV files'
+
+version Sycsvpro::VERSION
+
+desc 'CSV file to operate on'
+arg_name 'FILE'
+flag [:f,:file]
+
+desc 'CSV file to write the result to'
+arg_name 'OUT_FILE'
+flag [:o, :out]
+
+desc 'Analyze the CSV file regarding columns, rows and content'
+command :analyze do |c|
+
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a file to analyze '-f FILE'" if global_options[:f].nil?
+    analyzer = Sycsvpro::Analyzer.new(global_options[:f])
+    result = analyzer.result
+    puts "Analysis of #{global_options[:f]}"
+    puts "#{result.col_count} columns: #{result.cols}"
+    puts "#{result.row_count} rows"
+    result.cols.each_with_index do |col, index|
+      puts "#{index}: #{col}"
+    end
+    puts "Row sample data:"
+    puts "#{result.sample_row}"
+  end
+end
+
+desc 'Extract specified rows and columns from the file'
+command :extract do |c|
+  c.desc 'Rows to extract'
+  c.arg_name '1,2,10-30|REGEXP'
+  c.flag [:r, :row], :must_match => /\d+(?:,\d+|-\d+|,\/.*\/)*|\/.*\/(?:,\/.*\/|\d+)*/
+
+  c.desc 'Columns to extract'
+  c.arg_name '1,2,10-30'
+  c.flag [:c, :col], :must_match => /\d+(?:,\d+|-\d+)*/
+
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a file to extract data from '-f FILE'" if global_options[:f].nil?
+    help_now! "You need to provide a result file '-o OUT_FILE'" if global_options[:o].nil?
+
+    puts "Extracting ..."
+    extractor = Sycsvpro::Extractor.new(infile: global_options[:f], outfile: global_options[:o],
+                                        rows: options[:r], cols: options[:c])
+    extractor.execute
+    puts "extract done"    
+  end
+end
+
+desc 'Collect values of specified rows and columns from the file and group them in categories'
+command :collect do |c|
+
+  c.desc 'Rows to consider for collection'
+  c.arg_name 'ROW1,ROW2,ROW10-ROW30|REGEXP'
+  c.flag [:r, :row], :must_match => /\d+(?:,\d+|-\d+|,\/.*\/)*|\/.*\/(?:,\/.*\/|\d+)*/
+
+  c.desc 'Columns to collect values from'
+  c.arg_name 'CATEGORY1:ROW1,ROW2,ROW10-ROW30+CATEGORY2:ROW3-ROW9'
+  c.flag [:c, :col], :must_match => /^\w*:\d+(?:,\d+|-\d+|\+\w*:\d+(?:,\d+|-\d+)*)*/
+
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a file to collect data from '-f FILE'" if global_options[:f].nil?
+    help_now! "You need to provide a result file '-o OUT_FILE'" if global_options[:o].nil?
+
+    puts "Collecting ..."
+    collector = Sycsvpro::Collector.new(infile: global_options[:f], outfile: global_options[:o],
+                                        rows: options[:r], cols: options[:c])
+    collector.execute
+    puts "collect done"    
+  end
+end
+
+desc 'Executes the code provided in a file'
+arg_name 'PRO_FILE METHOD'
+command :execute do |c|
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a script FILE and a METHOD to call" if args.size < 2
+    profiler = Sycsvpro::Profiler.new(args[0])
+    profiler.execute(args[1])
+    puts "execute done"
+  end
+end
+
+desc 'Counts the occurences of column values. Uses column values as headings with count as ' +
+     'values. Columns with a condition will be added as new columns and the condition will ' +
+     'be set as column name'
+
+command :count do |c|
+
+  c.desc 'Key column that is assigned the count of column values to'
+  c.arg_name 'KEY_COLUMN'
+  c.flag [:k, :key], :must_match => /^\d+/
+
+  c.desc 'Rows to consider'
+  c.arg_name '1,2,10-30|REGEXP'
+  c.flag [:r, :row], :must_match => /\d+(?:,\d+|-\d+|,\/.*\/)*|\/.*\/(?:,\/.*\/|\d+)*/
+
+  c.desc 'Columns to count where column 2 is counted conditionally'
+  c.arg_name '1,2:<14.2.2014,10-30'
+  c.flag [:c, :col], :must_match => /^\d+(?:,\d+|(?::[<=>]\d+.\d+.\d+|:\d+.\d+.\d+-\d+.\d+.\d+|:\/.*?\/|-\d+)*)*/
+
+  c.desc 'Format of date values'
+  c.arg_name '%d.%m.%Y|%m/%d/%Y|...'
+  c.flag [:df]
+
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a file to count data from '-f FILE'" if global_options[:f].nil?
+    help_now! "You need to provide a result file '-o OUT_FILE'" if global_options[:o].nil?
+
+    counter = Sycsvpro::Counter.new(infile: global_options[:f], outfile: global_options[:o], 
+                                    key: options[:k], rows: options[:r], cols: options[:c],
+                                    df: options[:df])
+    counter.execute
+    puts "count done"
+  end
+end
+
+desc 'Map values in columns to new values'
+arg_name 'MAPPINGS-FILE'
+command :map do |c|
+  c.desc 'Rows to consider'
+  c.arg_name 'ROW1,ROW2,ROW10-ROW30|REGEXP'
+  c.flag [:r, :row], :must_match => /\d+(?:,\d+|-\d+|,\/.*\/)*|\/.*\/(?:,\/.*\/|\d+)*/
+
+  c.desc 'Columns to consider for mapping'
+  c.arg_name 'COL1,COL2,COL10-COL30'
+  c.flag [:c, :col], :must_match => /\d+(?:,\d+|-\d+)*/
+
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a file to map data from '-f FILE'" if global_options[:f].nil?
+    help_now! "You need to provide a result file '-o OUT_FILE'" if global_options[:o].nil?
+    help_now! "You need to provide a mapping file" if args.size == 0
+
+    mapper = Sycsvpro::Mapper.new(infile: global_options[:f], outfile: global_options[:o],
+                                  mapping: args[0], rows: options[:r], cols: options[:c])
+    mapper.execute
+    puts "mapping done"
+  end
+end
+
+desc 'Process math operations on columns'
+command :calc do |c|
+    
+    c.desc 'The first non-empty column is considered the header. '+
+           'If additional columns are created then *,COL1,COL2 will create the additional header '+
+           'columns COL1 and COL2'
+    c.arg_name '*,COL2,COL2'
+    default_value '*'
+    c.flag [:h, :header], :must_match => /\*(?:,\w+)*/
+
+    c.desc 'Columns to consider for calculations'
+    c.arg_name 'ROW1,ROW2-ROW10|REGEXP'
+    c.flag [:r, :row], :must_match => /\d+(?:,\d+|-\d+|,\/.*\/)*|\/.*\/(?:,\/.*\/|\d+)*/
+
+    c.desc 'Column to do calculations on'
+    c.arg_name 'COL1:*2,COL2:-C3,COL3:*2+(4+C5),COL6:NEW_COL=C1+5'
+    c.flag [:c, :col], :must_match => /\d+:(?:[\*\/\+\-]|\w+=[\d|(]*)[\*\/\+\-\dc()]*(?:,\d+:(?:[\*\/\+\-]|\w+=[\d|(]*)[\*\/\+\-\dc()]*)*/
+
+  c.action do |global_options,options,args|
+    help_now! "You need to provide a file to calculate data at '-f FILE'" if global_options[:f].nil?
+    help_now! "You need to provide a result file '-o OUT_FILE'" if global_options[:o].nil?
+    help_now! "You need to provide the column flag" if options[:c].nil?
+
+    calculator = Sycsvpro::Calculator.new(infile: global_options[:f], outfile: global_options[:o],
+                                          header: options[:h], rows: options[:r], cols: options[:c])
+    calculator.execute 
+    puts "calc done"
+  end
+end
+
+pre do |global,command,options,args|
+  # Pre logic here
+  # Return true to proceed; false to abort and not call the
+  # chosen command
+  # Use skips_pre before a command to skip this block
+  # on that command only
+  true
+end
+
+post do |global,command,options,args|
+  # Post logic here
+  # Use skips_post before a command to skip this
+  # block on that command only
+end
+
+on_error do |exception|
+  # Error logic here
+  # return false to skip default error handling
+  true
+end
+
+exit run(ARGV)

data/features/step_definitions/sycsvpro_steps.rb

@@ -0,0 +1,6 @@
+When /^I get help for "([^"]*)"$/ do |app_name|
+  @app_name = app_name
+  step %(I run `#{app_name} help`)
+end
+
+# Add more step definitions here

data/features/support/env.rb

@@ -0,0 +1,15 @@
+require 'aruba/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/features/sycsvpro.feature

@@ -0,0 +1,8 @@
+Feature: My bootstrapped app kinda works
+  In order to get going on coding my awesome app
+  I want to have aruba and cucumber setup
+  So I don't have to do it myself
+
+  Scenario: App just runs
+    When I get help for "sycsvpro"
+    Then the exit status should be 0