csv_piper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea1cbd12bae76e95d785d1c5b0f1e5b5226d6e15
+  data.tar.gz: f93996e0438a347161ced80ad8eee12875099029
+SHA512:
+  metadata.gz: b1812adb041655a4952e44ddba1b0cd8d90c97aeb1bc83dc09547a0805930da9dcaf2b02cab390e2adac9e0f823043ff5cf138972b2973cf6362a8560bf1f2fc
+  data.tar.gz: 220b7d3a231c03ba2c5196eaf590d9ea11f18b62ff3b2d1b6f1bfd192cd1c05f106db2d56b6614a0d074740ce49da2ebb7327507f42b532fa825aa492b2c601b

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/vendor
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in csv_piper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Jarrod
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,174 @@
+# CsvPiper
+
+A simple wrapper to create a pipeline style csv processor that makes your transforms easily testable.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'csv_piper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install csv_piper
+
+## Usage
+
+CsvPiper handles CSV reading row by row passing each row through a series of processors.
+
+#### Requirements
+* Currently source csv must have headers
+
+#### Basic Usage
+```ruby
+File.open("my/file/path", "r") do |io_stream|
+    CsvPiper::Builder.new.from(io_stream).with_processors([your_processors]).build.process
+end
+```
+`io_stream` can be any subclass of [IO](ruby-doc.org/core/IO.html).
+
+`build` returns an instance of `CsvPiper::Piper` but you will only need this object to call `process` unless you are utilising the `requires_headers()` method _(see builder options below)_
+
+#### Basic Usage with Processors
+_Extracted from `spec/end_to_end_spec.rb`_
+```ruby
+# Build some processors beforehand so we can access them later
+output_collector = ProcessedEquationCollector.new
+error_collector = CsvPiper::Processors::ErrorCollector.new
+
+# Open the csv file to get our io source
+File.open(File.join(File.dirname(__FILE__),"/data/csv_1.csv")) do |file|
+
+    # Build piper
+    csv_piper = CsvPiper::Builder.new.from(file)
+        .requires_headers(required_headers)
+        .with_processors([
+          BuildEquation.new, EvaluateEquation.new, output_collector, error_collector
+        ])
+        .build
+
+    # Process csv
+    csv_piper.process if csv_piper.has_required_headers?
+end
+
+# Grab some output we wanted to collect (You don't have to do this, espicially when processing lots of data)
+output = output_collector.output
+errors = error_collector.errors
+
+
+class BuildEquation
+    def process(source, transformed, errors)
+        transformed[:equation] = [ source['Input 1'], source['Process'], source['Input 2'], '==', source['Result'] ].join(' ')
+        [transformed, errors]
+    end
+end
+
+class EvaluateEquation
+    def process(source, transformed, errors)
+        begin
+            transformed[:valid] = eval(transformed[:equation]) == true
+        rescue Exception
+            errors.add(:msg, transformed[:equation] + ' is not a valid equation')
+        end
+        [transformed, errors]
+    end
+end
+
+class ProcessedEquationCollector
+    attr_reader :output
+    def initialize
+        @output = []
+    end
+
+    def process(source, transformed, errors)
+        @output << {row: errors.row_index}.merge(transformed) if errors.empty?
+        [transformed, errors]
+    end
+end
+```
+
+#### Processors
+Each processor can do whatever it wants, transformation, logging, saving to a database etc.
+
+Here is an example of a processor that passes the values from the csv straight along to the transformed output:
+```ruby
+class PassThrough
+  def process(source, transformed, errors)
+    [transformed.merge(source), errors]
+  end
+end
+```
+
+* `source` is a frozen hash representing the row data out of the csv (with headers as keys)
+* `transformed` is whatever has been passed on by the previous processor. The first processor will receive an empty hash.
+* `errors` is an instance of `CsvPiper::Errors::Row`
+
+_Return value_ is what will be passed into _transformed_ and _errors_ of the next processor
+
+#### Pre-Processors
+
+Pre-processors work the same as processors except that their purpose is to modify the source row data that will be passed into all processors. It's useful for doing things like converting strings to primitives, removing columns etc.
+
+They are also allowed to add errors against the row.
+
+Here is an example of a pre-processor that converts all values to uppercase:
+```ruby
+class UpCase
+  def process(source, errors)
+    transformed = source.each_with_object({}) { |(key, value), memo| memo[key] = value.upcase }
+    [transformed, errors]
+  end
+end
+```
+* `source` is a hash representing the row data out of the csv which may have been modified by a previous pre-processor
+* `errors` is an instance of `CsvPiper::Errors::Row`
+
+_Return value_ is what will be passed into _source_ and _errors_ of the next pre-processor (and processors). Final pre-processor value of _source_ will be passed to each processor as a frozen hash. Final pre-processor value of _errors_ will be passed to the first processor.
+
+#### Row Errors
+This object is passed into each processor (which must pass it on) and is used to accumulate any and all errors for the particular row. You can access the row number through `row_index`.
+
+Add errors using `errors.add(error_key, error)`.
+
+#### Builder Options
+All builder options utilise the _fluent interface pattern_ and should be followed by a call to `build` to get the piper instance and then `process` to process the csv.
+
+Eg. `CsvPiper::Builder.new.from(io).with_processors(processors).build.process`
+
+* `from(io_stream)`: Specifies the **open** io stream to read csv data from
+* `with_pre_processors(pre_processors)`: Takes an array of pre-processors which will transform each row before it is handled by processors
+* `with_processors(processors)`: Takes an array of processors which do all the interesting domain based work
+* `with_csv_options(options)`: Takes an options hash which is passed to `CSV.new` to set any options on the CSV library
+* `requires_headers(headers)`: Takes an array of strings representing the headers that must be present in the CSV. If this build option is used and `process` is called on a io source missing a header then a exception is thrown. Before calling `process` you should make use of the `has_required_headers?` check and then retrieve the missing headers through `missing_headers` if necessary.
+
+## Pre-made Processors
+Over time we will collect a bunch of general purpose processors that anyone can use. They can be found in the `lib/processors` folder but here are a couple:
+
+* `OutputCollector`: Collects the transformed object of every row that is passed through it
+* `ErrorCollector`: Collects the `RowError` object of every row that is passed through it
+
+By using `OutputCollector` and to a lesser extent `ErrorCollector` you will start to build up objects in memory. For very large csv files you might not want to use these convenience processors and rather create a new processor that does whatever you need with the row (Ie. log, write to db) which will then be discarded rather than collected.
+
+Require them explicitly if you want to use them.
+
+Eg. `require 'csv_piper/processors/output_collector'`
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/csv_piper.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "csv_piper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install

data/csv_piper.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'csv_piper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "csv_piper"
+  spec.version       = CsvPiper::VERSION
+  spec.authors       = ["Jarrod Sibbison"]
+  spec.email         = [""]
+
+  spec.summary       = %q{CSV processing pipeline}
+  spec.description   = %q{Simple wrapper to process csv's with a pipeline of testable processors.}
+  spec.homepage      = "https://github.com/jazzarati/csv_piper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3"
+end

data/lib/csv_piper/builder.rb

@@ -0,0 +1,40 @@
+module CsvPiper
+  class Builder
+    def initialize
+      @pre_processors = []
+      @processors = []
+    end
+
+    def from(io_stream)
+      @io = io_stream
+      self
+    end
+
+    def with_pre_processors(pre_processors)
+      @pre_processors += pre_processors
+      self
+    end
+
+    def with_processors(processors)
+      @processors += processors
+      self
+    end
+
+    def with_csv_options(options)
+      @csv_options = options
+      self
+    end
+
+    def requires_headers(headers)
+      @required_headers = headers
+      self
+    end
+
+    def build
+      build_options = { io_stream: @io, pre_processors: @pre_processors, processors: @processors }
+      build_options[:csv_options] = @csv_options if @csv_options
+      build_options[:required_headers] = @required_headers if @required_headers
+      Piper.new(build_options)
+    end
+  end
+end

data/lib/csv_piper/errors/row.rb

@@ -0,0 +1,19 @@
+module CsvPiper
+  module Errors
+    class Row
+      extend Forwardable
+      delegate :empty? => :errors
+
+      attr_reader :row_index, :errors
+
+      def initialize(row_index)
+        @row_index = row_index
+        @errors = Hash.new { [].freeze }
+      end
+
+      def add(key, error)
+        @errors[key] += [error]
+      end
+    end
+  end
+end

data/lib/csv_piper/piper.rb

@@ -0,0 +1,73 @@
+module CsvPiper
+  class Piper
+    HEADER_LINE_INDEX = 1
+    FIRST_DATA_LINE_INDEX = 2
+    CSV_HEADER_OPTIONS = { headers: true, return_headers: true, skip_blanks: true, skip_lines: /^(\s*,)*$/ }
+
+    def initialize(io_stream:, pre_processors: [], processors: [], csv_options: {}, required_headers: [])
+      @pre_processors = [PreProcessors::RemoveExtraColumns.new] + pre_processors
+      @processors = processors
+      @required_headers = required_headers
+      @csv_options = csv_options.merge(CSV_HEADER_OPTIONS)
+      @csv_options = @csv_options.merge(skip_lines: "^(\s*#{@csv_options[:col_sep]})*$") if @csv_options[:col_sep]
+      @io = io_stream
+    end
+
+    def has_required_headers?
+      missing_headers.empty?
+    end
+
+    def missing_headers
+      headers = csv.headers
+      required_headers.reject { |header| headers.include?(header) }
+    end
+
+    def process
+      validate_process_configuration!
+      validate_headers!
+
+      process_csv_body
+
+      self
+    end
+
+    private
+
+    attr_reader :io, :pre_processors, :processors, :required_headers, :csv_options
+
+    def process_csv_body
+      csv.each.with_index(FIRST_DATA_LINE_INDEX) do |row, index|
+        processed_data, row_errors = process_row(index, row.to_hash)
+      end
+    end
+
+    def process_row(row_index, row)
+      pre_processed_row, row_errors = pre_processors.reduce([row, Errors::Row.new(row_index)]) do |memo, processor|
+        processor.process(*memo)
+      end
+
+      frozen_row = pre_processed_row.freeze
+
+      processed_data = {}
+      processed_data, row_errors = processors.reduce([processed_data, row_errors]) do |memo, processor|
+        processor.process(frozen_row, *memo)
+      end
+    end
+
+    def csv
+      @csv ||= begin
+        csv_object = CSV.new(io, csv_options)
+        csv_object.readline # Read headers through
+        csv_object
+      end
+    end
+
+    def validate_process_configuration!
+      raise 'Requires an IO object to process from' if io.nil?
+    end
+
+    def validate_headers!
+      raise "Missing required headers: #{missing_headers.join(', ')}" unless has_required_headers?
+    end
+  end
+end

data/lib/csv_piper/pre_processors/remove_extra_columns.rb

@@ -0,0 +1,10 @@
+module CsvPiper
+  module PreProcessors
+    class RemoveExtraColumns
+      def process(origin, errors)
+        modified_origin = origin.reject { |key, value| key.nil? || key.empty? }
+        [modified_origin, errors]
+      end
+    end
+  end
+end

data/lib/csv_piper/processors/error_collector.rb

@@ -0,0 +1,15 @@
+module CsvPiper
+  module Processors
+    class ErrorCollector
+      attr_reader :errors
+      def initialize
+        @errors = {}
+      end
+
+      def process(source, transformed, errors)
+        @errors[errors.row_index] = errors unless errors.empty?
+        [transformed, errors]
+      end
+    end
+  end
+end

data/lib/csv_piper/processors/output_collector.rb

@@ -0,0 +1,16 @@
+module CsvPiper
+  module Processors
+    class OutputCollector
+      attr_reader :output
+      def initialize(collect_when_invalid: true)
+        @output = []
+        @collect_when_invalid = collect_when_invalid
+      end
+
+      def process(source, transformed, errors)
+        @output << transformed if @collect_when_invalid || errors.empty?
+        [transformed, errors]
+      end
+    end
+  end
+end

data/lib/csv_piper/version.rb

@@ -0,0 +1,3 @@
+module CsvPiper
+  VERSION = "0.1.0"
+end

data/lib/csv_piper.rb

@@ -0,0 +1,6 @@
+require "csv_piper/version"
+require "csv_piper/piper"
+require "csv_piper/builder"
+require "csv_piper/errors/row"
+require "csv_piper/pre_processors/remove_extra_columns"
+require "csv"

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: csv_piper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jarrod Sibbison
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+description: Simple wrapper to process csv's with a pipeline of testable processors.
+email:
+- ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- csv_piper.gemspec
+- lib/csv_piper.rb
+- lib/csv_piper/builder.rb
+- lib/csv_piper/errors/row.rb
+- lib/csv_piper/piper.rb
+- lib/csv_piper/pre_processors/remove_extra_columns.rb
+- lib/csv_piper/processors/error_collector.rb
+- lib/csv_piper/processors/output_collector.rb
+- lib/csv_piper/version.rb
+homepage: https://github.com/jazzarati/csv_piper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: CSV processing pipeline
+test_files: []