suds 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e0fc3f6bfeb9132463f70ae8bfb07e1ec04c0fa4
+  data.tar.gz: 770b6fe86d994af04989b28d7fd22c4b1d335053
+SHA512:
+  metadata.gz: 116913488fb79c62e8f75ba6bd0adc6232ca1124b23bc0b2598b3709c9419f55d831c948bf2b8585f8857228812e95fe3df499cd5f8ed92bc7035781f39281c0
+  data.tar.gz: c8601f3759c37aa3150ed440e9af5bd0d8d37bbfc428441dfa7e66d73b47b06ddd61df653c70722944bb6cf0df1a4ca6be99d0130427fb16398d947ca9917094

data/.coveralls.yml

@@ -0,0 +1 @@
+repo_token: ULz8tz46aXDE4AbHGI2sh3SwgqjJvHLQs

data/.gitignore

@@ -0,0 +1,62 @@
+data/*
+
+### Ruby
+
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+### Mac
+
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear on external disk
+.Spotlight-V100
+.Trashes
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk

data/.rspec

@@ -0,0 +1,3 @@
+#--warnings
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.1.3

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+group :development, :test do
+  gem 'guard-rspec', require: false
+  gem 'terminal-notifier-guard'
+  gem 'coveralls', require: false
+  gem "codeclimate-test-reporter", group: :test, require: nil
+end

data/Gemfile.lock

@@ -0,0 +1,77 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    codeclimate-test-reporter (0.3.0)
+      simplecov (>= 0.7.1, < 1.0.0)
+    coderay (1.1.0)
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    ffi (1.9.3)
+    formatador (0.2.5)
+    guard (2.6.1)
+      formatador (>= 0.2.4)
+      listen (~> 2.7)
+      lumberjack (~> 1.0)
+      pry (>= 0.9.12)
+      thor (>= 0.18.1)
+    guard-rspec (4.2.10)
+      guard (~> 2.1)
+      rspec (>= 2.14, < 4.0)
+    listen (2.7.9)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.7)
+    method_source (0.8.2)
+    mime-types (2.3)
+    multi_json (1.10.1)
+    pry (0.10.0)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.2)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slop (3.5.0)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    terminal-notifier-guard (1.5.3)
+    thor (0.19.1)
+    timers (1.1.0)
+    tins (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  codeclimate-test-reporter
+  coveralls
+  guard-rspec
+  terminal-notifier-guard

data/Guardfile

@@ -0,0 +1,7 @@
+guard :rspec, cmd: "bundle exec rspec"  do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  notification :terminal_notifier
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 HealthWave.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,68 @@
+# suds
+
+Welcome to the suds project. This is very much a work in progress, so please use this at your own risk. I am changing the DSL constantly and rewriting the history.
+
+This project will become stable fairly soon.
+
+Suds is a essentially a tool for working with data files. Suds is broken up into three modular components: **interpreters**, **cleaners**, and **converters**.
+
+**Interpreters** load the data into the `suds` object. This can be in the form of raw text, json, csv, etc. Interpreteres can be chained together, for instance a (fictional) `GoogleDriveInterpreter` can pass it's data to the  `CSVInterpreter`.
+
+**Cleaners** manipulate the data. You can modify/remove rows/columns (in the context of a CSV file).
+
+**Converters** export the file either to disk or to a remote location.
+
+
+## Requirements
+
+Ruby `2.0.x` is tested. If you need support for `1.9.3` use the `1.9.3` branch.
+
+
+## Installation
+
+```bash
+gem install suds
+```
+
+Or if you're using Bundler, include the following line in your Gemfile:
+
+```ruby
+gem 'suds', git: 'git@github.com:HealthWave/suds.git'
+```
+
+## Getting started starting
+
+Initialize your interpreter(s):
+
+
+```ruby
+f_interpreter = FileInterpreter.new('./path/to/my/file')
+csv_interpreter = CSVInterpreter.new(f_interpreter.data)
+```
+
+Create a suds list and add the interpreters to it:
+
+```ruby
+list = Suds.new( csv_interpreter )
+```
+
+Add some cleaners to it:
+
+```ruby
+list.add_cleaner Cleaner.new {|_,v| v.strip! if v } # Inline initialization of a generic cleaner
+# Configuring an existing cleaner
+list.add_cleaner ColumnConverterCleaner.new({
+  company:     :name,
+  nlacno:      :partner_practitioner_id,
+})
+list.add_cleaner ColumnFilterCleaner.new(include_columns: [:name, :partner_practitioner_id, :email])
+list.add_cleaner DowncaseCleaner.new()
+```
+
+Run all the cleaners:
+
+```ruby
+list.clean
+```
+
+

data/lib/suds.rb

@@ -0,0 +1 @@
+require 'suds/suds'

data/lib/suds/all.rb

@@ -0,0 +1,12 @@
+
+require 'suds/suds'
+require 'suds/cleaner/column_converter_cleaner'
+require 'suds/cleaner/column_filter_cleaner'
+require 'suds/cleaner/downcase_cleaner'
+require 'suds/cleaner/regex_cleaner'
+require 'suds/cleaner/whitespace_cleaner'
+
+require 'suds/interpreter/file_interpreter'
+require 'suds/interpreter/csv_interpreter'
+
+require 'suds/converter/json_converter'

data/lib/suds/cleaner.rb

@@ -0,0 +1,32 @@
+
+class Cleaner
+  attr_accessor :data, :action
+
+  def initialize &b
+    block_given? ? @action = b : raise("The generic Cleaner must be provided a block.")
+  end
+
+  def self.clean_array array, &b
+    raise "Please provide either a hash or an array as the main parameter" unless Array === array
+    array.each do |row|
+      clean_hash row, &b
+    end
+    return array
+  end
+
+  def self.clean_hash hash, &b
+    raise "Please provide either a hash or an array as the main parameter" unless Hash === hash
+
+    hash.each do |k,v|
+      b.call(k,v)
+    end
+    return hash
+  end
+
+  def clean data
+    @data = data
+    self.class.clean_array(@data) do |k,v|
+      @action.call(k,v)
+    end
+  end
+end

data/lib/suds/cleaner/column_converter_cleaner.rb

@@ -0,0 +1,26 @@
+require 'suds/cleaner'
+
+class ColumnConverterCleaner < Cleaner
+  def initialize convert_hash, force_strings = false
+    @force_strings = force_strings
+    @convert_hash = convert_hash
+  end
+
+  def clean data
+    data.each do |row|
+      keys = row.keys
+      keys.each do |key|
+        if new_key = @convert_hash[key]
+          old_value = row[key]
+          row.delete key
+          if @force_strings
+            new_key = new_key.to_s
+          else
+            new_key = new_key.to_sym
+          end
+          row[new_key] = old_value
+        end
+      end
+    end
+  end
+end

data/lib/suds/cleaner/column_filter_cleaner.rb

@@ -0,0 +1,26 @@
+require 'suds/cleaner'
+
+class ColumnFilterCleaner < Cleaner
+  attr_accessor :exclude_columns, :include_columns
+  def initialize exclude_columns:[], include_columns:[]
+    @exclude_columns = [exclude_columns].flatten.map(&:to_s)
+    @include_columns = [include_columns].flatten.map(&:to_s)
+
+    raise "You must provide include_columns or exclude_columns." if @exclude_columns.empty? and @include_columns.empty?
+  end
+
+  def clean data
+    if not @include_columns.empty?
+      exclude_columns = data.first.keys.map(&:to_s) - @include_columns
+    else
+      exclude_columns = @exclude_columns
+    end
+
+    data.each do |row|
+      exclude_columns.each do |col|
+        row.delete col
+        row.delete col.to_sym
+      end
+    end
+  end
+end

data/lib/suds/cleaner/downcase_cleaner.rb

@@ -0,0 +1,21 @@
+require 'suds/cleaner'
+
+class DowncaseCleaner < Cleaner
+  def initialize  exclude_columns:[], include_columns:[]
+    @exclude_columns = [exclude_columns].flatten
+    @include_columns = [include_columns].flatten
+  end
+
+  def clean data
+    Cleaner.clean_array(data) do |k,v|
+      next if v.nil?
+      if !@include_columns.empty?
+        v.downcase if @include_columns.include?(k)
+      elsif !@exclude_columns.empty?
+        v.downcase! unless @exclude_columns.include?(k)
+      else
+        v.downcase!
+      end
+    end
+  end
+end

data/lib/suds/cleaner/regex_cleaner.rb

@@ -0,0 +1,41 @@
+require 'suds/cleaner'
+
+class RegexCleaner < Cleaner
+  attr_accessor :regex_map
+  EMAIL_REGEX = /^[a-zA-Z0-9_.+\-]+@[a-zA-Z0-9\-]+\.[a-zA-Z0-9\-.]+$/
+
+  def initialize regex_map, destroy_row: false
+    @regex_map = {}
+    regex_map.each do |column,regex|
+      if Array === column
+        column.each do |col|
+          @regex_map[col] = regex
+        end
+      else
+        @regex_map[column] = regex
+      end
+    end
+    @columns = @regex_map.keys.map(&:to_s)
+    @destroy_row = destroy_row
+  end
+
+  def clean data
+    ret_data = data.select do |row|
+      save_row = true
+      row.keys.each do |key|
+        if @regex_map[key]
+          if !(row[key].to_s =~ @regex_map[key])
+            if @destroy_row
+              save_row = false
+            else
+              row.delete key
+            end
+          end
+        end
+      end
+      save_row # for select
+    end
+    return ret_data
+  end
+
+end

data/lib/suds/cleaner/whitespace_cleaner.rb

@@ -0,0 +1,9 @@
+require 'suds/cleaner'
+
+class WhitespaceCleaner
+  def self.clean data, options={}
+    Cleaner.clean_array(data) do |_,v|
+      v.strip! if v
+    end
+  end
+end

data/lib/suds/converter.rb

@@ -0,0 +1,27 @@
+require 'fileutils'
+
+class Converter
+  attr_accessor :outfile, :converter, :converted_data
+
+  def initialize outfile=nil, &b
+    @outfile = outfile
+    @converter = b
+  end
+
+  def convert data
+    raise "A generic Converter can only convert with a block" unless @converter
+    @converted_data = @converter.call(data)
+  end
+
+  def convert! data
+    raise "Cannot output to file if outfile is not set." unless @outfile
+    fname = File.expand_path(@outfile)
+    dir = File.dirname fname
+
+    if !File.directory?(dir)
+      FileUtils.mkdir_p dir
+    end
+
+    File.open(fname, 'w').write(convert(data))
+  end
+end

data/lib/suds/converter/json_converter.rb

@@ -0,0 +1,8 @@
+require 'json'
+require 'suds/converter'
+
+class JSONConverter < Converter
+  def convert data
+    self.converted_data = data.to_json
+  end
+end

data/lib/suds/interpreter.rb

@@ -0,0 +1,19 @@
+
+class Interpreter
+  attr_accessor :headers, :data
+
+  def initialize
+    @headers = []
+    @data = []
+  end
+
+
+  def interpret
+    raise "No interpretation defined."
+  end
+
+  def data
+    interpret if @data.nil? || @data.empty?
+    @data
+  end
+end

data/lib/suds/interpreter/csv_interpreter.rb

@@ -0,0 +1,28 @@
+require 'csv'
+require 'suds/interpreter'
+
+class CSVInterpreter < Interpreter
+  attr_accessor :filepath
+
+  def initialize raw_data
+    @raw_data = raw_data
+    super()
+  end
+
+
+  def interpret
+    CSV.parse(@raw_data, headers: true, header_converters: :symbol).each do |row|
+      @headers = row.headers if @headers.nil? || @headers.empty?
+      interpret_unit row
+    end
+    @data
+  end
+
+  def interpret_unit unit
+    raise "Headers have not be set." if @headers.empty?
+    raise "Invalid data for current headers." if @headers.size != unit.size
+
+    @data << unit.to_h
+  end
+
+end

data/lib/suds/interpreter/file_interpreter.rb

@@ -0,0 +1,11 @@
+
+class FileInterpreter < Interpreter
+  def initialize filepath
+    # TODO File validation
+    @raw_data = open(filepath, 'r').read
+  end
+
+  def interpret
+    @data = @raw_data
+  end
+end

data/lib/suds/suds.rb

@@ -0,0 +1,46 @@
+require 'suds/interpreter'
+require 'suds/cleaner'
+require 'suds/converter'
+
+class Suds
+
+  attr_accessor :interpreter, :cleaners, :converters, :data
+
+  def initialize interpreter
+    @interpreter = interpreter
+    @converters = []
+    @cleaners = []
+  end
+
+  def add_converter converter
+    raise "#{converter.class} is not a valid Converter." unless converter.is_a?(Converter)
+    @converters << converter
+  end
+
+  def add_cleaner cleaner
+    raise "#{cleaner.class} is not a valid Cleaner." unless cleaner.is_a?(Cleaner)
+    @cleaners << cleaner
+  end
+
+  def clean
+    @data = @cleaners.inject(interpreter.data) do |data,cleaner|
+      data = cleaner.clean data
+    end
+  end
+
+  def convert!
+    @converters.each { |converter| converter.convert! data}
+  end
+
+  def convert
+    @converters.map { |converter| converter.convert data}
+  end
+
+  def raw_data
+    interpreter.data
+  end
+
+  def data
+    @data ||= interpreter.interpret
+  end
+end

data/lib/suds/version.rb

@@ -0,0 +1,4 @@
+
+class Suds
+  VERSION = '0.1.0'
+end

data/spec/lib/suds/all_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'suds/all'
+
+describe "all requires" do
+  it { JSONConverter }
+  it { CSVInterpreter }
+  it { FileInterpreter }
+  it { ColumnConverterCleaner }
+  it { ColumnFilterCleaner }
+  it { DowncaseCleaner }
+  it { RegexCleaner }
+  it { WhitespaceCleaner }
+end

data/spec/lib/suds/cleaner/column_converter_cleaner_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'suds/cleaner/column_converter_cleaner'
+
+describe ColumnConverterCleaner do
+  subject { ColumnConverterCleaner }
+  let(:convert_hash) { {company: :name} }
+
+  describe "#initialize" do
+    it "takes a hash of strings mapped to strings" do
+      subject.new convert_hash
+    end
+  end
+
+  describe "#clean" do
+    let(:data) { [{ company: "uri gorelik" }] }
+    let(:cleaner) { cleaner = subject.new convert_hash, false }
+    it "converts the keys of a hash" do
+      results = cleaner.clean data
+      expect(results.first.keys - convert_hash.values).to eq []
+    end
+
+    it 'retains the original key type' do
+      results = cleaner.clean data
+      expect(results.first.keys.first).to be_a(Symbol)
+      results = cleaner.clean([{ "company" => "test"}])
+      expect(results.first.keys.first).to be_a(String)
+    end
+  end
+end

data/spec/lib/suds/cleaner/column_filter_cleaner_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+require 'suds/cleaner/column_filter_cleaner'
+
+describe ColumnFilterCleaner do
+  subject { ColumnFilterCleaner }
+
+  describe '#initialize' do
+
+    it 'raises an error if both exclude and include columns are not present or empty' do
+      error_reg = /You must provide include_columns or exclude_columns./
+      expect{ subject.new }.to raise_error(error_reg)
+      expect{ subject.new include_columns: [] }.to raise_error(error_reg)
+      expect{ subject.new exclude_columns: [] }.to raise_error(error_reg)
+      expect{ subject.new include_columns: [], exclude_columns: [] }.to raise_error(error_reg)
+    end
+
+    it 'makes include columns publicly available' do
+      cleaner = subject.new include_columns: [{}]
+      expect(cleaner).to respond_to(:include_columns)
+    end
+
+    it 'makes exclude columns publicly available' do
+      cleaner = subject.new exclude_columns: [{}]
+      expect(cleaner).to respond_to(:exclude_columns)
+    end
+  end
+
+  describe '#clean' do
+    let(:data) { [{row1: 'a', row2: 'b'}] }
+    let(:original_data) { data.clone }
+
+    it 'excludes certain columns' do
+      cleaner = subject.new exclude_columns: :row2
+      result = cleaner.clean data
+      expect(result.first.keys).to include(:row1)
+      expect(result.first.keys).to_not include(:row2)
+    end
+
+    it 'only includes the listed columns' do
+      cleaner = subject.new include_columns: :row2
+      result = cleaner.clean data
+      expect(result.first.keys).to_not include(:row1)
+      expect(result.first.keys).to include(:row2)
+    end
+
+    it 'accepts both strings and symbol arrays' do
+      cleaner = subject.new include_columns: "row2"
+      result = cleaner.clean data
+      expect(result.first.keys).to_not include(:row1)
+      expect(result.first.keys).to include(:row2)
+
+    end
+  end
+
+
+end

data/spec/lib/suds/cleaner/downcase_cleaner_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'cleaner/downcase_cleaner'
+
+describe DowncaseCleaner do
+  include_context "shared cleaner"
+  subject { DowncaseCleaner }
+
+  let(:data) { {a: "   Test", b: "Double test   "} }
+  # let!(:original_value) { data.first.last.clone }
+
+  # it "downcases an array of objects" do
+  #expect(   modified_value ).to match /[^A-Z]/
+  # end
+
+end

data/spec/lib/suds/cleaner/regex_cleaner_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'suds/cleaner/regex_cleaner'
+
+describe RegexCleaner do
+  subject { RegexCleaner }
+
+  context 'constants' do
+    it "has a regex constant for emails" do
+      expect(subject).to be_const_defined(:EMAIL_REGEX)
+    end
+  end
+
+
+  describe "#initialize" do
+    it 'accepts a hash that maps column names to regexes' do
+      subject.new({name: /uri/})
+    end
+
+    it 'accepts arrays mapped to regular expressions' do
+      cleaner = subject.new({[:first_name, :last_name] => /uri/})
+      expect(cleaner.instance_variable_get(:@columns)).to eq(%w[first_name last_name])
+    end
+
+    it 'accepts a hash that decides whether or not to delete the entire row' do
+      subject.new({name: /uri/}, destroy_row: true)
+    end
+  end
+
+  describe "#clean" do
+    let(:bad_data) { [{email: "uri"}] }
+    let(:good_data) { [{email: "uri@healthwave.co"}] }
+    let(:cleaner) { subject.new({email: subject::EMAIL_REGEX}) }
+
+
+    it 'keeps columns if they match the regex' do
+      results = cleaner.clean good_data
+      expect(results.first.keys).to include(:email)
+    end
+
+    it 'removes columns if they do not match the regex' do
+      results = cleaner.clean bad_data
+      expect(results.first.keys).to_not include(:email)
+    end
+
+    it "destroys rows if the regex doesn't match" do
+      cleaner =  subject.new({email: subject::EMAIL_REGEX}, destroy_row: true)
+      results = cleaner.clean bad_data
+      expect(results).to be_empty
+    end
+
+  end
+end

data/spec/lib/suds/cleaner/whitespace_cleaner_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'cleaner/whitespace_cleaner'
+
+describe WhitespaceCleaner do
+  include_context "shared cleaner"
+  subject { WhitespaceCleaner }
+
+  let(:data) { {a: "   Test", b: "Double test   "} }
+  # let!(:original_value) { data.first.last.clone }
+
+  # it "downcases an array of objects" do
+  #expect(   modified_value ).to match /^[^\s].+[^\s]$/
+  # end
+
+end

data/spec/lib/suds/cleaner_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'cleaner'
+
+describe Cleaner do
+  subject { Cleaner.new(){} }
+
+  it { expect(Cleaner).to respond_to(:clean_array) }
+  it { expect(Cleaner).to respond_to(:clean_hash) }
+
+  describe "#clean" do
+    let(:data) { [{row1: "TEST"}] }
+
+    it "runs the provided block" do
+      cleaner = Cleaner.new() do |_,v|
+        v.downcase!
+      end
+
+      result = cleaner.clean data
+      expect( result.first.to_a.flatten.last ).to match(/[^A-Z]/)
+    end
+  end
+
+  context "failure" do
+    it "raises an error when an non hash or array is passed in as the data" do
+      expect {Cleaner.clean_array double("Fake")}.to raise_error(/Please provide either a hash or an array as the main parameter/)
+    end
+  end
+end

data/spec/lib/suds/converter/json_converter_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'converter/json_converter'
+
+describe JSONConverter do
+  subject { JSONConverter }
+  let(:unconverted_data) { [{row1: 1, row2:2}] * 5 }
+  it "converts an array to json" do
+    converter = subject.new(unconverted_data)
+    expect( converter.convert(unconverted_data).class ).to be String
+    expect( JSON.parse(converter.converted_data, symbolize_names: true) ).to eq unconverted_data
+  end
+end

data/spec/lib/suds/converter_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'converter'
+
+describe Converter do
+  subject { Converter }
+  
+  describe "#convert!" do
+    it "it raises an error if no output file was specified" do
+      expect { Converter.new.convert!(double) }.to raise_error(/Cannot output to file if outfile is not set./)
+    end
+  end
+end

data/spec/lib/suds/interpreter/csv_interpreter_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+require 'interpreter/csv_interpreter'
+
+describe CSVInterpreter do
+  subject {CSVInterpreter.new("path")}
+
+  describe "#interpret_unit" do
+    let(:headers) { %w{foo} }
+    let(:row1) { {"foo" => "bar"} }
+    let(:csvint) do
+      csvint = CSVInterpreter.new("path")
+      csvint.headers = headers
+      csvint.interpret_unit row1
+      csvint
+    end
+
+    context 'success' do
+      subject { csvint.data }
+
+      it {expect( subject.size ).to eq 1 }
+
+      context "data" do
+        subject { csvint.data.first }
+        it {expect( subject.keys ).to include(*headers) }
+        it {expect( subject.values ).to include(*row1.values) }
+      end
+    end
+
+    context 'failure' do
+      subject { csvint }
+      let(:csvint) { CSVInterpreter.new("path") }
+
+      it "raises an error if the headers aren't set" do
+        expect{subject.interpret_unit %w{too many values}}.to raise_error(/Headers have not be set./)
+      end
+
+      it "raises an error if the header doesn't match the data" do
+        subject.headers = ["row1"]
+        expect{subject.interpret_unit %w{too many values}}.to raise_error(/Invalid data for current headers./)
+      end
+    end
+
+
+  end
+end

data/spec/lib/suds/interpreter_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+require 'interpreter'
+
+describe Interpreter do
+  subject { Interpreter.new }
+
+  context "interface" do
+    it { should respond_to :data }
+    it { should respond_to :headers }
+  end
+end

data/spec/lib/suds/suds_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+require 'suds'
+require 'cleaner'
+require 'converter'
+require 'interpreter'
+
+describe Suds do
+  subject { Suds }
+
+  describe '#initialize' do
+    subject { Suds.new double("Interpreter") }
+    it { should respond_to(:data) }
+  end
+
+  context 'plugins' do
+    let(:list) { subject.new double("Interpreter") }
+
+    describe '#add_cleaner' do
+      it "adds a cleaner" do
+        cleaner = Cleaner.new(){}
+        list.add_cleaner cleaner
+        expect(list.cleaners ).to_not be_empty
+      end
+
+      it 'raises an error if the added cleaner is not a Cleaner' do
+        cleaner = double
+        expect { list.add_cleaner cleaner }.to raise_error(Regexp.new("#{cleaner.class} is not a valid Cleaner."))
+      end
+    end
+
+    describe '#add_converter' do
+      it 'adds a converter' do
+        list.add_converter Converter.new([])
+        expect( list.converters ).to_not be_empty
+      end
+      it 'raises an error if the added converter is not a Converter' do
+        converter = double
+        expect { list.add_converter converter }.to raise_error(Regexp.new("#{converter.class} is not a valid Converter."))
+      end
+    end
+
+  end
+
+  context 'data manipulation' do
+    let(:data) { {row1: :a, row2: :b} }
+
+    let(:special_converter) do
+      converter = double("SpecialConverter")
+      allow(converter).to receive(:is_a?).with(Converter).and_return(true)
+      allow(converter).to receive(:convert)
+      allow(converter).to receive(:convert!)
+      allow(converter).to receive(:data).and_return([{a: '1'}])
+      converter
+    end
+
+    let(:special_cleaner) do
+      cleaner = double("SpecialCleaner")
+      allow(cleaner).to receive(:is_a?).with(Cleaner).and_return(true)
+      allow(cleaner).to receive(:clean).and_return({z: "9"})
+      cleaner
+    end
+
+    let(:special_interpreter) do
+      interpreter = double("SpecialInterpreter")
+      allow(interpreter).to receive(:is_a?).with(Interpreter).and_return(true)
+      allow(interpreter).to receive(:interpret)
+      allow(interpreter).to receive(:data).and_return(data)
+      interpreter
+    end
+
+    describe '#clean' do
+      let(:list) { Suds.new(special_interpreter) }
+      before do
+        list.add_cleaner(special_cleaner)
+      end
+
+      it 'cleans the data' do
+        expect( special_cleaner ).to receive(:clean)
+        list.clean
+        expect( list.interpreter.data ).to_not eq( list.data )
+      end
+
+    end
+
+    describe '#convert' do
+      let(:list) { Suds.new(special_interpreter) }
+      before do
+        list.add_converter special_converter
+      end
+
+      it 'converts the data into a portable type' do
+        expect( special_converter ).to receive(:convert)
+        list.convert
+      end
+
+      it 'can convert without cleaning first' do
+        allow( special_interpreter ).to receive(:interpret).and_return([])
+        list.convert
+        expect( list.data ).to_not be_nil
+      end
+
+      it 'returns a list of strings' do
+        allow( special_converter ).to receive(:convert).and_return("converted data")
+        convert = list.convert
+        expect( convert ).to be_an( Array )
+        expect( convert.first ).to be_a( String )
+      end
+    end
+
+    describe '#convert!' do
+      it 'writes out the data to a file' do
+        expect( special_converter ).to receive(:convert!)
+
+        list = Suds.new(special_interpreter)
+        list.add_converter special_converter
+        list.convert!
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,87 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+
+$LOAD_PATH.unshift(File.dirname(__FILE__), *%w{.. lib/suds})
+require 'coveralls'
+require "codeclimate-test-reporter"
+CodeClimate::TestReporter.start
+Coveralls.wear!
+
+Dir["./spec/support/**/*.rb"].sort.each { |f| require f}
+
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+=end
+end

data/spec/support/cleaner_context.rb

@@ -0,0 +1,5 @@
+shared_context "shared cleaner" do
+  let(:sample_array) { [data] }
+  let(:result) { subject.clean sample_array }
+  let(:modified_value) { result.first.first.last }
+end

data/suds.gemspec

@@ -0,0 +1,15 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'suds/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'suds'
+  s.version     = Suds::VERSION
+  s.licenses    = ['MIT']
+  s.summary     = "Interpret, clean, and convert lists."
+  s.description = "Suds is a versatile list manipulation library/dsl. It's meant to interpret various difference formats, apply mutations, and then export to various formats."
+  s.authors     = ["Uri Gorelik"]
+  s.email       = 'uri@healthwave.co'
+  s.files       = `git ls-files`.split("\n")
+  s.homepage    = 'https://healthwave.co/'
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: suds
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Uri Gorelik
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-27 00:00:00.000000000 Z
+dependencies: []
+description: Suds is a versatile list manipulation library/dsl. It's meant to interpret
+  various difference formats, apply mutations, and then export to various formats.
+email: uri@healthwave.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE.txt
+- README.md
+- lib/suds.rb
+- lib/suds/all.rb
+- lib/suds/cleaner.rb
+- lib/suds/cleaner/column_converter_cleaner.rb
+- lib/suds/cleaner/column_filter_cleaner.rb
+- lib/suds/cleaner/downcase_cleaner.rb
+- lib/suds/cleaner/regex_cleaner.rb
+- lib/suds/cleaner/whitespace_cleaner.rb
+- lib/suds/converter.rb
+- lib/suds/converter/json_converter.rb
+- lib/suds/interpreter.rb
+- lib/suds/interpreter/csv_interpreter.rb
+- lib/suds/interpreter/file_interpreter.rb
+- lib/suds/suds.rb
+- lib/suds/version.rb
+- spec/lib/suds/all_spec.rb
+- spec/lib/suds/cleaner/column_converter_cleaner_spec.rb
+- spec/lib/suds/cleaner/column_filter_cleaner_spec.rb
+- spec/lib/suds/cleaner/downcase_cleaner_spec.rb
+- spec/lib/suds/cleaner/regex_cleaner_spec.rb
+- spec/lib/suds/cleaner/whitespace_cleaner_spec.rb
+- spec/lib/suds/cleaner_spec.rb
+- spec/lib/suds/converter/json_converter_spec.rb
+- spec/lib/suds/converter_spec.rb
+- spec/lib/suds/interpreter/csv_interpreter_spec.rb
+- spec/lib/suds/interpreter_spec.rb
+- spec/lib/suds/suds_spec.rb
+- spec/spec_helper.rb
+- spec/support/cleaner_context.rb
+- suds.gemspec
+homepage: https://healthwave.co/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Interpret, clean, and convert lists.
+test_files: []