conformist 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in conformist.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Tate Johnson <tate@tatey.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,126 @@
+# Conformist 
+
+Conformist lets you bend CSVs to your will. Let multiple, different input files conform to a single interface without rewriting your parser each time.
+
+Motivation for this project came from the desire to simplify importing data from various government organisations into [Antenna Mate](http://antennamate.com). The data from each government was similar, but had completely different formatting. Some pieces of data needed preprocessing while others simply needed to be concatenated together. I did not want to write a new parser for each new government organisation. Instead, I created Conformist.
+
+## Installation
+
+Conformist is available from rubygems.org.
+
+``` sh
+$ gem install conformist
+```
+
+## Usage
+
+You create a Ruby class, mix-in `Conformist::Base` and declare how an input file should map to a single interface.
+
+``` ruby
+require 'conformist'
+
+class Citizen1
+  include Conformist::Base
+    
+  column :name, 0, 1
+  column :address, 2 do |value|
+    value.upcase
+  end
+end
+
+class Citizen2
+  include Conformist::Base
+  
+  column :name, 0
+  column :address, 1
+end
+```
+
+Load up your definitions and enumerate over each row in the input file.
+
+``` ruby
+Conformist.foreach Citizen1.load('citizens1.csv'), Citizen2.load('citizens2.csv') do |row|
+  Citizen.create! row
+end
+```
+
+Each object passed into the block will be a hash. Perfect for passing to a model to save into a datastore (Or whatever tickles your fancy).
+
+### Option
+
+Conformist uses CSV (Formally FasterCSV) to perform the heavy lifting. You can declare options that should be passed to CSV at runtime. It is safe to call `option` multiple times.
+
+``` ruby
+option :col_sep => ','
+option :quote_char => '"'
+```
+
+### One Column
+
+Maps the first column in the input file to the `:first_name` key-value pair. Indexing starts at zero.
+
+``` ruby
+column :first_name, 0
+```
+
+### Many Columns
+
+Maps the first and second columns in the input file to the `:name` key-value pair and implicitly concatenates the values.
+
+``` ruby
+column :name, 0, 1
+```
+
+Indexing is completely arbitrary and you can map any combination.
+
+``` ruby
+column :name_and_address, 0, 2
+```
+
+### Preprocessing
+
+Sometimes you will want to manipulate values before they're conformed. You can pass a block and get access to the values. The return value of the expression becomes the conformed output.
+
+``` ruby
+column :name, 0, 1 do |values|
+  values.map { |v| v.upcase } * ' '
+end
+```
+
+Works with one column too. Instead of getting an array of strings, you get one string.
+
+``` ruby
+column :first_name, 0 do |value|
+  value.upcase
+end
+```
+
+### Virtual Columns
+
+Declare a column you want included in the conformed output that is not based on the input file. This is useful when you need to set values based on the conformist definition. 
+
+``` ruby
+column :day do 
+  1
+end
+```
+
+## Compatibility
+
+* 1.9.2-p180
+* 1.8.7-p334
+* jruby-1.6.1
+
+## Contributing
+
+Patches welcome! 
+
+1. Fork the repository
+2. Create a topic branch
+3. Write tests and make changes
+4. Make sure the tests pass by running `rake`
+5. Push and send a pull request on GitHub
+
+## Copyright
+
+Copyright © 2011 Tate Johnson. Conformist is released under the MIT license. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,23 @@
+begin
+  require 'rubygems'
+  require 'bundler'
+rescue LoadError
+  raise 'Could not load the bundler gem. Install it with `gem install bundler`.'
+end
+
+begin
+  Bundler.setup
+rescue Bundler::GemNotFound
+  raise RuntimeError, "Bundler couldn't find some gems." +
+    "Did you run `bundle install`?"
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/*_test.rb'
+end
+
+task :default => :test

data/conformist.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "conformist/version"
+
+Gem::Specification.new do |s|
+  s.name        = "conformist"
+  s.version     = Conformist::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Tate Johnson"]
+  s.email       = ["tate@tatey.com"]
+  s.homepage    = "https://github.com/tatey/conformist"
+  s.summary     = %q{Let multiple, different input files conform to a single interface.}
+  s.description = %q{Conformist lets you bend CSVs to your will. Let multiple, different input files conform to a single interface without rewriting your parser each time.}
+
+  s.rubyforge_project = "conformist"
+
+  s.required_ruby_version = '>= 1.8.7'
+  
+  s.add_development_dependency 'rake', '~> 0.8.7'
+  s.add_development_dependency 'minitest', '~> 2.1.0' if RUBY_VERSION < '1.9.0'
+  s.add_dependency 'fastercsv', "~> 1.5.4" if RUBY_VERSION < '1.9.0'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/conformist.rb

@@ -0,0 +1,26 @@
+if RUBY_VERSION >= '1.9.0'
+  require 'csv'
+else
+  require 'fastercsv'
+end
+
+require 'conformist/base'
+require 'conformist/column'
+require 'conformist/row'
+
+module Conformist
+  CSV = RUBY_VERSION >= '1.9.0' ? ::CSV : FasterCSV
+  
+  # Enumerate over each row from multiple input files.
+  #
+  # Example:
+  #
+  #   Conformist::Base.foreach Input1.load('input.csv'), Input2.load('input.csv') do |row|
+  #     Model.create! row
+  #   end
+  #
+  # Returns nothing.
+  def self.foreach *bases, &block
+    bases.each { |base| base.foreach(&block) }
+  end
+end

data/lib/conformist/base.rb

@@ -0,0 +1,49 @@
+module Conformist
+  module Base
+    def self.included base
+      base.class_eval do
+        extend ClassMethods
+        
+        attr_accessor :path
+      end
+    end
+
+    # Enumerate over each row in the input file.
+    #
+    # Example:
+    #
+    #   input1 = Input1.load 'input1.csv'
+    #   input1.foreach do |row|
+    #     Model.create! row
+    #   end
+    #
+    # Returns nothing.
+    def foreach &block
+      CSV.foreach(path, self.class.options) do |row| 
+        yield Row.new(self.class.columns, row).to_hash
+      end
+    end
+    
+    module ClassMethods      
+      def column name, *indexes, &preprocessor
+        columns << Column.new(name, *indexes, &preprocessor)
+      end
+
+      def columns
+        @columns ||= []
+      end
+
+      def option value
+        options.merge! value
+      end
+
+      def options
+        @options ||= {}
+      end
+            
+      def load path
+        new.tap { |object| object.path = path }
+      end
+    end
+  end
+end

data/lib/conformist/column.rb

@@ -0,0 +1,30 @@
+module Conformist
+  class Column
+    attr_reader :name, :indexes, :preprocessor
+        
+    def initialize name, *indexes, &preprocessor
+      @name, @indexes, @preprocessor = name, indexes, preprocessor
+    end
+    
+    # Map column(s) into a single value, strips whitespace and performs
+    # any preprocessing. Takes a +row+ argument that should behave as an
+    # array-like object.
+    #
+    # Example:
+    #
+    #   row    = ['Hello', 'World']
+    #   column = Column.new :first, 0
+    #   column.value_in row # => 'Hello'
+    #
+    # Returns preprocessed value.
+    def values_in row      
+      values = indexes.map { |index| (row[index] || String.new).strip }
+      values = values.first if values.size == 1
+      if preprocessor.respond_to? :call
+        preprocessor.call values
+      else
+        values
+      end
+    end
+  end
+end

data/lib/conformist/row.rb

@@ -0,0 +1,26 @@
+module Conformist
+  class Row
+    attr_reader :columns, :row
+    
+    def initialize columns, row
+      @columns, @row = columns, row
+    end
+    
+    # Extracts key-value pairs for each column in the row. Will implicitly
+    # join an array into a string.
+    #
+    # Example:
+    #
+    #   row = Row.new columns, csv_row
+    #   row.to_hash # => {:first => 'Hello', :second => 'World'}
+    #
+    # Returns conformed hash.
+    def to_hash
+      columns.inject({}) do |attributes, column|
+        values = column.values_in row
+        values = values.join if values.respond_to? :join
+        attributes.merge column.name => values
+      end
+    end
+  end
+end

data/lib/conformist/version.rb

@@ -0,0 +1,3 @@
+module Conformist
+  VERSION = '0.0.1'
+end

data/test/conformist/base_test.rb

@@ -0,0 +1,39 @@
+require 'helper'
+
+class BaseTest < MiniTest::Unit::TestCase
+  def setup
+    @klass = Class.new.tap { |klass| klass.send :include, Conformist::Base }
+  end
+  
+  def test_option
+    @klass.option :a => 1
+    @klass.option :b => 2
+    assert_equal({:a => 1, :b => 2}, @klass.options)
+  end
+  
+  def test_column
+    @klass.column :a, 0
+    @klass.column :b, 1
+    assert_equal 2, @klass.columns.size
+    assert_instance_of Conformist::Column, @klass.columns[rand(2)]
+  end
+  
+  def test_load
+    instance = @klass.load 'file'
+    assert_equal 'file', instance.path
+    assert_instance_of @klass, instance
+  end
+  
+  def test_foreach
+    rows = [
+      {:name => "CRAFERS", :callsign => "ABS2",  :latitude => "34 58 49S", :signtal_type => "digital"},
+      {:name => "CRAFERS", :callsign => "SAS7",  :latitude => "34 58 57S", :signtal_type => "digital"},
+      {:name => "CRAFERS", :callsign => "NWS9",  :latitude => "34 59 02S", :signtal_type => "digital"},
+      {:name => "CRAFERS", :callsign => "ADS10", :latitude => "34 59 02S", :signtal_type => "digital"},
+      {:name => "CRAFERS", :callsign => "ADS10", :latitude => "34 58 57S", :signtal_type => "digital"}
+    ]
+    ACMA.load(fixture('acma.csv')).foreach do |row|
+      assert_equal rows.shift, row
+    end
+  end
+end

data/test/conformist/column_test.rb

@@ -0,0 +1,43 @@
+require 'helper'
+
+class ColumnTest < MiniTest::Unit::TestCase
+  def test_name
+    column = Conformist::Column.new :foo
+    assert_equal :foo, column.name
+  end
+  
+  def test_one_index
+    column = Conformist::Column.new :foo, 0
+    assert_equal 'a', column.values_in(stub_row)
+  end
+  
+  def test_preprocess_with_one_index
+    column = Conformist::Column.new(:foo, 0) { |value| value.upcase }
+    assert_equal 'A', column.values_in(stub_row)
+  end
+  
+  def test_many_indexes
+    column = Conformist::Column.new :foo, 1, 2, 3
+    assert_equal ['b', 'c', 'd'], column.values_in(stub_row)
+  end
+  
+  def test_preprocess_with_many_indexes
+    column = Conformist::Column.new(:foo, 1, 2, 3) { |values| values.reverse }
+    assert_equal ['d', 'c', 'b'], column.values_in(stub_row)
+  end
+  
+  def test_virtual
+    column = Conformist::Column.new(:foo) { 'a' }
+    assert_equal 'a', column.values_in(stub_row)
+  end
+  
+  def test_strip_whitespace
+    column = Conformist::Column.new(:foo, 0)
+    assert_equal 'a', column.values_in(['  a  '])
+  end
+  
+  def test_nil_empty_string
+    column = Conformist::Column.new(:foo, 0)
+    assert_equal '', column.values_in([])
+  end
+end

data/test/conformist/conformist_test.rb

@@ -0,0 +1,21 @@
+require 'helper'
+
+class ConformistTest < MiniTest::Unit::TestCase
+  def test_foreach
+    rows = [
+      {:name => "CRAFERS",         :callsign => "ABS2",    :latitude => "34 58 49S",     :signtal_type => "digital"},
+      {:name => "CRAFERS",         :callsign => "SAS7",    :latitude => "34 58 57S",     :signtal_type => "digital"},
+      {:name => "CRAFERS",         :callsign => "NWS9",    :latitude => "34 59 02S",     :signtal_type => "digital"},
+      {:name => "CRAFERS",         :callsign => "ADS10",   :latitude => "34 59 02S",     :signtal_type => "digital"},
+      {:name => "CRAFERS",         :callsign => "ADS10",   :latitude => "34 58 57S",     :signtal_type => "digital"},
+      {:name => "LOS ANGELES, CA", :callsign => "KVTU-LP", :latitude => "34 11 13.90 N", :signtal_type => "digital"},
+      {:name => "BAKERSFIELD, CA", :callsign => "NEW",     :latitude => "35 12 6.00 N",  :signtal_type => "digital"},
+      {:name => "YUCA VALLEY, CA", :callsign => "NEW",     :latitude => "34 9 10.10 N",  :signtal_type => "digital"},
+      {:name => "SACRAMENTO, CA",  :callsign => "KCSO-LD", :latitude => "38 7 10.00 N",  :signtal_type => "digital"},
+      {:name => "LOS ANGELES, CA", :callsign => "KVTU-LP", :latitude => "34 13 38.00 N", :signtal_type => "digital"}
+    ]
+    Conformist.foreach ACMA.load(fixture('acma.csv')), FCC.load(fixture('fcc.txt')) do |row|
+      assert_equal rows.shift, row
+    end
+  end
+end

data/test/conformist/row_test.rb

@@ -0,0 +1,21 @@
+require 'helper'
+
+class RowTest < MiniTest::Unit::TestCase
+  def test_to_hash_should_join_array
+    column = Conformist::Column.new(:foo, 0, 1, 2)
+    row    = Conformist::Row.new [column], stub_row
+    assert_equal({:foo => 'abc'}, row.to_hash)
+  end
+  
+  def test_to_hash_should_not_join_string
+    column = Conformist::Column.new(:foo, 0, 1, 2) { |values| values.join('-').upcase }
+    row    = Conformist::Row.new [column], stub_row
+    assert_equal({:foo => 'A-B-C'}, row.to_hash)
+  end
+  
+  def test_to_hash_with_many_columns
+    columns = Conformist::Column.new(:foo, 0), Conformist::Column.new(:bar, 1, 2, 3)
+    row     = Conformist::Row.new columns, stub_row
+    assert_equal({:foo => 'a', :bar => 'bcd'}, row.to_hash)
+  end
+end

data/test/definitions/acma.rb

@@ -0,0 +1,16 @@
+class FCC
+  include Conformist::Base
+  
+  option :col_sep => '|'
+   
+  column :name, 10, 11 do |values|
+    "#{values[0].upcase}, #{values[-1]}"
+  end
+  column :callsign, 1
+  column :latitude, 20, 21, 22, 19 do |values|
+    values.join(' ')
+  end
+  column :signtal_type do
+    'digital'
+  end
+end

data/test/definitions/fcc.rb

@@ -0,0 +1,15 @@
+class ACMA
+  include Conformist::Base
+  
+  option :col_sep => ','
+  option :quote_char => '"'
+
+  column :name, 11 do |value| 
+    value.match(/[A-Z]+$/)[0].upcase
+  end
+  column :callsign, 1
+  column :latitude, 15
+  column :signtal_type do 
+    'digital'
+  end
+end

data/test/fixtures/acma.csv

@@ -0,0 +1,5 @@
+Adelaide,ABS2,64.26,National,H,124,OD,200000,57001,1198821,23139,ntl Tower Summit Road CRAFERS,54,290707,6126723,34 58 49S,138 42 25E,SA,2,,,,Issued
+Adelaide,SAS7,182.26,Commercial,H,122,OD,200000,122001,1139429,23181,Channel 7/10 Site 115 Summit Road CRAFERS,54,290670,6126481,34 58 57S,138 42 24E,SA,7,122,ADELAIDE TV1,162,Issued
+Adelaide,NWS9,196.26,Commercial,H,144,OD,200000,121001,1384392,23132,NWS 9 site 109 Summit Road CRAFERS,54,290698,6126331,34 59 02S,138 42 25E,SA,9,121,ADELAIDE TV1,162,Issued
+Adelaide,ADS10,209.25,Commercial,H,144,OD,200000,120001,1904367,23132,NWS 9 site 109 Summit Road CRAFERS,54,290698,6126331,34 59 02S,138 42 25E,SA,10,120,ADELAIDE TV1,162,Issued
+Adelaide,ADS10,209.25,Commercial,H,122,OD,200000,120001,1384005,23181,Channel 7/10 Site 115 Summit Road CRAFERS,54,290670,6126481,34 58 57S,138 42 24E,SA,10,120,ADELAIDE TV1,162,Issued

data/test/fixtures/fcc.txt

@@ -0,0 +1,5 @@
+|KVTU-LP     |-         |LD |3   |DA  |                    |-  |-  |APP    |LOS ANGELES              |CA |US |BDFCDVL-20100309ABS |0.25   kW |-         |0.0     |-       |130176     |N |34 |11 |13.90 |W |117 |42 |1.00  |MARK C. ALLEN                                                               |   0.00 km |   0.00 mi |  0.00 deg |1702.  m|-       |20780     |240.    |1015693 |9.      |1359585   |
+|NEW         |-         |LD |3   |ND  |                    |-  |-  |APP    |BAKERSFIELD              |CA |US |BMJADVL-20100520ABV |0.3    kW |-         |0.0     |-       |130049     |N |35 |12 |6.00  |W |119 |5  |26.10 |JOHN R. POWLEY                                                              |   0.00 km |   0.00 mi |  0.00 deg |143.2  m|-       |100298    |-       |1015627 |50.     |1368486   |
+|NEW         |-         |LD |3   |ND  |                    |-  |-  |APP    |YUCA VALLEY              |CA |US |BMJADVL-20100520ADI |0.3    kW |-         |0.0     |-       |129831     |N |34 |9  |10.10 |W |116 |23 |19.10 |JOHN R. POWLEY                                                              |   0.00 km |   0.00 mi |  0.00 deg |1179.  m|-       |100305    |-       |1028564 |20.     |1370146   |
+|KCSO-LD     |-         |LD |3   |DA  |                    |-  |-  |LIC    |SACRAMENTO               |CA |US |BLDVL  -20100825AAQ |0.3    kW |-         |0.0     |-       |18998      |N |38 |7  |10.00 |W |120 |43 |27.00 |SAINTE 51, L.P.                                                             |   0.00 km |   0.00 mi |  0.00 deg |948.5  m|-       |96076     |270.    |1050375 |114.    |1396970   |
+|KVTU-LP     |-         |TX |3   |DA  |+                   |-  |-  |CP MOD |LOS ANGELES              |CA |US |BMPTVL -20100826AFT |0.498  kW |-         |0.0     |-       |130176     |N |34 |13 |38.00 |W |118 |4  |3.00  |MARK C. ALLEN                                                               |   0.00 km |   0.00 mi |  0.00 deg |1784.  m|-       |101585    |0.      |1007719 |43.     |1397199   |

data/test/helper.rb

@@ -0,0 +1,13 @@
+require 'minitest/autorun'
+
+require 'conformist'
+require 'definitions/acma'
+require 'definitions/fcc'
+
+def stub_row
+  ('a'..'d').to_a
+end
+
+def fixture file
+  File.expand_path "../fixtures/#{file}", __FILE__
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: conformist
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Tate Johnson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-27 00:00:00 +10:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: *id001
+description: Conformist lets you bend CSVs to your will. Let multiple, different input files conform to a single interface without rewriting your parser each time.
+email: 
+- tate@tatey.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE
+- README.markdown
+- Rakefile
+- conformist.gemspec
+- lib/conformist.rb
+- lib/conformist/base.rb
+- lib/conformist/column.rb
+- lib/conformist/row.rb
+- lib/conformist/version.rb
+- test/conformist/base_test.rb
+- test/conformist/column_test.rb
+- test/conformist/conformist_test.rb
+- test/conformist/row_test.rb
+- test/definitions/acma.rb
+- test/definitions/fcc.rb
+- test/fixtures/acma.csv
+- test/fixtures/fcc.txt
+- test/helper.rb
+has_rdoc: true
+homepage: https://github.com/tatey/conformist
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: -2005375605145254914
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: conformist
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Let multiple, different input files conform to a single interface.
+test_files: 
+- test/conformist/base_test.rb
+- test/conformist/column_test.rb
+- test/conformist/conformist_test.rb
+- test/conformist/row_test.rb
+- test/definitions/acma.rb
+- test/definitions/fcc.rb
+- test/fixtures/acma.csv
+- test/fixtures/fcc.txt
+- test/helper.rb