csv_madness 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/CHANGELOG.markdown +21 -3
- data/Gemfile +10 -7
- data/README.rdoc +96 -63
- data/Rakefile +7 -15
- data/VERSION +1 -1
- data/lib/csv_madness.rb +4 -13
- data/lib/csv_madness/builder.rb +97 -0
- data/lib/csv_madness/gem_api.rb +12 -0
- data/lib/csv_madness/record.rb +38 -1
- data/lib/csv_madness/sheet.rb +196 -33
- data/test/csv/forbidden_column.csv +2 -0
- data/test/csv/splitter.csv +11 -0
- data/test/csv/test_column_types.csv +3 -0
- data/test/csv/with_nils.csv +5 -0
- data/test/helper.rb +26 -19
- data/test/test_builder.rb +33 -0
- data/test/test_csv_madness.rb +2 -3
- data/test/test_merging_columns.rb +40 -0
- data/test/test_reloading_spreadsheet.rb +30 -0
- data/test/test_sheet.rb +102 -3
- metadata +26 -85
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZGQ4MTA4MGJlYjA1MzIwOTU4NzEyODViMmFhYmYwMTA2YjQzMmY2MQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ZTRkOWNjMjYyNTNmMjA3YzMwZThlMzAyOWRjMTM0ZWYwMTQ3NWE0MA==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MzAzNTJmNjU5YmM2NjUxODY4YjRkMDkwMDc5NTQxOWIxZWRjMjRmYmU1YzFh
|
10
|
+
Zjg2ZjA5ZGZlZTEyNjg1NjFmNzE5NzY3NTFhNzY3ZTIyNDFhZGIzZTEzOGI5
|
11
|
+
ZDUzMjZjYjJkODdiM2M1M2JiMjZiMGNlMTViNWE2Yzk1ZjU1YjI=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
MjJlYzI1YTQxMzU4NzA0MmVlNzFkYzVmNDE3NjYyMDM5YmI3Y2JmMmQyOWZk
|
14
|
+
OGQ0NzFhODUyMzRjNWI3ZWRlZDQ3YmQ0NGU0Zjc4ZDFmNWRkMWY4MDc4OTdi
|
15
|
+
YjkwY2U1MGY3MmVhNTY2NTYzYzc1ZDViZWViMTg0OTE5OGU1ZTE=
|
data/CHANGELOG.markdown
CHANGED
@@ -1,17 +1,35 @@
|
|
1
|
+
Changelog
|
2
|
+
=========
|
3
|
+
|
4
|
+
0.0.6
|
5
|
+
-----
|
6
|
+
|
7
|
+
* CsvMadness::Builder added (easily generate spreadsheets from an array of objects)
|
8
|
+
|
9
|
+
|
10
|
+
0.0.5
|
11
|
+
-----
|
12
|
+
|
13
|
+
* Splitting spreadsheets, creating blank copies of spreadsheets, adding/removing records.
|
14
|
+
* Convert records to hashes/arrays.
|
15
|
+
* Created a dependency on `fun_with_testing`.
|
16
|
+
* CsvMadness.version
|
17
|
+
|
18
|
+
|
1
19
|
0.0.4
|
2
|
-
|
20
|
+
-----
|
3
21
|
|
4
22
|
* Fixed serious bug for default column names. record.middle_name instead of record.middlename
|
5
23
|
* Spreadsheet can be re-read from file by calling @sheet.reload_spreadsheet(opts)
|
6
24
|
|
7
25
|
|
8
26
|
0.0.3
|
9
|
-
|
27
|
+
-----
|
10
28
|
|
11
29
|
* Feature: can add and drop columns from spreadsheets
|
12
30
|
|
13
31
|
|
14
32
|
0.0.2
|
15
|
-
|
33
|
+
-----
|
16
34
|
|
17
35
|
* Aw, hell. I don't remember.
|
data/Gemfile
CHANGED
@@ -6,12 +6,15 @@ source "http://rubygems.org"
|
|
6
6
|
# Add dependencies to develop your gem here.
|
7
7
|
# Include everything needed to run rake, tests, features, etc.
|
8
8
|
|
9
|
-
gem "fun_with_files"
|
10
|
-
|
11
9
|
group :development do
|
12
|
-
gem "shoulda", ">=
|
13
|
-
gem "rdoc", "~> 3.12"
|
14
|
-
gem "bundler", "~> 1.
|
15
|
-
gem "jeweler", "~>
|
16
|
-
gem "
|
10
|
+
# gem "shoulda", ">= 3.5"
|
11
|
+
# gem "rdoc", "~> 3.12"
|
12
|
+
# gem "bundler", "~> 1.5"
|
13
|
+
# gem "jeweler", "~> 2"
|
14
|
+
gem "fun_with_testing"
|
15
|
+
# gem "debugger"
|
17
16
|
end
|
17
|
+
|
18
|
+
# gem "fun_with_files", "~> 0.0", ">= 0.0.7"
|
19
|
+
# gem "fun_with_version_strings", "~> 0.0"
|
20
|
+
gem 'fun_with_gems', '~> 0.0', ">= 0.0.2"
|
data/README.rdoc
CHANGED
@@ -66,89 +66,122 @@ It's useful to clean up your files. Say you have:
|
|
66
66
|
|
67
67
|
Ick. Missing IDs, inconsistent date formats, leading and trailing whitespace... We can fix this.
|
68
68
|
|
69
|
-
|
70
|
-
|
71
|
-
sheet.
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
#
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
69
|
+
require 'csv_madness'
|
70
|
+
|
71
|
+
sheet = CsvMadness.load( "~/data/people.csv" )
|
72
|
+
|
73
|
+
# remove leading and trailing whitespace, and turns nils into ""
|
74
|
+
sheet.alter_cells do |cell, record|
|
75
|
+
(cell || "").strip
|
76
|
+
end
|
77
|
+
|
78
|
+
# the last argument provides a default for blank records.
|
79
|
+
sheet.set_column_type(:id, :integer, nil)
|
80
|
+
|
81
|
+
# assumes the missing ids can be filled in sequentially.
|
82
|
+
# While .alter_column() does take a default (second argument),
|
83
|
+
# which will fill in the blank cells,
|
84
|
+
# that's not what we want here.
|
85
|
+
#
|
86
|
+
# If a blank is provided, your
|
87
|
+
# code will never see the records with blank entries.
|
88
|
+
sheet.alter_column(:id) do |id, record|
|
89
|
+
@count ||= 1
|
90
|
+
if id.nil?
|
91
|
+
id = @count
|
92
|
+
@count += 1
|
93
|
+
else
|
94
|
+
@count = id + 1
|
95
|
+
end
|
89
96
|
|
90
|
-
|
91
|
-
end
|
92
|
-
|
93
|
-
sheet.
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
end
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
#
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
end
|
117
|
-
|
118
|
-
|
119
|
-
|
97
|
+
id
|
98
|
+
end
|
99
|
+
|
100
|
+
sheet.column(:id) # => [1, 2, 3, 4]
|
101
|
+
|
102
|
+
# Reformat a column of dates.
|
103
|
+
require 'time'
|
104
|
+
sheet.alter_column(:born) do |date_string|
|
105
|
+
begin
|
106
|
+
Time.parse( date_string ).strftime("%Y-%m-%d")
|
107
|
+
rescue ArgumentError
|
108
|
+
""
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
sheet.column(:date) # => ["1986-04-08", "1974-02-22", "", "1901-03-02"]
|
113
|
+
|
114
|
+
# The same thing can be accomplished more simply by saying <tt>sheet.set_column_type(:date)</tt>.
|
115
|
+
# Even better, record.date is then a Time object
|
116
|
+
|
117
|
+
# Now calculate everyone's age (ignoring and overriding the existing data)
|
118
|
+
sheet.alter_column(:age) do |age, record|
|
119
|
+
if record.blank?(:born)
|
120
|
+
""
|
121
|
+
else
|
122
|
+
((Time.now - Time.parse(record.born)) / 365.25 / 24 / 60 / 60).to_i # not production-worthy code
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
# save the cleaned data for the next stage of your process
|
127
|
+
sheet.write_to_file( "~/data/people.clean.csv", force_quotes: true )
|
120
128
|
|
121
129
|
You could do something similar to clean and standardize phone numbers, detect and delete/complete invalid emails, etc.
|
122
130
|
|
123
131
|
|
124
132
|
=== Adding, removing, and renaming columns ===
|
125
133
|
|
126
|
-
# Add 72 years to the date born.
|
127
|
-
sheet.set_column_type( :born, :date ) # replace date strings with Time objects
|
128
134
|
|
129
|
-
|
130
|
-
|
131
|
-
|
135
|
+
# Add 72 years to the date born.
|
136
|
+
sheet.set_column_type( :born, :date ) # replace date strings with Time objects
|
137
|
+
|
138
|
+
sheet.add_column( :expected_death_date ) do |date, record|
|
139
|
+
record.born + ( 72 * 365 * 24 * 60 * 60 )
|
140
|
+
end
|
141
|
+
|
142
|
+
puts sheet[0].expected_death_date # should be in 2058
|
132
143
|
|
133
|
-
|
144
|
+
# But that's just morbid, so we drop the column
|
145
|
+
sheet.drop_column( :expected_death_date )
|
134
146
|
|
135
|
-
#
|
136
|
-
sheet.
|
147
|
+
# Or, if you think you need the information, but need to be a bit more euphemistic about it
|
148
|
+
sheet.rename_column( :expected_death_date, :expiration_date )
|
137
149
|
|
138
|
-
# Or, if you think you need the information, but need to be a bit more euphemistic about it
|
139
|
-
sheet.rename_column( :expected_death_date, :expiration_date )
|
140
150
|
|
141
151
|
=== Using columns ===
|
142
152
|
|
143
|
-
sheet.set_column_type( :id, :integer )
|
144
153
|
|
145
154
|
|
146
|
-
|
147
|
-
|
155
|
+
sheet.set_column_type( :id, :integer )
|
156
|
+
|
157
|
+
|
158
|
+
# Returns each record's id as an array of integers
|
159
|
+
sheet.column( :id ).max # ==> 4
|
160
|
+
|
161
|
+
|
162
|
+
=== Builder ===
|
148
163
|
|
164
|
+
You have an array of objects. You want to write them to a spreadsheet.
|
149
165
|
|
166
|
+
```ruby
|
167
|
+
sb = CsvMadness::Builder.new do |sb|
|
168
|
+
sb.column( :id )
|
169
|
+
sb.column( :addressee, "addressee_custom" )
|
170
|
+
sb.column( :street_address, "primary_address.street_address" )
|
171
|
+
sb.column( :supplemental_address_1, "primary_address.supplemental_address_1" )
|
172
|
+
sb.column( :city, "primary_address.city" )
|
173
|
+
sb.column( :state_code, "primary_address.state_code" )
|
174
|
+
sb.column( :formatted_zip, "primary_address.formatted_zip" )
|
175
|
+
sb.column( :phone, "phones.first.phone" )
|
176
|
+
sb.column( :leader, "congregation_fieldset.congregation_leader" )
|
177
|
+
sb.column( :denomination, "congregation_fieldset.denomination" )
|
178
|
+
sb.column( :email, "emails.first.email" )
|
179
|
+
end
|
150
180
|
|
181
|
+
sheet = sb.build( [address1, address2, address3...] )
|
182
|
+
```
|
151
183
|
|
184
|
+
=== Documentation is incomplete ===
|
152
185
|
|
153
186
|
There are lots of other features, but they'll take time to test and document.
|
154
187
|
|
data/Rakefile
CHANGED
@@ -20,24 +20,16 @@ Jeweler::Tasks.new do |gem|
|
|
20
20
|
gem.name = "csv_madness"
|
21
21
|
gem.homepage = "http://github.com/darthschmoo/csv_madness"
|
22
22
|
gem.license = "MIT"
|
23
|
-
gem.summary =
|
24
|
-
gem.description =
|
23
|
+
gem.summary = "CSV Madness turns your CSV rows into happycrazy objects."
|
24
|
+
gem.description = "CSV Madness removes what little pain is left from Ruby's CSV class. Load a CSV file, and get back an array of objects with customizable getter/setter methods."
|
25
25
|
gem.email = "keeputahweird@gmail.com"
|
26
26
|
gem.authors = ["Bryce Anderson"]
|
27
|
+
|
27
28
|
# dependencies defined in Gemfile
|
28
|
-
gem.files =
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
"./Gemfile",
|
33
|
-
"./VERSION",
|
34
|
-
"./README.rdoc",
|
35
|
-
"./Rakefile",
|
36
|
-
"./CHANGELOG.markdown",
|
37
|
-
"./test/csv/simple.csv",
|
38
|
-
"./test/helper.rb",
|
39
|
-
"./test/test_csv_madness.rb",
|
40
|
-
"./test/test_sheet.rb" ]
|
29
|
+
gem.files = Dir.glob( File.join( ".", "lib", "**", "*.rb" ) ) +
|
30
|
+
Dir.glob( File.join( ".", "test", "**", "*" ) ) +
|
31
|
+
%w( Gemfile Rakefile LICENSE.txt README.rdoc VERSION CHANGELOG.markdown )
|
32
|
+
|
41
33
|
end
|
42
34
|
|
43
35
|
Jeweler::RubygemsDotOrgTasks.new
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.6
|
data/lib/csv_madness.rb
CHANGED
@@ -1,17 +1,8 @@
|
|
1
1
|
require 'csv'
|
2
|
-
require '
|
2
|
+
require 'fun_with_gems'
|
3
|
+
# require 'fun_with_version_strings'
|
3
4
|
require 'time' # to use Time.parse to parse cells to get the date
|
4
|
-
require 'debugger'
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
require_relative 'csv_madness/record'
|
6
|
+
lib_dir = __FILE__.fwf_filepath.dirname
|
7
|
+
FunWith::Gems.make_gem_fun( "CsvMadness", :require => lib_dir.join( "csv_madness" ) )
|
9
8
|
|
10
|
-
|
11
|
-
FunWith::Files::RootPath.rootify( CsvMadness, __FILE__.fwf_filepath.dirname.up )
|
12
|
-
|
13
|
-
CsvMadness.class_eval do
|
14
|
-
def self.load( csv, opts = {} )
|
15
|
-
CsvMadness::Sheet.from( csv, opts )
|
16
|
-
end
|
17
|
-
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module CsvMadness
|
2
|
+
class Builder
|
3
|
+
def initialize( &block )
|
4
|
+
@columns = {}
|
5
|
+
@column_syms = []
|
6
|
+
@module = Module.new # for extending
|
7
|
+
self.extend( @module )
|
8
|
+
yield self
|
9
|
+
end
|
10
|
+
|
11
|
+
def column( sym, method_path = nil, &block )
|
12
|
+
warn( "#{sym} already defined. Overwriting." ) if @column_syms.include?( sym )
|
13
|
+
|
14
|
+
@column_syms << sym
|
15
|
+
@columns[sym] = if block_given?
|
16
|
+
block
|
17
|
+
elsif method_path
|
18
|
+
method_path
|
19
|
+
else
|
20
|
+
Proc.new(&sym)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# Three :on_error values:
|
25
|
+
# :print => Put an error message in the cell instead of a value
|
26
|
+
# :raise => Raise the error, halting the process
|
27
|
+
# :ignore => Hand back an empty cell
|
28
|
+
#
|
29
|
+
# Although ideally it should be configurable by column...
|
30
|
+
def build( objects, opts = { :on_error => :print } )
|
31
|
+
spreadsheet = CsvMadness::Sheet.new( @column_syms )
|
32
|
+
|
33
|
+
for object in objects
|
34
|
+
STDOUT << "."
|
35
|
+
record = {}
|
36
|
+
for sym in @column_syms
|
37
|
+
record[sym] = build_cell( object, sym, opts )
|
38
|
+
end
|
39
|
+
|
40
|
+
spreadsheet.add_record( record ) # hash form
|
41
|
+
end
|
42
|
+
|
43
|
+
spreadsheet
|
44
|
+
end
|
45
|
+
|
46
|
+
def build_cell( object, sym, opts = { :on_error => :print } )
|
47
|
+
column = @columns[sym]
|
48
|
+
|
49
|
+
case column
|
50
|
+
when String
|
51
|
+
build_cell_by_pathstring( object, column, opts )
|
52
|
+
when Proc
|
53
|
+
build_cell_by_proc( object, column, opts )
|
54
|
+
else
|
55
|
+
"no idea what to do"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def def( method_name, &block )
|
60
|
+
@module.send( :define_method, method_name, &block )
|
61
|
+
end
|
62
|
+
|
63
|
+
protected
|
64
|
+
def build_cell_by_pathstring( object, str, opts )
|
65
|
+
handle_cell_build_error( :build_cell_by_pathstring, opts ) do
|
66
|
+
for method in str.split(".").map(&:to_sym)
|
67
|
+
object = object.send(method)
|
68
|
+
end
|
69
|
+
|
70
|
+
object
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def build_cell_by_proc( object, proc, opts )
|
75
|
+
handle_cell_build_error( :build_cell_by_proc, opts ) do
|
76
|
+
proc.call(object)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def handle_cell_build_error( caller, opts, &block )
|
81
|
+
begin
|
82
|
+
yield
|
83
|
+
end
|
84
|
+
rescue Exception => e
|
85
|
+
case opts[:on_error]
|
86
|
+
when nil, :print
|
87
|
+
puts "error #{e.message} #{caller}" if opts[:verbose]
|
88
|
+
"ERROR: #{e.message} (#{caller}())"
|
89
|
+
when :raise
|
90
|
+
puts "Re-raisinge error #{e.message}. Set opts[:on_error] to :print or :ignore if you want Builder to continue on errors."
|
91
|
+
raise e
|
92
|
+
when :ignore
|
93
|
+
""
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/csv_madness/record.rb
CHANGED
@@ -8,8 +8,9 @@ module CsvMadness
|
|
8
8
|
# symbol.
|
9
9
|
class Record
|
10
10
|
attr_accessor :csv_data
|
11
|
+
|
11
12
|
def initialize( data )
|
12
|
-
|
13
|
+
import_record_data( data )
|
13
14
|
end
|
14
15
|
|
15
16
|
def [] key
|
@@ -34,6 +35,10 @@ module CsvMadness
|
|
34
35
|
self.class.spreadsheet.columns
|
35
36
|
end
|
36
37
|
|
38
|
+
def self.columns
|
39
|
+
self.spreadsheet.columns
|
40
|
+
end
|
41
|
+
|
37
42
|
def self.spreadsheet= sheet
|
38
43
|
@spreadsheet = sheet
|
39
44
|
end
|
@@ -46,8 +51,40 @@ module CsvMadness
|
|
46
51
|
self.columns.map{|col| self.send(col) }.to_csv( opts )
|
47
52
|
end
|
48
53
|
|
54
|
+
def to_hash
|
55
|
+
self.columns.inject({}){ |hash, col| hash[col] = self.send( col ); hash }
|
56
|
+
end
|
57
|
+
|
58
|
+
def to_a
|
59
|
+
self.to_hash.to_a
|
60
|
+
end
|
61
|
+
|
49
62
|
def blank?( col )
|
50
63
|
(self.send( col.to_sym ).to_s || "").strip.length == 0
|
51
64
|
end
|
65
|
+
|
66
|
+
protected
|
67
|
+
def import_record_data( data )
|
68
|
+
case data
|
69
|
+
when Array
|
70
|
+
csv_data = CSV::Row.new( self.columns, data )
|
71
|
+
when Hash
|
72
|
+
fields = self.columns.map do |col|
|
73
|
+
data[col]
|
74
|
+
end
|
75
|
+
# for col in self.columns
|
76
|
+
# fields << data[col]
|
77
|
+
# end
|
78
|
+
|
79
|
+
csv_data = CSV::Row.new( self.columns, fields )
|
80
|
+
when CSV::Row
|
81
|
+
csv_data = data
|
82
|
+
else
|
83
|
+
raise "record.import_record_data() doesn't take objects of type #{data.inspect}" unless data.respond_to?(:csv_data)
|
84
|
+
csv_data = data.csv_data.clone
|
85
|
+
end
|
86
|
+
|
87
|
+
@csv_data = csv_data
|
88
|
+
end
|
52
89
|
end
|
53
90
|
end
|