loady 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +4 -4
- data/Rakefile +3 -7
- data/lib/loady.rb +6 -1
- data/lib/loady/attribute_array.rb +5 -5
- data/lib/loady/csv_loader.rb +50 -44
- data/lib/loady/version.rb +1 -1
- data/loady.gemspec +3 -2
- data/test/test_attribute_array.rb +1 -1
- data/test/test_csv_loader.rb +18 -4
- data/test/test_memory_logger.rb +4 -4
- metadata +28 -6
data/README.rdoc
CHANGED
@@ -20,7 +20,7 @@ By default, messages are logged to the standard output.
|
|
20
20
|
|
21
21
|
Basic usage:
|
22
22
|
|
23
|
-
Loady.
|
23
|
+
Loady.read "/your/file.csv" do |row|
|
24
24
|
# your code to process each row goes here
|
25
25
|
puts "#{row[0]}, #{row[1]}, etc."
|
26
26
|
end
|
@@ -29,19 +29,19 @@ Skip the first row and log to a file:
|
|
29
29
|
|
30
30
|
logger = Logger.new "/your/file.log"
|
31
31
|
|
32
|
-
Loady.
|
32
|
+
Loady.read "/your/file.csv", logger: logger, skip_first_row: true do |row|
|
33
33
|
# do some stuff for each row
|
34
34
|
end
|
35
35
|
|
36
36
|
Name your attributes:
|
37
37
|
|
38
|
-
Loady.
|
38
|
+
Loady.read "/your/file.csv" do |row|
|
39
39
|
Monkey.create row.to_attributes [:name, :year, :mom]
|
40
40
|
end
|
41
41
|
|
42
42
|
Load a tab-delimited file:
|
43
43
|
|
44
|
-
Loady.
|
44
|
+
Loady.read "/your/file.tab", col_sep: "\t" do |row|
|
45
45
|
# go bananas
|
46
46
|
end
|
47
47
|
|
data/Rakefile
CHANGED
@@ -1,14 +1,10 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
|
4
|
-
require 'bundler'
|
5
|
-
Bundler::GemHelper.install_tasks
|
6
|
-
|
7
|
-
require 'rake/testtask'
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "rake/testtask"
|
8
3
|
|
9
4
|
task default: [:test]
|
10
5
|
|
11
6
|
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs << 'test'
|
12
8
|
t.test_files = %w(test/**/*.rb)
|
13
9
|
t.verbose = false
|
14
10
|
end
|
data/lib/loady.rb
CHANGED
@@ -3,8 +3,13 @@ require 'loady/csv_loader'
|
|
3
3
|
require 'loady/memory_logger'
|
4
4
|
|
5
5
|
module Loady
|
6
|
+
def read(*args, &block)
|
7
|
+
Loady::CsvLoader.new.read(*args, &block)
|
8
|
+
end
|
9
|
+
module_function :read
|
10
|
+
|
6
11
|
def csv(*args, &block)
|
7
|
-
Loady::CsvLoader.read(*args, &block)
|
12
|
+
Loady::CsvLoader.new.read(*args, &block)
|
8
13
|
end
|
9
14
|
module_function :csv
|
10
15
|
end
|
@@ -12,21 +12,21 @@ module Loady
|
|
12
12
|
def to_attributes(names, options={})
|
13
13
|
options = { strip: true }.merge(options)
|
14
14
|
|
15
|
-
|
15
|
+
attr_hash = {}
|
16
16
|
|
17
17
|
names.each_with_index do |name, i|
|
18
18
|
if i < self.size
|
19
19
|
if options[:strip] && self[i]
|
20
|
-
|
20
|
+
attr_hash[name] = self[i].strip
|
21
21
|
else
|
22
|
-
|
22
|
+
attr_hash[name] = self[i]
|
23
23
|
end
|
24
24
|
else
|
25
|
-
|
25
|
+
attr_hash[name] = nil
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
|
29
|
+
attr_hash
|
30
30
|
end
|
31
31
|
|
32
32
|
end
|
data/lib/loady/csv_loader.rb
CHANGED
@@ -3,57 +3,63 @@ require 'logger'
|
|
3
3
|
|
4
4
|
module Loady
|
5
5
|
class CsvLoader
|
6
|
+
def initialize
|
7
|
+
@success = 0
|
8
|
+
@warning = 0
|
9
|
+
@line_number = 0
|
10
|
+
end
|
6
11
|
|
7
12
|
class << self
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
begin
|
25
|
-
while (line = f.readline)
|
26
|
-
begin
|
27
|
-
line_number += 1
|
28
|
-
|
29
|
-
row = Loady::AttributeArray.new CSV.parse(line, options)[0]
|
30
|
-
|
31
|
-
unless row.empty?
|
32
|
-
yield row
|
33
|
-
success += 1
|
34
|
-
end
|
35
|
-
rescue Exception => message
|
36
|
-
warning += 1
|
37
|
-
logger.warn "#{message.to_s.gsub("line 1", "line #{line_number}")}\n#{line}"
|
38
|
-
end
|
39
|
-
end
|
40
|
-
rescue EOFError
|
41
|
-
f.close
|
42
|
-
rescue Exception => message
|
43
|
-
warning += 1
|
44
|
-
line_number += 1
|
45
|
-
logger.error "#{message.to_s} - at line #{line_number}\n#{line}"
|
46
|
-
end
|
13
|
+
def read(*args, &block)
|
14
|
+
self.new.read(*args, &block)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# valid options:
|
19
|
+
# skip_first_row: true -- default = false
|
20
|
+
# logger: Logger.new('/somewhere/file.log') -- default = Logger.new(STDOUT)
|
21
|
+
# plus any valid options you can pass to CSV.new:
|
22
|
+
# see http://www.ruby-doc.org/stdlib/libdoc/csv/rdoc/classes/CSV.html#M000190
|
23
|
+
def read(filename, options={}, &block)
|
24
|
+
@logger = options.delete(:logger) || default_logger
|
25
|
+
|
26
|
+
f = File.new(filename)
|
27
|
+
f.readline if options.delete(:skip_first_row)
|
47
28
|
|
48
|
-
|
29
|
+
begin
|
30
|
+
while (line = f.readline)
|
31
|
+
readline(line, options, &block)
|
32
|
+
end
|
33
|
+
rescue EOFError
|
34
|
+
f.close
|
35
|
+
rescue Exception => message
|
36
|
+
@warning += 1
|
37
|
+
@line_number += 1
|
38
|
+
@logger.error "#{message} - at line #@line_number\n#{line}"
|
49
39
|
end
|
50
40
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
41
|
+
@logger.info "Finished. Loaded #@success rows. #@warning unprocessed rows."
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def readline(line, options)
|
47
|
+
@line_number += 1
|
48
|
+
row = AttributeArray.new(CSV.parse(line, options)[0])
|
49
|
+
|
50
|
+
unless row.empty?
|
51
|
+
yield row
|
52
|
+
@success += 1
|
55
53
|
end
|
54
|
+
rescue Exception => message
|
55
|
+
@warning += 1
|
56
|
+
@logger.warn "#{message.to_s.gsub("line 1", "line #@line_number")}\n#{line}"
|
57
|
+
end
|
56
58
|
|
59
|
+
def default_logger
|
60
|
+
logger = Logger.new(STDOUT)
|
61
|
+
logger.datetime_format = "%H:%M:%S"
|
62
|
+
logger
|
57
63
|
end
|
58
64
|
end
|
59
65
|
end
|
data/lib/loady/version.rb
CHANGED
data/loady.gemspec
CHANGED
@@ -19,7 +19,8 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
20
|
s.require_paths = %w(lib)
|
21
21
|
|
22
|
-
s.add_development_dependency "
|
23
|
-
s.add_development_dependency "shoulda"
|
22
|
+
s.add_development_dependency "mocha"
|
24
23
|
s.add_development_dependency "rake"
|
24
|
+
s.add_development_dependency "shoulda"
|
25
|
+
s.add_development_dependency "test-unit"
|
25
26
|
end
|
data/test/test_csv_loader.rb
CHANGED
@@ -1,11 +1,25 @@
|
|
1
|
-
require
|
1
|
+
require 'test_helper'
|
2
2
|
|
3
3
|
class CsvLoaderTest < Test::Unit::TestCase
|
4
|
+
should "delegate Loady.csv to instance #read" do
|
5
|
+
Loady::CsvLoader.any_instance.expects :read
|
6
|
+
Loady.csv("file")
|
7
|
+
end
|
8
|
+
|
9
|
+
should "delegate Loady.read to instance #read" do
|
10
|
+
Loady::CsvLoader.any_instance.expects :read
|
11
|
+
Loady.read("file")
|
12
|
+
end
|
13
|
+
|
14
|
+
should "delegate #read to instance #read" do
|
15
|
+
Loady::CsvLoader.any_instance.expects :read
|
16
|
+
Loady::CsvLoader.read("file")
|
17
|
+
end
|
4
18
|
|
5
19
|
should "read file1" do
|
6
20
|
monkeys = []
|
7
21
|
|
8
|
-
Loady
|
22
|
+
Loady.read "test/csv/file1.csv", skip_first_row: true do |row|
|
9
23
|
monkeys << { name: row[0], year: row[1] }
|
10
24
|
end
|
11
25
|
|
@@ -20,7 +34,7 @@ class CsvLoaderTest < Test::Unit::TestCase
|
|
20
34
|
logger = Logger.new("/dev/null")
|
21
35
|
monkeys = []
|
22
36
|
|
23
|
-
Loady.
|
37
|
+
Loady.read "test/csv/file2.csv", logger: logger do |row|
|
24
38
|
monkeys << row.to_attributes([:name, :year])
|
25
39
|
end
|
26
40
|
|
@@ -34,7 +48,7 @@ class CsvLoaderTest < Test::Unit::TestCase
|
|
34
48
|
should "read file3, a tab-delimited file" do
|
35
49
|
monkeys = []
|
36
50
|
|
37
|
-
Loady.
|
51
|
+
Loady.read "test/csv/file3.dat", skip_first_row: true, col_sep: "\t" do |row|
|
38
52
|
monkeys << { name: row[0], year: row[1] }
|
39
53
|
end
|
40
54
|
|
data/test/test_memory_logger.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'test_helper'
|
2
2
|
|
3
3
|
class MemoryLoggerTest < Test::Unit::TestCase
|
4
4
|
context "memory logger" do
|
@@ -12,17 +12,17 @@ class MemoryLoggerTest < Test::Unit::TestCase
|
|
12
12
|
|
13
13
|
should "log warning" do
|
14
14
|
@logger.warn "message"
|
15
|
-
assert_equal "message", @logger.messages.first
|
15
|
+
assert_equal "message", @logger.messages.first
|
16
16
|
end
|
17
17
|
|
18
18
|
should "log info" do
|
19
19
|
@logger.info "message"
|
20
|
-
assert_equal "message", @logger.messages.first
|
20
|
+
assert_equal "message", @logger.messages.first
|
21
21
|
end
|
22
22
|
|
23
23
|
should "log error" do
|
24
24
|
@logger.error "message"
|
25
|
-
assert_equal "message", @logger.messages.first
|
25
|
+
assert_equal "message", @logger.messages.first
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
metadata
CHANGED
@@ -1,18 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: loady
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.0
|
5
4
|
prerelease:
|
5
|
+
version: 0.7.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Tee Parham
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
15
|
+
name: mocha
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
@@ -20,13 +20,29 @@ dependencies:
|
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ! '>='
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
version: '0'
|
23
29
|
prerelease: false
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
24
39
|
version_requirements: !ruby/object:Gem::Requirement
|
25
40
|
none: false
|
26
41
|
requirements:
|
27
42
|
- - ! '>='
|
28
43
|
- !ruby/object:Gem::Version
|
29
44
|
version: '0'
|
45
|
+
prerelease: false
|
30
46
|
- !ruby/object:Gem::Dependency
|
31
47
|
name: shoulda
|
32
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -36,15 +52,15 @@ dependencies:
|
|
36
52
|
- !ruby/object:Gem::Version
|
37
53
|
version: '0'
|
38
54
|
type: :development
|
39
|
-
prerelease: false
|
40
55
|
version_requirements: !ruby/object:Gem::Requirement
|
41
56
|
none: false
|
42
57
|
requirements:
|
43
58
|
- - ! '>='
|
44
59
|
- !ruby/object:Gem::Version
|
45
60
|
version: '0'
|
61
|
+
prerelease: false
|
46
62
|
- !ruby/object:Gem::Dependency
|
47
|
-
name:
|
63
|
+
name: test-unit
|
48
64
|
requirement: !ruby/object:Gem::Requirement
|
49
65
|
none: false
|
50
66
|
requirements:
|
@@ -52,13 +68,13 @@ dependencies:
|
|
52
68
|
- !ruby/object:Gem::Version
|
53
69
|
version: '0'
|
54
70
|
type: :development
|
55
|
-
prerelease: false
|
56
71
|
version_requirements: !ruby/object:Gem::Requirement
|
57
72
|
none: false
|
58
73
|
requirements:
|
59
74
|
- - ! '>='
|
60
75
|
- !ruby/object:Gem::Version
|
61
76
|
version: '0'
|
77
|
+
prerelease: false
|
62
78
|
description: CSV file loader with simple logging
|
63
79
|
email:
|
64
80
|
- tee@neighborland.com
|
@@ -95,12 +111,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
95
111
|
requirements:
|
96
112
|
- - ! '>='
|
97
113
|
- !ruby/object:Gem::Version
|
114
|
+
segments:
|
115
|
+
- 0
|
116
|
+
hash: 1518214605886805454
|
98
117
|
version: '0'
|
99
118
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
119
|
none: false
|
101
120
|
requirements:
|
102
121
|
- - ! '>='
|
103
122
|
- !ruby/object:Gem::Version
|
123
|
+
segments:
|
124
|
+
- 0
|
125
|
+
hash: 1518214605886805454
|
104
126
|
version: '0'
|
105
127
|
requirements: []
|
106
128
|
rubyforge_project: loady
|