theman 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +7 -4
- data/lib/theman/agency.rb +29 -0
- data/lib/theman/version.rb +1 -1
- data/spec/agency_spec.rb +17 -2
- metadata +5 -13
data/README.rdoc
CHANGED
@@ -10,7 +10,8 @@ Theman lets you import lots of data into PostgreSQL very fast.
|
|
10
10
|
|
11
11
|
== Basic usage
|
12
12
|
|
13
|
-
Say we have a csv file called <tt>sample.csv</tt>
|
13
|
+
Say we have a csv file called <tt>sample.csv</tt> and we want to count how many
|
14
|
+
rows we created:
|
14
15
|
|
15
16
|
conn = PGconn.open(:dbname => 'test')
|
16
17
|
|
@@ -38,7 +39,7 @@ Theman will call the <tt>create!</tt> method if you pass in a block.
|
|
38
39
|
|
39
40
|
agent = Theman::Agency.new conn, 'ugly.csv' do |smith|
|
40
41
|
smith.nulls /"N"/, /"UNKNOWN"/, /""/
|
41
|
-
smith.
|
42
|
+
smith.chop 15
|
42
43
|
smith.delimiter "|"
|
43
44
|
smith.table do |t|
|
44
45
|
t.string :name, :limit => 50
|
@@ -117,10 +118,12 @@ dd/mm/yyyy
|
|
117
118
|
|
118
119
|
mm/dd/yyyy
|
119
120
|
|
120
|
-
==
|
121
|
+
== Errors
|
121
122
|
|
122
123
|
PostgreSQL <tt>COPY</tt> requires that the data be well formed, any rows that
|
123
|
-
are different to what is expected by the table
|
124
|
+
are different to what is expected by the table the import will raise a
|
125
|
+
<tt>Theman::Agency::Error</tt>.
|
126
|
+
|
124
127
|
If you are importing very large files and the import fails space on disc will still
|
125
128
|
be used untill <tt>VACUUM</tt>.
|
126
129
|
|
data/lib/theman/agency.rb
CHANGED
@@ -77,6 +77,11 @@ module Theman
|
|
77
77
|
def seds(*args)
|
78
78
|
@seds = args
|
79
79
|
end
|
80
|
+
|
81
|
+
# line to finish copy at
|
82
|
+
def chop(line = 1)
|
83
|
+
@chop = line
|
84
|
+
end
|
80
85
|
|
81
86
|
# delimter used in stream - comma is the default
|
82
87
|
def delimiter(arg)
|
@@ -100,8 +105,13 @@ module Theman
|
|
100
105
|
def sed_command(sed = []) #:nodoc:
|
101
106
|
sed << nulls_to_sed unless @nulls.nil?
|
102
107
|
sed << @seds unless @seds.nil?
|
108
|
+
sed << chop_to_sed unless @chop.nil?
|
103
109
|
sed
|
104
110
|
end
|
111
|
+
|
112
|
+
def chop_to_sed #:nodoc:
|
113
|
+
"-n -e :a -e '1,#{@chop}!{P;N;D;};N;ba'"
|
114
|
+
end
|
105
115
|
|
106
116
|
def nulls_to_sed #:nodoc:
|
107
117
|
@nulls.map do |regex|
|
@@ -162,6 +172,25 @@ module Theman
|
|
162
172
|
f.close
|
163
173
|
end
|
164
174
|
connection.put_copy_end
|
175
|
+
res = connection.get_result
|
176
|
+
status_code = res.result_status
|
177
|
+
if status_code != 1
|
178
|
+
raise Error.new status_code, res.res_status(status_code), res.result_error_message
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
class Error < Exception
|
183
|
+
attr_accessor :code, :constant, :error, :context
|
184
|
+
|
185
|
+
def initialize(code, constant, message)
|
186
|
+
@code = code
|
187
|
+
@constant = constant
|
188
|
+
@error, @context = message.split(/\n/)
|
189
|
+
end
|
190
|
+
|
191
|
+
def to_s
|
192
|
+
@error
|
193
|
+
end
|
165
194
|
end
|
166
195
|
end
|
167
196
|
end
|
data/lib/theman/version.rb
CHANGED
data/spec/agency_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe Theman::Agency, "sed chomp" do
|
|
6
6
|
csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_two.csv'))
|
7
7
|
|
8
8
|
agent = ::Theman::Agency.new conn, csv do |agent|
|
9
|
-
agent.
|
9
|
+
agent.chop 15
|
10
10
|
end
|
11
11
|
|
12
12
|
@model = Theman::Object.new(agent.table_name, ActiveRecord::Base)
|
@@ -123,7 +123,7 @@ describe Theman::Agency, "procedural" do
|
|
123
123
|
it "should be able to be called procedural" do
|
124
124
|
smith = ::Theman::Agency.new @conn, @csv
|
125
125
|
smith.datestyle "European"
|
126
|
-
smith.
|
126
|
+
smith.chop 15
|
127
127
|
smith.nulls /"XXXX"/
|
128
128
|
|
129
129
|
smith.table do |t|
|
@@ -244,3 +244,18 @@ describe Theman::Agency, "data types" do
|
|
244
244
|
end
|
245
245
|
end
|
246
246
|
end
|
247
|
+
|
248
|
+
describe Theman::Agency, "data types" do
|
249
|
+
before do
|
250
|
+
@conn = ActiveRecord::Base.connection.raw_connection
|
251
|
+
@csv = File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec', 'fixtures', 'temp_one.csv'))
|
252
|
+
end
|
253
|
+
|
254
|
+
it "should raise an error if the columns are wrong" do
|
255
|
+
agent = ::Theman::Agency.new @conn, @csv
|
256
|
+
agent.table do |t|
|
257
|
+
t.date :column_not_in_csv
|
258
|
+
end
|
259
|
+
lambda{ @agent.create! }.should raise_error
|
260
|
+
end
|
261
|
+
end
|
metadata
CHANGED
@@ -1,13 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: theman
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
prerelease:
|
4
|
+
prerelease: false
|
6
5
|
segments:
|
7
6
|
- 0
|
8
7
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
8
|
+
- 3
|
9
|
+
version: 0.1.3
|
11
10
|
platform: ruby
|
12
11
|
authors:
|
13
12
|
- Rufus Post
|
@@ -15,7 +14,7 @@ autorequire:
|
|
15
14
|
bindir: bin
|
16
15
|
cert_chain: []
|
17
16
|
|
18
|
-
date: 2011-
|
17
|
+
date: 2011-03-09 00:00:00 +11:00
|
19
18
|
default_executable:
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
@@ -26,7 +25,6 @@ dependencies:
|
|
26
25
|
requirements:
|
27
26
|
- - ">="
|
28
27
|
- !ruby/object:Gem::Version
|
29
|
-
hash: 23
|
30
28
|
segments:
|
31
29
|
- 1
|
32
30
|
- 0
|
@@ -42,7 +40,6 @@ dependencies:
|
|
42
40
|
requirements:
|
43
41
|
- - ">="
|
44
42
|
- !ruby/object:Gem::Version
|
45
|
-
hash: 15
|
46
43
|
segments:
|
47
44
|
- 2
|
48
45
|
- 0
|
@@ -58,7 +55,6 @@ dependencies:
|
|
58
55
|
requirements:
|
59
56
|
- - ">="
|
60
57
|
- !ruby/object:Gem::Version
|
61
|
-
hash: 7
|
62
58
|
segments:
|
63
59
|
- 3
|
64
60
|
- 0
|
@@ -74,7 +70,6 @@ dependencies:
|
|
74
70
|
requirements:
|
75
71
|
- - ">="
|
76
72
|
- !ruby/object:Gem::Version
|
77
|
-
hash: 3
|
78
73
|
segments:
|
79
74
|
- 0
|
80
75
|
version: "0"
|
@@ -88,7 +83,6 @@ dependencies:
|
|
88
83
|
requirements:
|
89
84
|
- - ">="
|
90
85
|
- !ruby/object:Gem::Version
|
91
|
-
hash: 3
|
92
86
|
segments:
|
93
87
|
- 0
|
94
88
|
version: "0"
|
@@ -143,7 +137,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
143
137
|
requirements:
|
144
138
|
- - ">="
|
145
139
|
- !ruby/object:Gem::Version
|
146
|
-
hash: 3
|
147
140
|
segments:
|
148
141
|
- 0
|
149
142
|
version: "0"
|
@@ -152,7 +145,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
152
145
|
requirements:
|
153
146
|
- - ">="
|
154
147
|
- !ruby/object:Gem::Version
|
155
|
-
hash: 23
|
156
148
|
segments:
|
157
149
|
- 1
|
158
150
|
- 3
|
@@ -161,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
161
153
|
requirements: []
|
162
154
|
|
163
155
|
rubyforge_project: theman
|
164
|
-
rubygems_version: 1.
|
156
|
+
rubygems_version: 1.3.7
|
165
157
|
signing_key:
|
166
158
|
specification_version: 3
|
167
159
|
summary: PostgreSQL AR temporary table generator using PostgreSQL COPY
|