postgres-copy 0.3.4 → 0.3.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.rspec +2 -0
- data/Gemfile +2 -2
- data/Gemfile.lock +72 -56
- data/README.md +76 -5
- data/Rakefile +1 -2
- data/VERSION +1 -1
- data/lib/postgres-copy/active_record.rb +13 -7
- data/lib/postgres-copy/csv_responder.rb +3 -3
- data/lib/postgres-copy/zip_responder.rb +1 -1
- data/postgres-copy.gemspec +16 -14
- data/spec/fixtures/comma_with_header.csv +2 -0
- data/spec/pg_copy_from_spec.rb +16 -15
- data/spec/pg_copy_to_spec.rb +29 -15
- metadata +75 -79
data/.rspec
ADDED
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,66 +1,76 @@
|
|
1
1
|
GEM
|
2
2
|
remote: http://rubygems.org/
|
3
3
|
specs:
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
rack (~> 1.
|
15
|
-
rack-mount (~> 0.
|
16
|
-
rack-test (~> 0.
|
17
|
-
|
18
|
-
activemodel (3.
|
19
|
-
activesupport (= 3.
|
20
|
-
builder (~>
|
21
|
-
i18n (~> 0.
|
22
|
-
activerecord (3.
|
23
|
-
activemodel (= 3.
|
24
|
-
activesupport (= 3.
|
25
|
-
arel (~> 2.
|
26
|
-
tzinfo (~> 0.3.
|
27
|
-
activeresource (3.
|
28
|
-
activemodel (= 3.
|
29
|
-
activesupport (= 3.
|
30
|
-
activesupport (3.
|
31
|
-
|
32
|
-
|
4
|
+
actionmailer (3.1.3)
|
5
|
+
actionpack (= 3.1.3)
|
6
|
+
mail (~> 2.3.0)
|
7
|
+
actionpack (3.1.3)
|
8
|
+
activemodel (= 3.1.3)
|
9
|
+
activesupport (= 3.1.3)
|
10
|
+
builder (~> 3.0.0)
|
11
|
+
erubis (~> 2.7.0)
|
12
|
+
i18n (~> 0.6)
|
13
|
+
rack (~> 1.3.5)
|
14
|
+
rack-cache (~> 1.1)
|
15
|
+
rack-mount (~> 0.8.2)
|
16
|
+
rack-test (~> 0.6.1)
|
17
|
+
sprockets (~> 2.0.3)
|
18
|
+
activemodel (3.1.3)
|
19
|
+
activesupport (= 3.1.3)
|
20
|
+
builder (~> 3.0.0)
|
21
|
+
i18n (~> 0.6)
|
22
|
+
activerecord (3.1.3)
|
23
|
+
activemodel (= 3.1.3)
|
24
|
+
activesupport (= 3.1.3)
|
25
|
+
arel (~> 2.2.1)
|
26
|
+
tzinfo (~> 0.3.29)
|
27
|
+
activeresource (3.1.3)
|
28
|
+
activemodel (= 3.1.3)
|
29
|
+
activesupport (= 3.1.3)
|
30
|
+
activesupport (3.1.3)
|
31
|
+
multi_json (~> 1.0)
|
32
|
+
arel (2.2.1)
|
33
|
+
builder (3.0.0)
|
33
34
|
diff-lcs (1.1.2)
|
34
|
-
erubis (2.
|
35
|
-
|
36
|
-
i18n (0.
|
37
|
-
|
38
|
-
|
35
|
+
erubis (2.7.0)
|
36
|
+
hike (1.2.1)
|
37
|
+
i18n (0.6.0)
|
38
|
+
json (1.6.4)
|
39
|
+
mail (2.3.0)
|
39
40
|
i18n (>= 0.4.0)
|
40
41
|
mime-types (~> 1.16)
|
41
42
|
treetop (~> 1.4.8)
|
42
|
-
mime-types (1.
|
43
|
+
mime-types (1.17.2)
|
44
|
+
multi_json (1.0.4)
|
43
45
|
pg (0.11.0)
|
44
|
-
polyglot (0.3.
|
45
|
-
rack (1.
|
46
|
-
rack-
|
46
|
+
polyglot (0.3.3)
|
47
|
+
rack (1.3.6)
|
48
|
+
rack-cache (1.1)
|
49
|
+
rack (>= 0.4)
|
50
|
+
rack-mount (0.8.3)
|
47
51
|
rack (>= 1.0.0)
|
48
|
-
rack-
|
52
|
+
rack-ssl (1.3.2)
|
53
|
+
rack
|
54
|
+
rack-test (0.6.1)
|
49
55
|
rack (>= 1.0)
|
50
|
-
rails (3.
|
51
|
-
actionmailer (= 3.
|
52
|
-
actionpack (= 3.
|
53
|
-
activerecord (= 3.
|
54
|
-
activeresource (= 3.
|
55
|
-
activesupport (= 3.
|
56
|
+
rails (3.1.3)
|
57
|
+
actionmailer (= 3.1.3)
|
58
|
+
actionpack (= 3.1.3)
|
59
|
+
activerecord (= 3.1.3)
|
60
|
+
activeresource (= 3.1.3)
|
61
|
+
activesupport (= 3.1.3)
|
56
62
|
bundler (~> 1.0)
|
57
|
-
railties (= 3.
|
58
|
-
railties (3.
|
59
|
-
actionpack (= 3.
|
60
|
-
activesupport (= 3.
|
63
|
+
railties (= 3.1.3)
|
64
|
+
railties (3.1.3)
|
65
|
+
actionpack (= 3.1.3)
|
66
|
+
activesupport (= 3.1.3)
|
67
|
+
rack-ssl (~> 1.3.2)
|
61
68
|
rake (>= 0.8.7)
|
62
|
-
|
63
|
-
|
69
|
+
rdoc (~> 3.4)
|
70
|
+
thor (~> 0.14.6)
|
71
|
+
rake (0.8.7)
|
72
|
+
rdoc (3.12)
|
73
|
+
json (~> 1.4)
|
64
74
|
rspec (2.6.0)
|
65
75
|
rspec-core (~> 2.6.0)
|
66
76
|
rspec-expectations (~> 2.6.0)
|
@@ -69,10 +79,16 @@ GEM
|
|
69
79
|
rspec-expectations (2.6.0)
|
70
80
|
diff-lcs (~> 1.1.2)
|
71
81
|
rspec-mocks (2.6.0)
|
82
|
+
sprockets (2.0.3)
|
83
|
+
hike (~> 1.2)
|
84
|
+
rack (~> 1.0)
|
85
|
+
tilt (~> 1.1, != 1.3.0)
|
72
86
|
thor (0.14.6)
|
73
|
-
|
87
|
+
tilt (1.3.3)
|
88
|
+
treetop (1.4.10)
|
89
|
+
polyglot
|
74
90
|
polyglot (>= 0.3.1)
|
75
|
-
tzinfo (0.3.
|
91
|
+
tzinfo (0.3.31)
|
76
92
|
|
77
93
|
PLATFORMS
|
78
94
|
ruby
|
@@ -81,6 +97,6 @@ DEPENDENCIES
|
|
81
97
|
activerecord
|
82
98
|
pg
|
83
99
|
rails
|
84
|
-
rake
|
85
|
-
rspec (~> 2.
|
86
|
-
rspec-core (~> 2.
|
100
|
+
rake (= 0.8.7)
|
101
|
+
rspec (~> 2.6)
|
102
|
+
rspec-core (~> 2.6)
|
data/README.md
CHANGED
@@ -1,19 +1,90 @@
|
|
1
1
|
# postgres-copy
|
2
2
|
|
3
3
|
This Gem will enable your AR models to use the PostgreSQL COPY command to import/export data in CSV format.
|
4
|
-
|
4
|
+
If you need to tranfer data between a PostgreSQL database and CSV files, the PostgreSQL native CSV parser
|
5
|
+
will give you a greater performance than using the ruby CSV+INSERT commands.
|
6
|
+
I have not found time to make accurate benchmarks, but in the use scenario where I have developed the gem
|
7
|
+
I have had a four-fold performance gain.
|
8
|
+
This gem was written having the Rails framework in mind, I think it could work only with active-record,
|
9
|
+
but I will assume in this README that you are using Rails.
|
5
10
|
|
6
11
|
## Install
|
7
12
|
|
8
|
-
|
13
|
+
Put it in your Gemfile
|
14
|
+
|
15
|
+
gem 'postgres-copy'
|
16
|
+
|
17
|
+
Run the bundle command
|
18
|
+
|
19
|
+
bundle
|
9
20
|
|
10
21
|
## Usage
|
11
22
|
|
12
|
-
The gem will add two aditiontal class methods to ActiveRecord::Base
|
23
|
+
The gem will add two aditiontal class methods to ActiveRecord::Base:
|
24
|
+
|
25
|
+
* pg_copy_to
|
26
|
+
* pg_copy_to_string
|
27
|
+
* pg_copy_from
|
28
|
+
|
29
|
+
### Using pg_copy_to and pg_copy_to_string
|
30
|
+
|
31
|
+
You can go to the rails console and try some cool things first.
|
32
|
+
The first and most basic use case, let's copy the enteire content of a database table to a CSV file on the database server disk.
|
33
|
+
Assuming we have a users table and a User AR model:
|
34
|
+
|
35
|
+
User.pg_copy_to '/tmp/users.csv'
|
36
|
+
|
37
|
+
This will execute in the database the command:
|
38
|
+
|
39
|
+
COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
40
|
+
|
41
|
+
Remark that the file will be created in the database server disk.
|
42
|
+
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
43
|
+
In this case you can pass a block and retrieve the generated lines and then write them to a file:
|
44
|
+
|
45
|
+
File.open('/tmp/users.csv', 'w') do |f|
|
46
|
+
User.pg_copy_to do |line|
|
47
|
+
f.write line
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
Or, if you have enough memory, you can read all table contents to a string using .pg_copy_to_string
|
52
|
+
|
53
|
+
puts User.pg_copy_to_string
|
54
|
+
|
55
|
+
Another insteresting feature of pg_copy_to is that it uses the scoped relation, it means that you can use ARel
|
56
|
+
operations to generate different CSV files according to your needs.
|
57
|
+
Assuming we want to generate a file only with the names of users 1, 2 and 3:
|
58
|
+
|
59
|
+
User.select("name").where(:id => [1,2,3]).pg_copy_to "/tmp/users.csv"
|
60
|
+
|
61
|
+
Which will generate the following SQL command:
|
62
|
+
|
63
|
+
COPY (SELECT name FROM "users" WHERE "users"."id" IN (1, 2, 3)) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
64
|
+
|
65
|
+
### Using pg_copy_from
|
66
|
+
|
67
|
+
Now, if you want to copy data from a CSV file into the database, you can use the pg_copy_from method.
|
68
|
+
It will allow you to copy data from an arbritary IO object or from a file in the database server (when you pass the path as string).
|
69
|
+
Let's first copy from a file in the database server, assuming again that we have a users table and
|
70
|
+
that we are in the Rails console:
|
71
|
+
|
72
|
+
User.pg_copy_from "/tmp/users.csv"
|
73
|
+
|
74
|
+
This command will use the headers in the CSV file as fields of the target table, so beware to always have a header in the files you want to import.
|
75
|
+
If the column names in the CSV header do not match the field names of the target table, you can pass a map in the options parameter.
|
76
|
+
|
77
|
+
User.pg_copy_from "/tmp/users.csv", :map => {'name' => 'first_name'}
|
78
|
+
|
79
|
+
In the above example the header name in the CSV file will be mapped to the field called first_name in the users table.
|
80
|
+
You can also manipulate and modify the values of the file being imported before they enter into the database using a block:
|
13
81
|
|
14
|
-
|
15
|
-
|
82
|
+
User.pg_copy_from "/tmp/users.csv" do |row|
|
83
|
+
row[0] = "fixed string"
|
84
|
+
end
|
16
85
|
|
86
|
+
The above extample will always change the value of the first column to "fixed string" before storing it into the database.
|
87
|
+
For each iteration of the block row receives an array with the same order as the columns in the CSV file.
|
17
88
|
|
18
89
|
## Note on Patches/Pull Requests
|
19
90
|
|
data/Rakefile
CHANGED
@@ -22,7 +22,6 @@ end
|
|
22
22
|
require "rspec/core/rake_task" # RSpec 2.0
|
23
23
|
RSpec::Core::RakeTask.new(:spec) do |spec|
|
24
24
|
spec.pattern = 'spec/*_spec.rb'
|
25
|
-
spec.rspec_opts = ["--backtrace", "--color", "--format progress", "-r ./spec/spec_helper.rb"]
|
26
25
|
end
|
27
26
|
task :default => :spec
|
28
27
|
|
@@ -30,7 +29,7 @@ require 'rake/rdoctask'
|
|
30
29
|
Rake::RDocTask.new do |rdoc|
|
31
30
|
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
32
31
|
rdoc.rdoc_dir = 'rdoc'
|
33
|
-
rdoc.title = "
|
32
|
+
rdoc.title = "postgres-copy #{version}"
|
34
33
|
rdoc.rdoc_files.include('README*')
|
35
34
|
rdoc.rdoc_files.include('lib/**/*.rb')
|
36
35
|
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.6
|
@@ -1,11 +1,13 @@
|
|
1
1
|
module ActiveRecord
|
2
2
|
class Base
|
3
|
-
|
3
|
+
# Copy data to a file passed as a string (the file path) or to lines that are passed to a block
|
4
|
+
def self.pg_copy_to path = nil, options = {}
|
5
|
+
options = {:delimiter => ","}.merge(options)
|
4
6
|
if path
|
5
7
|
raise "You have to choose between exporting to a file or receiving the lines inside a block" if block_given?
|
6
|
-
connection.execute "COPY (#{self.scoped.to_sql}) TO #{sanitize(path)} WITH DELIMITER '
|
8
|
+
connection.execute "COPY (#{self.scoped.to_sql}) TO #{sanitize(path)} WITH DELIMITER '#{options[:delimiter]}' CSV HEADER"
|
7
9
|
else
|
8
|
-
connection.execute "COPY (#{self.scoped.to_sql}) TO STDOUT WITH DELIMITER '
|
10
|
+
connection.execute "COPY (#{self.scoped.to_sql}) TO STDOUT WITH DELIMITER '#{options[:delimiter]}' CSV HEADER"
|
9
11
|
while line = connection.raw_connection.get_copy_data do
|
10
12
|
yield(line) if block_given?
|
11
13
|
end
|
@@ -13,15 +15,19 @@ module ActiveRecord
|
|
13
15
|
return self
|
14
16
|
end
|
15
17
|
|
16
|
-
|
17
|
-
|
18
|
+
# Copy all data to a single string
|
19
|
+
def self.pg_copy_to_string options = {}
|
18
20
|
data = ''
|
19
|
-
self.pg_copy_to{|l| data += l }
|
21
|
+
self.pg_copy_to(nil, options){|l| data += l }
|
20
22
|
data
|
21
23
|
end
|
22
24
|
|
25
|
+
# Copy data from a CSV that can be passed as a string (the file path) or as an IO object.
|
26
|
+
# * You can change the default delimiter passing delimiter: '' in the options hash
|
27
|
+
# * You can map fields from the file to different fields in the table using a map in the options hash
|
28
|
+
# * For further details on usage take a look at the README.md
|
23
29
|
def self.pg_copy_from path_or_io, options = {}
|
24
|
-
options = {:delimiter => "
|
30
|
+
options = {:delimiter => ","}.merge(options)
|
25
31
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
26
32
|
# The first line should be always the HEADER.
|
27
33
|
line = io.gets
|
@@ -1,7 +1,7 @@
|
|
1
1
|
class ActionController::Responder
|
2
2
|
def to_csv
|
3
|
-
name = "#{controller.
|
3
|
+
name = "#{controller.resource.class.name.downcase}_#{Time.now.to_i}"
|
4
4
|
|
5
|
-
return controller.send_data controller.send(:end_of_association_chain).pg_copy_to_string, :filename => "/tmp/#{name}.csv", :type => "
|
5
|
+
return controller.send_data controller.send(:end_of_association_chain).pg_copy_to_string, :filename => "/tmp/#{name}.csv", :type => "text/csv", :disposition => 'inline'
|
6
6
|
end
|
7
|
-
end
|
7
|
+
end
|
@@ -3,7 +3,7 @@ Mime::Type.register 'application/zip', :zip
|
|
3
3
|
|
4
4
|
class ActionController::Responder
|
5
5
|
def to_zip
|
6
|
-
name = "#{controller.
|
6
|
+
name = "#{controller.resource.class.name.downcase}_#{Time.now.to_i}"
|
7
7
|
controller.send(:end_of_association_chain).pg_copy_to "/tmp/#{name}.csv"
|
8
8
|
Dir.chdir('/tmp') do
|
9
9
|
`zip #{name} #{name}.csv`
|
data/postgres-copy.gemspec
CHANGED
@@ -4,20 +4,21 @@
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
|
-
s.name =
|
8
|
-
s.version = "0.3.
|
7
|
+
s.name = "postgres-copy"
|
8
|
+
s.version = "0.3.6"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Diogo Biazus"]
|
12
|
-
s.date =
|
13
|
-
s.description =
|
14
|
-
s.email =
|
12
|
+
s.date = "2011-12-30"
|
13
|
+
s.description = "Now you can use the super fast COPY for import/export data directly from your AR models."
|
14
|
+
s.email = "diogob@gmail.com"
|
15
15
|
s.extra_rdoc_files = [
|
16
16
|
"LICENSE",
|
17
17
|
"README.md"
|
18
18
|
]
|
19
19
|
s.files = [
|
20
20
|
".document",
|
21
|
+
".rspec",
|
21
22
|
"Gemfile",
|
22
23
|
"Gemfile.lock",
|
23
24
|
"LICENSE",
|
@@ -29,6 +30,7 @@ Gem::Specification.new do |s|
|
|
29
30
|
"lib/postgres-copy/csv_responder.rb",
|
30
31
|
"lib/postgres-copy/zip_responder.rb",
|
31
32
|
"postgres-copy.gemspec",
|
33
|
+
"spec/fixtures/comma_with_header.csv",
|
32
34
|
"spec/fixtures/extra_field.rb",
|
33
35
|
"spec/fixtures/reserved_word_model.rb",
|
34
36
|
"spec/fixtures/reserved_words.csv",
|
@@ -46,10 +48,10 @@ Gem::Specification.new do |s|
|
|
46
48
|
"spec/spec.opts",
|
47
49
|
"spec/spec_helper.rb"
|
48
50
|
]
|
49
|
-
s.homepage =
|
51
|
+
s.homepage = "http://github.com/diogob/postgres-copy"
|
50
52
|
s.require_paths = ["lib"]
|
51
|
-
s.rubygems_version =
|
52
|
-
s.summary =
|
53
|
+
s.rubygems_version = "1.8.10"
|
54
|
+
s.summary = "Put COPY command functionality in ActiveRecord's model class"
|
53
55
|
|
54
56
|
if s.respond_to? :specification_version then
|
55
57
|
s.specification_version = 3
|
@@ -59,8 +61,8 @@ Gem::Specification.new do |s|
|
|
59
61
|
s.add_runtime_dependency(%q<activerecord>, [">= 0"])
|
60
62
|
s.add_runtime_dependency(%q<rails>, [">= 0"])
|
61
63
|
s.add_runtime_dependency(%q<rake>, ["= 0.8.7"])
|
62
|
-
s.add_runtime_dependency(%q<rspec>, ["~> 2.
|
63
|
-
s.add_runtime_dependency(%q<rspec-core>, ["~> 2.
|
64
|
+
s.add_runtime_dependency(%q<rspec>, ["~> 2.6"])
|
65
|
+
s.add_runtime_dependency(%q<rspec-core>, ["~> 2.6"])
|
64
66
|
s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
|
65
67
|
s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
|
66
68
|
else
|
@@ -68,8 +70,8 @@ Gem::Specification.new do |s|
|
|
68
70
|
s.add_dependency(%q<activerecord>, [">= 0"])
|
69
71
|
s.add_dependency(%q<rails>, [">= 0"])
|
70
72
|
s.add_dependency(%q<rake>, ["= 0.8.7"])
|
71
|
-
s.add_dependency(%q<rspec>, ["~> 2.
|
72
|
-
s.add_dependency(%q<rspec-core>, ["~> 2.
|
73
|
+
s.add_dependency(%q<rspec>, ["~> 2.6"])
|
74
|
+
s.add_dependency(%q<rspec-core>, ["~> 2.6"])
|
73
75
|
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
74
76
|
s.add_dependency(%q<activerecord>, [">= 3.0.0"])
|
75
77
|
end
|
@@ -78,8 +80,8 @@ Gem::Specification.new do |s|
|
|
78
80
|
s.add_dependency(%q<activerecord>, [">= 0"])
|
79
81
|
s.add_dependency(%q<rails>, [">= 0"])
|
80
82
|
s.add_dependency(%q<rake>, ["= 0.8.7"])
|
81
|
-
s.add_dependency(%q<rspec>, ["~> 2.
|
82
|
-
s.add_dependency(%q<rspec-core>, ["~> 2.
|
83
|
+
s.add_dependency(%q<rspec>, ["~> 2.6"])
|
84
|
+
s.add_dependency(%q<rspec-core>, ["~> 2.6"])
|
83
85
|
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
84
86
|
s.add_dependency(%q<activerecord>, [">= 3.0.0"])
|
85
87
|
end
|
data/spec/pg_copy_from_spec.rb
CHANGED
@@ -9,12 +9,12 @@ describe "COPY FROM" do
|
|
9
9
|
end
|
10
10
|
|
11
11
|
it "should import from file if path is passed without field_map" do
|
12
|
-
TestModel.pg_copy_from File.expand_path('spec/fixtures/
|
12
|
+
TestModel.pg_copy_from File.expand_path('spec/fixtures/comma_with_header.csv')
|
13
13
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
14
14
|
end
|
15
15
|
|
16
16
|
it "should import from IO without field_map" do
|
17
|
-
TestModel.pg_copy_from File.open(File.expand_path('spec/fixtures/
|
17
|
+
TestModel.pg_copy_from File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r')
|
18
18
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
19
19
|
end
|
20
20
|
|
@@ -29,14 +29,14 @@ describe "COPY FROM" do
|
|
29
29
|
end
|
30
30
|
|
31
31
|
it "should import and allow changes in block" do
|
32
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/
|
32
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r')) do |row|
|
33
33
|
row[1] = 'changed this data'
|
34
34
|
end
|
35
35
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'changed this data'}]
|
36
36
|
end
|
37
37
|
|
38
38
|
it "should import 2 lines and allow changes in block" do
|
39
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.csv'), 'r')) do |row|
|
39
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.csv'), 'r'), :delimiter => "\t") do |row|
|
40
40
|
row[1] = 'changed this data'
|
41
41
|
end
|
42
42
|
TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
|
@@ -44,17 +44,17 @@ describe "COPY FROM" do
|
|
44
44
|
end
|
45
45
|
|
46
46
|
it "should be able to copy from using custom set of columns" do
|
47
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_only_data.csv'), 'r'), :columns => ["data"])
|
47
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_only_data.csv'), 'r'), :delimiter => "\t", :columns => ["data"])
|
48
48
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
49
49
|
end
|
50
50
|
|
51
51
|
it "default set of columns should be all table columns minus [id, created_at, updated_at]" do
|
52
|
-
ExtraField.pg_copy_from(File.open(File.expand_path('spec/fixtures/
|
52
|
+
ExtraField.pg_copy_from(File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r'))
|
53
53
|
ExtraField.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1', 'created_at' => nil, 'updated_at' => nil}]
|
54
54
|
end
|
55
55
|
|
56
56
|
it "should be able to map the header in the file to diferent column names" do
|
57
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_different_header.csv'), 'r'), :map => {'cod' => 'id', 'info' => 'data'})
|
57
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_different_header.csv'), 'r'), :delimiter => "\t", :map => {'cod' => 'id', 'info' => 'data'})
|
58
58
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
59
59
|
end
|
60
60
|
|
@@ -64,19 +64,20 @@ describe "COPY FROM" do
|
|
64
64
|
end
|
65
65
|
|
66
66
|
it "should ignore empty lines" do
|
67
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_extra_line.csv'), 'r'))
|
67
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_extra_line.csv'), 'r'), :delimiter => "\t")
|
68
68
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
69
69
|
end
|
70
70
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
71
|
+
#we should implement this later
|
72
|
+
#it "should raise error in malformed files" do
|
73
|
+
#lambda do
|
74
|
+
#TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_error.csv'), 'r'))
|
75
|
+
#end.should raise_error
|
76
|
+
#TestModel.order(:id).all.map{|r| r.attributes}.should == []
|
77
|
+
#end
|
77
78
|
|
78
79
|
it "should copy from even when table fields need identifier quoting" do
|
79
|
-
ReservedWordModel.pg_copy_from File.expand_path('spec/fixtures/reserved_words.csv')
|
80
|
+
ReservedWordModel.pg_copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
|
80
81
|
ReservedWordModel.order(:id).all.map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
|
81
82
|
end
|
82
83
|
end
|
data/spec/pg_copy_to_spec.rb
CHANGED
@@ -9,27 +9,41 @@ describe "COPY TO" do
|
|
9
9
|
TestModel.create :data => 'test data 1'
|
10
10
|
end
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
TestModel.
|
15
|
-
|
16
|
-
|
12
|
+
describe ".pg_copy_to_string" do
|
13
|
+
context "with no options" do
|
14
|
+
subject{ TestModel.pg_copy_to_string }
|
15
|
+
it{ should == File.open('spec/fixtures/comma_with_header.csv', 'r').read }
|
16
|
+
end
|
17
|
+
|
18
|
+
context "with tab as delimiter" do
|
19
|
+
subject{ TestModel.pg_copy_to_string :delimiter => "\t" }
|
20
|
+
it{ should == File.open('spec/fixtures/tab_with_header.csv', 'r').read }
|
17
21
|
end
|
18
22
|
end
|
19
23
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
24
|
+
describe ".pg_copy_to" do
|
25
|
+
it "should copy and pass data to block if block is given and no path is passed" do
|
26
|
+
File.open('spec/fixtures/comma_with_header.csv', 'r') do |f|
|
27
|
+
TestModel.pg_copy_to do |row|
|
28
|
+
row.should == f.readline
|
29
|
+
end
|
25
30
|
end
|
26
31
|
end
|
27
|
-
end
|
28
32
|
|
29
|
-
|
30
|
-
|
31
|
-
|
33
|
+
it "should copy to disk if block is not given and a path is passed" do
|
34
|
+
TestModel.pg_copy_to '/tmp/export.csv'
|
35
|
+
File.open('spec/fixtures/comma_with_header.csv', 'r') do |fixture|
|
36
|
+
File.open('/tmp/export.csv', 'r') do |result|
|
37
|
+
result.read.should == fixture.read
|
38
|
+
end
|
32
39
|
end
|
33
|
-
end
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should raise exception if I pass a path and a block simultaneously" do
|
43
|
+
lambda do
|
44
|
+
TestModel.pg_copy_to('/tmp/bogus_path') do |row|
|
45
|
+
end
|
46
|
+
end.should raise_error
|
47
|
+
end
|
34
48
|
end
|
35
49
|
end
|
metadata
CHANGED
@@ -1,116 +1,115 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres-copy
|
3
|
-
version: !ruby/object:Gem::Version
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.6
|
4
5
|
prerelease:
|
5
|
-
version: 0.3.4
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Diogo Biazus
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
- !ruby/object:Gem::Dependency
|
12
|
+
date: 2011-12-30 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
16
15
|
name: pg
|
17
|
-
requirement: &
|
16
|
+
requirement: &70230257119200 !ruby/object:Gem::Requirement
|
18
17
|
none: false
|
19
|
-
requirements:
|
20
|
-
- -
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version:
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
23
22
|
type: :runtime
|
24
23
|
prerelease: false
|
25
|
-
version_requirements: *
|
26
|
-
- !ruby/object:Gem::Dependency
|
24
|
+
version_requirements: *70230257119200
|
25
|
+
- !ruby/object:Gem::Dependency
|
27
26
|
name: activerecord
|
28
|
-
requirement: &
|
27
|
+
requirement: &70230257118460 !ruby/object:Gem::Requirement
|
29
28
|
none: false
|
30
|
-
requirements:
|
31
|
-
- -
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version:
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
34
33
|
type: :runtime
|
35
34
|
prerelease: false
|
36
|
-
version_requirements: *
|
37
|
-
- !ruby/object:Gem::Dependency
|
35
|
+
version_requirements: *70230257118460
|
36
|
+
- !ruby/object:Gem::Dependency
|
38
37
|
name: rails
|
39
|
-
requirement: &
|
38
|
+
requirement: &70230257117880 !ruby/object:Gem::Requirement
|
40
39
|
none: false
|
41
|
-
requirements:
|
42
|
-
- -
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
-
version:
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
45
44
|
type: :runtime
|
46
45
|
prerelease: false
|
47
|
-
version_requirements: *
|
48
|
-
- !ruby/object:Gem::Dependency
|
46
|
+
version_requirements: *70230257117880
|
47
|
+
- !ruby/object:Gem::Dependency
|
49
48
|
name: rake
|
50
|
-
requirement: &
|
49
|
+
requirement: &70230257117360 !ruby/object:Gem::Requirement
|
51
50
|
none: false
|
52
|
-
requirements:
|
53
|
-
- -
|
54
|
-
- !ruby/object:Gem::Version
|
51
|
+
requirements:
|
52
|
+
- - =
|
53
|
+
- !ruby/object:Gem::Version
|
55
54
|
version: 0.8.7
|
56
55
|
type: :runtime
|
57
56
|
prerelease: false
|
58
|
-
version_requirements: *
|
59
|
-
- !ruby/object:Gem::Dependency
|
57
|
+
version_requirements: *70230257117360
|
58
|
+
- !ruby/object:Gem::Dependency
|
60
59
|
name: rspec
|
61
|
-
requirement: &
|
60
|
+
requirement: &70230257116780 !ruby/object:Gem::Requirement
|
62
61
|
none: false
|
63
|
-
requirements:
|
62
|
+
requirements:
|
64
63
|
- - ~>
|
65
|
-
- !ruby/object:Gem::Version
|
66
|
-
version:
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '2.6'
|
67
66
|
type: :runtime
|
68
67
|
prerelease: false
|
69
|
-
version_requirements: *
|
70
|
-
- !ruby/object:Gem::Dependency
|
68
|
+
version_requirements: *70230257116780
|
69
|
+
- !ruby/object:Gem::Dependency
|
71
70
|
name: rspec-core
|
72
|
-
requirement: &
|
71
|
+
requirement: &70230257116200 !ruby/object:Gem::Requirement
|
73
72
|
none: false
|
74
|
-
requirements:
|
73
|
+
requirements:
|
75
74
|
- - ~>
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version:
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '2.6'
|
78
77
|
type: :runtime
|
79
78
|
prerelease: false
|
80
|
-
version_requirements: *
|
81
|
-
- !ruby/object:Gem::Dependency
|
79
|
+
version_requirements: *70230257116200
|
80
|
+
- !ruby/object:Gem::Dependency
|
82
81
|
name: rspec
|
83
|
-
requirement: &
|
82
|
+
requirement: &70230257115420 !ruby/object:Gem::Requirement
|
84
83
|
none: false
|
85
|
-
requirements:
|
86
|
-
- -
|
87
|
-
- !ruby/object:Gem::Version
|
84
|
+
requirements:
|
85
|
+
- - ! '>='
|
86
|
+
- !ruby/object:Gem::Version
|
88
87
|
version: 1.2.9
|
89
88
|
type: :development
|
90
89
|
prerelease: false
|
91
|
-
version_requirements: *
|
92
|
-
- !ruby/object:Gem::Dependency
|
90
|
+
version_requirements: *70230257115420
|
91
|
+
- !ruby/object:Gem::Dependency
|
93
92
|
name: activerecord
|
94
|
-
requirement: &
|
93
|
+
requirement: &70230257113880 !ruby/object:Gem::Requirement
|
95
94
|
none: false
|
96
|
-
requirements:
|
97
|
-
- -
|
98
|
-
- !ruby/object:Gem::Version
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
99
98
|
version: 3.0.0
|
100
99
|
type: :runtime
|
101
100
|
prerelease: false
|
102
|
-
version_requirements: *
|
103
|
-
description: Now you can use the super fast COPY for import/export data directly from
|
101
|
+
version_requirements: *70230257113880
|
102
|
+
description: Now you can use the super fast COPY for import/export data directly from
|
103
|
+
your AR models.
|
104
104
|
email: diogob@gmail.com
|
105
105
|
executables: []
|
106
|
-
|
107
106
|
extensions: []
|
108
|
-
|
109
|
-
extra_rdoc_files:
|
107
|
+
extra_rdoc_files:
|
110
108
|
- LICENSE
|
111
109
|
- README.md
|
112
|
-
files:
|
110
|
+
files:
|
113
111
|
- .document
|
112
|
+
- .rspec
|
114
113
|
- Gemfile
|
115
114
|
- Gemfile.lock
|
116
115
|
- LICENSE
|
@@ -122,6 +121,7 @@ files:
|
|
122
121
|
- lib/postgres-copy/csv_responder.rb
|
123
122
|
- lib/postgres-copy/zip_responder.rb
|
124
123
|
- postgres-copy.gemspec
|
124
|
+
- spec/fixtures/comma_with_header.csv
|
125
125
|
- spec/fixtures/extra_field.rb
|
126
126
|
- spec/fixtures/reserved_word_model.rb
|
127
127
|
- spec/fixtures/reserved_words.csv
|
@@ -140,30 +140,26 @@ files:
|
|
140
140
|
- spec/spec_helper.rb
|
141
141
|
homepage: http://github.com/diogob/postgres-copy
|
142
142
|
licenses: []
|
143
|
-
|
144
143
|
post_install_message:
|
145
144
|
rdoc_options: []
|
146
|
-
|
147
|
-
require_paths:
|
145
|
+
require_paths:
|
148
146
|
- lib
|
149
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
147
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
150
148
|
none: false
|
151
|
-
requirements:
|
152
|
-
- -
|
153
|
-
- !ruby/object:Gem::Version
|
154
|
-
version:
|
155
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ! '>='
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
153
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
156
154
|
none: false
|
157
|
-
requirements:
|
158
|
-
- -
|
159
|
-
- !ruby/object:Gem::Version
|
160
|
-
version:
|
155
|
+
requirements:
|
156
|
+
- - ! '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
161
159
|
requirements: []
|
162
|
-
|
163
160
|
rubyforge_project:
|
164
|
-
rubygems_version: 1.
|
161
|
+
rubygems_version: 1.8.10
|
165
162
|
signing_key:
|
166
163
|
specification_version: 3
|
167
164
|
summary: Put COPY command functionality in ActiveRecord's model class
|
168
165
|
test_files: []
|
169
|
-
|