postgres-copy 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +2 -0
- data/Gemfile +2 -2
- data/Gemfile.lock +72 -56
- data/README.md +76 -5
- data/Rakefile +1 -2
- data/VERSION +1 -1
- data/lib/postgres-copy/active_record.rb +13 -7
- data/lib/postgres-copy/csv_responder.rb +3 -3
- data/lib/postgres-copy/zip_responder.rb +1 -1
- data/postgres-copy.gemspec +16 -14
- data/spec/fixtures/comma_with_header.csv +2 -0
- data/spec/pg_copy_from_spec.rb +16 -15
- data/spec/pg_copy_to_spec.rb +29 -15
- metadata +75 -79
data/.rspec
ADDED
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,66 +1,76 @@
|
|
|
1
1
|
GEM
|
|
2
2
|
remote: http://rubygems.org/
|
|
3
3
|
specs:
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
rack (~> 1.
|
|
15
|
-
rack-mount (~> 0.
|
|
16
|
-
rack-test (~> 0.
|
|
17
|
-
|
|
18
|
-
activemodel (3.
|
|
19
|
-
activesupport (= 3.
|
|
20
|
-
builder (~>
|
|
21
|
-
i18n (~> 0.
|
|
22
|
-
activerecord (3.
|
|
23
|
-
activemodel (= 3.
|
|
24
|
-
activesupport (= 3.
|
|
25
|
-
arel (~> 2.
|
|
26
|
-
tzinfo (~> 0.3.
|
|
27
|
-
activeresource (3.
|
|
28
|
-
activemodel (= 3.
|
|
29
|
-
activesupport (= 3.
|
|
30
|
-
activesupport (3.
|
|
31
|
-
|
|
32
|
-
|
|
4
|
+
actionmailer (3.1.3)
|
|
5
|
+
actionpack (= 3.1.3)
|
|
6
|
+
mail (~> 2.3.0)
|
|
7
|
+
actionpack (3.1.3)
|
|
8
|
+
activemodel (= 3.1.3)
|
|
9
|
+
activesupport (= 3.1.3)
|
|
10
|
+
builder (~> 3.0.0)
|
|
11
|
+
erubis (~> 2.7.0)
|
|
12
|
+
i18n (~> 0.6)
|
|
13
|
+
rack (~> 1.3.5)
|
|
14
|
+
rack-cache (~> 1.1)
|
|
15
|
+
rack-mount (~> 0.8.2)
|
|
16
|
+
rack-test (~> 0.6.1)
|
|
17
|
+
sprockets (~> 2.0.3)
|
|
18
|
+
activemodel (3.1.3)
|
|
19
|
+
activesupport (= 3.1.3)
|
|
20
|
+
builder (~> 3.0.0)
|
|
21
|
+
i18n (~> 0.6)
|
|
22
|
+
activerecord (3.1.3)
|
|
23
|
+
activemodel (= 3.1.3)
|
|
24
|
+
activesupport (= 3.1.3)
|
|
25
|
+
arel (~> 2.2.1)
|
|
26
|
+
tzinfo (~> 0.3.29)
|
|
27
|
+
activeresource (3.1.3)
|
|
28
|
+
activemodel (= 3.1.3)
|
|
29
|
+
activesupport (= 3.1.3)
|
|
30
|
+
activesupport (3.1.3)
|
|
31
|
+
multi_json (~> 1.0)
|
|
32
|
+
arel (2.2.1)
|
|
33
|
+
builder (3.0.0)
|
|
33
34
|
diff-lcs (1.1.2)
|
|
34
|
-
erubis (2.
|
|
35
|
-
|
|
36
|
-
i18n (0.
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
erubis (2.7.0)
|
|
36
|
+
hike (1.2.1)
|
|
37
|
+
i18n (0.6.0)
|
|
38
|
+
json (1.6.4)
|
|
39
|
+
mail (2.3.0)
|
|
39
40
|
i18n (>= 0.4.0)
|
|
40
41
|
mime-types (~> 1.16)
|
|
41
42
|
treetop (~> 1.4.8)
|
|
42
|
-
mime-types (1.
|
|
43
|
+
mime-types (1.17.2)
|
|
44
|
+
multi_json (1.0.4)
|
|
43
45
|
pg (0.11.0)
|
|
44
|
-
polyglot (0.3.
|
|
45
|
-
rack (1.
|
|
46
|
-
rack-
|
|
46
|
+
polyglot (0.3.3)
|
|
47
|
+
rack (1.3.6)
|
|
48
|
+
rack-cache (1.1)
|
|
49
|
+
rack (>= 0.4)
|
|
50
|
+
rack-mount (0.8.3)
|
|
47
51
|
rack (>= 1.0.0)
|
|
48
|
-
rack-
|
|
52
|
+
rack-ssl (1.3.2)
|
|
53
|
+
rack
|
|
54
|
+
rack-test (0.6.1)
|
|
49
55
|
rack (>= 1.0)
|
|
50
|
-
rails (3.
|
|
51
|
-
actionmailer (= 3.
|
|
52
|
-
actionpack (= 3.
|
|
53
|
-
activerecord (= 3.
|
|
54
|
-
activeresource (= 3.
|
|
55
|
-
activesupport (= 3.
|
|
56
|
+
rails (3.1.3)
|
|
57
|
+
actionmailer (= 3.1.3)
|
|
58
|
+
actionpack (= 3.1.3)
|
|
59
|
+
activerecord (= 3.1.3)
|
|
60
|
+
activeresource (= 3.1.3)
|
|
61
|
+
activesupport (= 3.1.3)
|
|
56
62
|
bundler (~> 1.0)
|
|
57
|
-
railties (= 3.
|
|
58
|
-
railties (3.
|
|
59
|
-
actionpack (= 3.
|
|
60
|
-
activesupport (= 3.
|
|
63
|
+
railties (= 3.1.3)
|
|
64
|
+
railties (3.1.3)
|
|
65
|
+
actionpack (= 3.1.3)
|
|
66
|
+
activesupport (= 3.1.3)
|
|
67
|
+
rack-ssl (~> 1.3.2)
|
|
61
68
|
rake (>= 0.8.7)
|
|
62
|
-
|
|
63
|
-
|
|
69
|
+
rdoc (~> 3.4)
|
|
70
|
+
thor (~> 0.14.6)
|
|
71
|
+
rake (0.8.7)
|
|
72
|
+
rdoc (3.12)
|
|
73
|
+
json (~> 1.4)
|
|
64
74
|
rspec (2.6.0)
|
|
65
75
|
rspec-core (~> 2.6.0)
|
|
66
76
|
rspec-expectations (~> 2.6.0)
|
|
@@ -69,10 +79,16 @@ GEM
|
|
|
69
79
|
rspec-expectations (2.6.0)
|
|
70
80
|
diff-lcs (~> 1.1.2)
|
|
71
81
|
rspec-mocks (2.6.0)
|
|
82
|
+
sprockets (2.0.3)
|
|
83
|
+
hike (~> 1.2)
|
|
84
|
+
rack (~> 1.0)
|
|
85
|
+
tilt (~> 1.1, != 1.3.0)
|
|
72
86
|
thor (0.14.6)
|
|
73
|
-
|
|
87
|
+
tilt (1.3.3)
|
|
88
|
+
treetop (1.4.10)
|
|
89
|
+
polyglot
|
|
74
90
|
polyglot (>= 0.3.1)
|
|
75
|
-
tzinfo (0.3.
|
|
91
|
+
tzinfo (0.3.31)
|
|
76
92
|
|
|
77
93
|
PLATFORMS
|
|
78
94
|
ruby
|
|
@@ -81,6 +97,6 @@ DEPENDENCIES
|
|
|
81
97
|
activerecord
|
|
82
98
|
pg
|
|
83
99
|
rails
|
|
84
|
-
rake
|
|
85
|
-
rspec (~> 2.
|
|
86
|
-
rspec-core (~> 2.
|
|
100
|
+
rake (= 0.8.7)
|
|
101
|
+
rspec (~> 2.6)
|
|
102
|
+
rspec-core (~> 2.6)
|
data/README.md
CHANGED
|
@@ -1,19 +1,90 @@
|
|
|
1
1
|
# postgres-copy
|
|
2
2
|
|
|
3
3
|
This Gem will enable your AR models to use the PostgreSQL COPY command to import/export data in CSV format.
|
|
4
|
-
|
|
4
|
+
If you need to tranfer data between a PostgreSQL database and CSV files, the PostgreSQL native CSV parser
|
|
5
|
+
will give you a greater performance than using the ruby CSV+INSERT commands.
|
|
6
|
+
I have not found time to make accurate benchmarks, but in the use scenario where I have developed the gem
|
|
7
|
+
I have had a four-fold performance gain.
|
|
8
|
+
This gem was written having the Rails framework in mind, I think it could work only with active-record,
|
|
9
|
+
but I will assume in this README that you are using Rails.
|
|
5
10
|
|
|
6
11
|
## Install
|
|
7
12
|
|
|
8
|
-
|
|
13
|
+
Put it in your Gemfile
|
|
14
|
+
|
|
15
|
+
gem 'postgres-copy'
|
|
16
|
+
|
|
17
|
+
Run the bundle command
|
|
18
|
+
|
|
19
|
+
bundle
|
|
9
20
|
|
|
10
21
|
## Usage
|
|
11
22
|
|
|
12
|
-
The gem will add two aditiontal class methods to ActiveRecord::Base
|
|
23
|
+
The gem will add two aditiontal class methods to ActiveRecord::Base:
|
|
24
|
+
|
|
25
|
+
* pg_copy_to
|
|
26
|
+
* pg_copy_to_string
|
|
27
|
+
* pg_copy_from
|
|
28
|
+
|
|
29
|
+
### Using pg_copy_to and pg_copy_to_string
|
|
30
|
+
|
|
31
|
+
You can go to the rails console and try some cool things first.
|
|
32
|
+
The first and most basic use case, let's copy the enteire content of a database table to a CSV file on the database server disk.
|
|
33
|
+
Assuming we have a users table and a User AR model:
|
|
34
|
+
|
|
35
|
+
User.pg_copy_to '/tmp/users.csv'
|
|
36
|
+
|
|
37
|
+
This will execute in the database the command:
|
|
38
|
+
|
|
39
|
+
COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
|
40
|
+
|
|
41
|
+
Remark that the file will be created in the database server disk.
|
|
42
|
+
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
|
43
|
+
In this case you can pass a block and retrieve the generated lines and then write them to a file:
|
|
44
|
+
|
|
45
|
+
File.open('/tmp/users.csv', 'w') do |f|
|
|
46
|
+
User.pg_copy_to do |line|
|
|
47
|
+
f.write line
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
Or, if you have enough memory, you can read all table contents to a string using .pg_copy_to_string
|
|
52
|
+
|
|
53
|
+
puts User.pg_copy_to_string
|
|
54
|
+
|
|
55
|
+
Another insteresting feature of pg_copy_to is that it uses the scoped relation, it means that you can use ARel
|
|
56
|
+
operations to generate different CSV files according to your needs.
|
|
57
|
+
Assuming we want to generate a file only with the names of users 1, 2 and 3:
|
|
58
|
+
|
|
59
|
+
User.select("name").where(:id => [1,2,3]).pg_copy_to "/tmp/users.csv"
|
|
60
|
+
|
|
61
|
+
Which will generate the following SQL command:
|
|
62
|
+
|
|
63
|
+
COPY (SELECT name FROM "users" WHERE "users"."id" IN (1, 2, 3)) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
|
64
|
+
|
|
65
|
+
### Using pg_copy_from
|
|
66
|
+
|
|
67
|
+
Now, if you want to copy data from a CSV file into the database, you can use the pg_copy_from method.
|
|
68
|
+
It will allow you to copy data from an arbritary IO object or from a file in the database server (when you pass the path as string).
|
|
69
|
+
Let's first copy from a file in the database server, assuming again that we have a users table and
|
|
70
|
+
that we are in the Rails console:
|
|
71
|
+
|
|
72
|
+
User.pg_copy_from "/tmp/users.csv"
|
|
73
|
+
|
|
74
|
+
This command will use the headers in the CSV file as fields of the target table, so beware to always have a header in the files you want to import.
|
|
75
|
+
If the column names in the CSV header do not match the field names of the target table, you can pass a map in the options parameter.
|
|
76
|
+
|
|
77
|
+
User.pg_copy_from "/tmp/users.csv", :map => {'name' => 'first_name'}
|
|
78
|
+
|
|
79
|
+
In the above example the header name in the CSV file will be mapped to the field called first_name in the users table.
|
|
80
|
+
You can also manipulate and modify the values of the file being imported before they enter into the database using a block:
|
|
13
81
|
|
|
14
|
-
|
|
15
|
-
|
|
82
|
+
User.pg_copy_from "/tmp/users.csv" do |row|
|
|
83
|
+
row[0] = "fixed string"
|
|
84
|
+
end
|
|
16
85
|
|
|
86
|
+
The above extample will always change the value of the first column to "fixed string" before storing it into the database.
|
|
87
|
+
For each iteration of the block row receives an array with the same order as the columns in the CSV file.
|
|
17
88
|
|
|
18
89
|
## Note on Patches/Pull Requests
|
|
19
90
|
|
data/Rakefile
CHANGED
|
@@ -22,7 +22,6 @@ end
|
|
|
22
22
|
require "rspec/core/rake_task" # RSpec 2.0
|
|
23
23
|
RSpec::Core::RakeTask.new(:spec) do |spec|
|
|
24
24
|
spec.pattern = 'spec/*_spec.rb'
|
|
25
|
-
spec.rspec_opts = ["--backtrace", "--color", "--format progress", "-r ./spec/spec_helper.rb"]
|
|
26
25
|
end
|
|
27
26
|
task :default => :spec
|
|
28
27
|
|
|
@@ -30,7 +29,7 @@ require 'rake/rdoctask'
|
|
|
30
29
|
Rake::RDocTask.new do |rdoc|
|
|
31
30
|
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
|
32
31
|
rdoc.rdoc_dir = 'rdoc'
|
|
33
|
-
rdoc.title = "
|
|
32
|
+
rdoc.title = "postgres-copy #{version}"
|
|
34
33
|
rdoc.rdoc_files.include('README*')
|
|
35
34
|
rdoc.rdoc_files.include('lib/**/*.rb')
|
|
36
35
|
end
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.3.
|
|
1
|
+
0.3.6
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
module ActiveRecord
|
|
2
2
|
class Base
|
|
3
|
-
|
|
3
|
+
# Copy data to a file passed as a string (the file path) or to lines that are passed to a block
|
|
4
|
+
def self.pg_copy_to path = nil, options = {}
|
|
5
|
+
options = {:delimiter => ","}.merge(options)
|
|
4
6
|
if path
|
|
5
7
|
raise "You have to choose between exporting to a file or receiving the lines inside a block" if block_given?
|
|
6
|
-
connection.execute "COPY (#{self.scoped.to_sql}) TO #{sanitize(path)} WITH DELIMITER '
|
|
8
|
+
connection.execute "COPY (#{self.scoped.to_sql}) TO #{sanitize(path)} WITH DELIMITER '#{options[:delimiter]}' CSV HEADER"
|
|
7
9
|
else
|
|
8
|
-
connection.execute "COPY (#{self.scoped.to_sql}) TO STDOUT WITH DELIMITER '
|
|
10
|
+
connection.execute "COPY (#{self.scoped.to_sql}) TO STDOUT WITH DELIMITER '#{options[:delimiter]}' CSV HEADER"
|
|
9
11
|
while line = connection.raw_connection.get_copy_data do
|
|
10
12
|
yield(line) if block_given?
|
|
11
13
|
end
|
|
@@ -13,15 +15,19 @@ module ActiveRecord
|
|
|
13
15
|
return self
|
|
14
16
|
end
|
|
15
17
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
# Copy all data to a single string
|
|
19
|
+
def self.pg_copy_to_string options = {}
|
|
18
20
|
data = ''
|
|
19
|
-
self.pg_copy_to{|l| data += l }
|
|
21
|
+
self.pg_copy_to(nil, options){|l| data += l }
|
|
20
22
|
data
|
|
21
23
|
end
|
|
22
24
|
|
|
25
|
+
# Copy data from a CSV that can be passed as a string (the file path) or as an IO object.
|
|
26
|
+
# * You can change the default delimiter passing delimiter: '' in the options hash
|
|
27
|
+
# * You can map fields from the file to different fields in the table using a map in the options hash
|
|
28
|
+
# * For further details on usage take a look at the README.md
|
|
23
29
|
def self.pg_copy_from path_or_io, options = {}
|
|
24
|
-
options = {:delimiter => "
|
|
30
|
+
options = {:delimiter => ","}.merge(options)
|
|
25
31
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
|
26
32
|
# The first line should be always the HEADER.
|
|
27
33
|
line = io.gets
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
class ActionController::Responder
|
|
2
2
|
def to_csv
|
|
3
|
-
name = "#{controller.
|
|
3
|
+
name = "#{controller.resource.class.name.downcase}_#{Time.now.to_i}"
|
|
4
4
|
|
|
5
|
-
return controller.send_data controller.send(:end_of_association_chain).pg_copy_to_string, :filename => "/tmp/#{name}.csv", :type => "
|
|
5
|
+
return controller.send_data controller.send(:end_of_association_chain).pg_copy_to_string, :filename => "/tmp/#{name}.csv", :type => "text/csv", :disposition => 'inline'
|
|
6
6
|
end
|
|
7
|
-
end
|
|
7
|
+
end
|
|
@@ -3,7 +3,7 @@ Mime::Type.register 'application/zip', :zip
|
|
|
3
3
|
|
|
4
4
|
class ActionController::Responder
|
|
5
5
|
def to_zip
|
|
6
|
-
name = "#{controller.
|
|
6
|
+
name = "#{controller.resource.class.name.downcase}_#{Time.now.to_i}"
|
|
7
7
|
controller.send(:end_of_association_chain).pg_copy_to "/tmp/#{name}.csv"
|
|
8
8
|
Dir.chdir('/tmp') do
|
|
9
9
|
`zip #{name} #{name}.csv`
|
data/postgres-copy.gemspec
CHANGED
|
@@ -4,20 +4,21 @@
|
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |s|
|
|
7
|
-
s.name =
|
|
8
|
-
s.version = "0.3.
|
|
7
|
+
s.name = "postgres-copy"
|
|
8
|
+
s.version = "0.3.6"
|
|
9
9
|
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
11
|
s.authors = ["Diogo Biazus"]
|
|
12
|
-
s.date =
|
|
13
|
-
s.description =
|
|
14
|
-
s.email =
|
|
12
|
+
s.date = "2011-12-30"
|
|
13
|
+
s.description = "Now you can use the super fast COPY for import/export data directly from your AR models."
|
|
14
|
+
s.email = "diogob@gmail.com"
|
|
15
15
|
s.extra_rdoc_files = [
|
|
16
16
|
"LICENSE",
|
|
17
17
|
"README.md"
|
|
18
18
|
]
|
|
19
19
|
s.files = [
|
|
20
20
|
".document",
|
|
21
|
+
".rspec",
|
|
21
22
|
"Gemfile",
|
|
22
23
|
"Gemfile.lock",
|
|
23
24
|
"LICENSE",
|
|
@@ -29,6 +30,7 @@ Gem::Specification.new do |s|
|
|
|
29
30
|
"lib/postgres-copy/csv_responder.rb",
|
|
30
31
|
"lib/postgres-copy/zip_responder.rb",
|
|
31
32
|
"postgres-copy.gemspec",
|
|
33
|
+
"spec/fixtures/comma_with_header.csv",
|
|
32
34
|
"spec/fixtures/extra_field.rb",
|
|
33
35
|
"spec/fixtures/reserved_word_model.rb",
|
|
34
36
|
"spec/fixtures/reserved_words.csv",
|
|
@@ -46,10 +48,10 @@ Gem::Specification.new do |s|
|
|
|
46
48
|
"spec/spec.opts",
|
|
47
49
|
"spec/spec_helper.rb"
|
|
48
50
|
]
|
|
49
|
-
s.homepage =
|
|
51
|
+
s.homepage = "http://github.com/diogob/postgres-copy"
|
|
50
52
|
s.require_paths = ["lib"]
|
|
51
|
-
s.rubygems_version =
|
|
52
|
-
s.summary =
|
|
53
|
+
s.rubygems_version = "1.8.10"
|
|
54
|
+
s.summary = "Put COPY command functionality in ActiveRecord's model class"
|
|
53
55
|
|
|
54
56
|
if s.respond_to? :specification_version then
|
|
55
57
|
s.specification_version = 3
|
|
@@ -59,8 +61,8 @@ Gem::Specification.new do |s|
|
|
|
59
61
|
s.add_runtime_dependency(%q<activerecord>, [">= 0"])
|
|
60
62
|
s.add_runtime_dependency(%q<rails>, [">= 0"])
|
|
61
63
|
s.add_runtime_dependency(%q<rake>, ["= 0.8.7"])
|
|
62
|
-
s.add_runtime_dependency(%q<rspec>, ["~> 2.
|
|
63
|
-
s.add_runtime_dependency(%q<rspec-core>, ["~> 2.
|
|
64
|
+
s.add_runtime_dependency(%q<rspec>, ["~> 2.6"])
|
|
65
|
+
s.add_runtime_dependency(%q<rspec-core>, ["~> 2.6"])
|
|
64
66
|
s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
|
|
65
67
|
s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
|
|
66
68
|
else
|
|
@@ -68,8 +70,8 @@ Gem::Specification.new do |s|
|
|
|
68
70
|
s.add_dependency(%q<activerecord>, [">= 0"])
|
|
69
71
|
s.add_dependency(%q<rails>, [">= 0"])
|
|
70
72
|
s.add_dependency(%q<rake>, ["= 0.8.7"])
|
|
71
|
-
s.add_dependency(%q<rspec>, ["~> 2.
|
|
72
|
-
s.add_dependency(%q<rspec-core>, ["~> 2.
|
|
73
|
+
s.add_dependency(%q<rspec>, ["~> 2.6"])
|
|
74
|
+
s.add_dependency(%q<rspec-core>, ["~> 2.6"])
|
|
73
75
|
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
|
74
76
|
s.add_dependency(%q<activerecord>, [">= 3.0.0"])
|
|
75
77
|
end
|
|
@@ -78,8 +80,8 @@ Gem::Specification.new do |s|
|
|
|
78
80
|
s.add_dependency(%q<activerecord>, [">= 0"])
|
|
79
81
|
s.add_dependency(%q<rails>, [">= 0"])
|
|
80
82
|
s.add_dependency(%q<rake>, ["= 0.8.7"])
|
|
81
|
-
s.add_dependency(%q<rspec>, ["~> 2.
|
|
82
|
-
s.add_dependency(%q<rspec-core>, ["~> 2.
|
|
83
|
+
s.add_dependency(%q<rspec>, ["~> 2.6"])
|
|
84
|
+
s.add_dependency(%q<rspec-core>, ["~> 2.6"])
|
|
83
85
|
s.add_dependency(%q<rspec>, [">= 1.2.9"])
|
|
84
86
|
s.add_dependency(%q<activerecord>, [">= 3.0.0"])
|
|
85
87
|
end
|
data/spec/pg_copy_from_spec.rb
CHANGED
|
@@ -9,12 +9,12 @@ describe "COPY FROM" do
|
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
it "should import from file if path is passed without field_map" do
|
|
12
|
-
TestModel.pg_copy_from File.expand_path('spec/fixtures/
|
|
12
|
+
TestModel.pg_copy_from File.expand_path('spec/fixtures/comma_with_header.csv')
|
|
13
13
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
it "should import from IO without field_map" do
|
|
17
|
-
TestModel.pg_copy_from File.open(File.expand_path('spec/fixtures/
|
|
17
|
+
TestModel.pg_copy_from File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r')
|
|
18
18
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
19
19
|
end
|
|
20
20
|
|
|
@@ -29,14 +29,14 @@ describe "COPY FROM" do
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
it "should import and allow changes in block" do
|
|
32
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/
|
|
32
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r')) do |row|
|
|
33
33
|
row[1] = 'changed this data'
|
|
34
34
|
end
|
|
35
35
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'changed this data'}]
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
it "should import 2 lines and allow changes in block" do
|
|
39
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.csv'), 'r')) do |row|
|
|
39
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.csv'), 'r'), :delimiter => "\t") do |row|
|
|
40
40
|
row[1] = 'changed this data'
|
|
41
41
|
end
|
|
42
42
|
TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
|
|
@@ -44,17 +44,17 @@ describe "COPY FROM" do
|
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
it "should be able to copy from using custom set of columns" do
|
|
47
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_only_data.csv'), 'r'), :columns => ["data"])
|
|
47
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_only_data.csv'), 'r'), :delimiter => "\t", :columns => ["data"])
|
|
48
48
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
it "default set of columns should be all table columns minus [id, created_at, updated_at]" do
|
|
52
|
-
ExtraField.pg_copy_from(File.open(File.expand_path('spec/fixtures/
|
|
52
|
+
ExtraField.pg_copy_from(File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r'))
|
|
53
53
|
ExtraField.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1', 'created_at' => nil, 'updated_at' => nil}]
|
|
54
54
|
end
|
|
55
55
|
|
|
56
56
|
it "should be able to map the header in the file to diferent column names" do
|
|
57
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_different_header.csv'), 'r'), :map => {'cod' => 'id', 'info' => 'data'})
|
|
57
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_different_header.csv'), 'r'), :delimiter => "\t", :map => {'cod' => 'id', 'info' => 'data'})
|
|
58
58
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
59
59
|
end
|
|
60
60
|
|
|
@@ -64,19 +64,20 @@ describe "COPY FROM" do
|
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
it "should ignore empty lines" do
|
|
67
|
-
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_extra_line.csv'), 'r'))
|
|
67
|
+
TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_extra_line.csv'), 'r'), :delimiter => "\t")
|
|
68
68
|
TestModel.order(:id).all.map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
71
|
+
#we should implement this later
|
|
72
|
+
#it "should raise error in malformed files" do
|
|
73
|
+
#lambda do
|
|
74
|
+
#TestModel.pg_copy_from(File.open(File.expand_path('spec/fixtures/tab_with_error.csv'), 'r'))
|
|
75
|
+
#end.should raise_error
|
|
76
|
+
#TestModel.order(:id).all.map{|r| r.attributes}.should == []
|
|
77
|
+
#end
|
|
77
78
|
|
|
78
79
|
it "should copy from even when table fields need identifier quoting" do
|
|
79
|
-
ReservedWordModel.pg_copy_from File.expand_path('spec/fixtures/reserved_words.csv')
|
|
80
|
+
ReservedWordModel.pg_copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
|
|
80
81
|
ReservedWordModel.order(:id).all.map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
|
|
81
82
|
end
|
|
82
83
|
end
|
data/spec/pg_copy_to_spec.rb
CHANGED
|
@@ -9,27 +9,41 @@ describe "COPY TO" do
|
|
|
9
9
|
TestModel.create :data => 'test data 1'
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
TestModel.
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
describe ".pg_copy_to_string" do
|
|
13
|
+
context "with no options" do
|
|
14
|
+
subject{ TestModel.pg_copy_to_string }
|
|
15
|
+
it{ should == File.open('spec/fixtures/comma_with_header.csv', 'r').read }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
context "with tab as delimiter" do
|
|
19
|
+
subject{ TestModel.pg_copy_to_string :delimiter => "\t" }
|
|
20
|
+
it{ should == File.open('spec/fixtures/tab_with_header.csv', 'r').read }
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
describe ".pg_copy_to" do
|
|
25
|
+
it "should copy and pass data to block if block is given and no path is passed" do
|
|
26
|
+
File.open('spec/fixtures/comma_with_header.csv', 'r') do |f|
|
|
27
|
+
TestModel.pg_copy_to do |row|
|
|
28
|
+
row.should == f.readline
|
|
29
|
+
end
|
|
25
30
|
end
|
|
26
31
|
end
|
|
27
|
-
end
|
|
28
32
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
33
|
+
it "should copy to disk if block is not given and a path is passed" do
|
|
34
|
+
TestModel.pg_copy_to '/tmp/export.csv'
|
|
35
|
+
File.open('spec/fixtures/comma_with_header.csv', 'r') do |fixture|
|
|
36
|
+
File.open('/tmp/export.csv', 'r') do |result|
|
|
37
|
+
result.read.should == fixture.read
|
|
38
|
+
end
|
|
32
39
|
end
|
|
33
|
-
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
it "should raise exception if I pass a path and a block simultaneously" do
|
|
43
|
+
lambda do
|
|
44
|
+
TestModel.pg_copy_to('/tmp/bogus_path') do |row|
|
|
45
|
+
end
|
|
46
|
+
end.should raise_error
|
|
47
|
+
end
|
|
34
48
|
end
|
|
35
49
|
end
|
metadata
CHANGED
|
@@ -1,116 +1,115 @@
|
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: postgres-copy
|
|
3
|
-
version: !ruby/object:Gem::Version
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.3.6
|
|
4
5
|
prerelease:
|
|
5
|
-
version: 0.3.4
|
|
6
6
|
platform: ruby
|
|
7
|
-
authors:
|
|
7
|
+
authors:
|
|
8
8
|
- Diogo Biazus
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
- !ruby/object:Gem::Dependency
|
|
12
|
+
date: 2011-12-30 00:00:00.000000000 Z
|
|
13
|
+
dependencies:
|
|
14
|
+
- !ruby/object:Gem::Dependency
|
|
16
15
|
name: pg
|
|
17
|
-
requirement: &
|
|
16
|
+
requirement: &70230257119200 !ruby/object:Gem::Requirement
|
|
18
17
|
none: false
|
|
19
|
-
requirements:
|
|
20
|
-
- -
|
|
21
|
-
- !ruby/object:Gem::Version
|
|
22
|
-
version:
|
|
18
|
+
requirements:
|
|
19
|
+
- - ! '>='
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: '0'
|
|
23
22
|
type: :runtime
|
|
24
23
|
prerelease: false
|
|
25
|
-
version_requirements: *
|
|
26
|
-
- !ruby/object:Gem::Dependency
|
|
24
|
+
version_requirements: *70230257119200
|
|
25
|
+
- !ruby/object:Gem::Dependency
|
|
27
26
|
name: activerecord
|
|
28
|
-
requirement: &
|
|
27
|
+
requirement: &70230257118460 !ruby/object:Gem::Requirement
|
|
29
28
|
none: false
|
|
30
|
-
requirements:
|
|
31
|
-
- -
|
|
32
|
-
- !ruby/object:Gem::Version
|
|
33
|
-
version:
|
|
29
|
+
requirements:
|
|
30
|
+
- - ! '>='
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '0'
|
|
34
33
|
type: :runtime
|
|
35
34
|
prerelease: false
|
|
36
|
-
version_requirements: *
|
|
37
|
-
- !ruby/object:Gem::Dependency
|
|
35
|
+
version_requirements: *70230257118460
|
|
36
|
+
- !ruby/object:Gem::Dependency
|
|
38
37
|
name: rails
|
|
39
|
-
requirement: &
|
|
38
|
+
requirement: &70230257117880 !ruby/object:Gem::Requirement
|
|
40
39
|
none: false
|
|
41
|
-
requirements:
|
|
42
|
-
- -
|
|
43
|
-
- !ruby/object:Gem::Version
|
|
44
|
-
version:
|
|
40
|
+
requirements:
|
|
41
|
+
- - ! '>='
|
|
42
|
+
- !ruby/object:Gem::Version
|
|
43
|
+
version: '0'
|
|
45
44
|
type: :runtime
|
|
46
45
|
prerelease: false
|
|
47
|
-
version_requirements: *
|
|
48
|
-
- !ruby/object:Gem::Dependency
|
|
46
|
+
version_requirements: *70230257117880
|
|
47
|
+
- !ruby/object:Gem::Dependency
|
|
49
48
|
name: rake
|
|
50
|
-
requirement: &
|
|
49
|
+
requirement: &70230257117360 !ruby/object:Gem::Requirement
|
|
51
50
|
none: false
|
|
52
|
-
requirements:
|
|
53
|
-
- -
|
|
54
|
-
- !ruby/object:Gem::Version
|
|
51
|
+
requirements:
|
|
52
|
+
- - =
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
55
54
|
version: 0.8.7
|
|
56
55
|
type: :runtime
|
|
57
56
|
prerelease: false
|
|
58
|
-
version_requirements: *
|
|
59
|
-
- !ruby/object:Gem::Dependency
|
|
57
|
+
version_requirements: *70230257117360
|
|
58
|
+
- !ruby/object:Gem::Dependency
|
|
60
59
|
name: rspec
|
|
61
|
-
requirement: &
|
|
60
|
+
requirement: &70230257116780 !ruby/object:Gem::Requirement
|
|
62
61
|
none: false
|
|
63
|
-
requirements:
|
|
62
|
+
requirements:
|
|
64
63
|
- - ~>
|
|
65
|
-
- !ruby/object:Gem::Version
|
|
66
|
-
version:
|
|
64
|
+
- !ruby/object:Gem::Version
|
|
65
|
+
version: '2.6'
|
|
67
66
|
type: :runtime
|
|
68
67
|
prerelease: false
|
|
69
|
-
version_requirements: *
|
|
70
|
-
- !ruby/object:Gem::Dependency
|
|
68
|
+
version_requirements: *70230257116780
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
71
70
|
name: rspec-core
|
|
72
|
-
requirement: &
|
|
71
|
+
requirement: &70230257116200 !ruby/object:Gem::Requirement
|
|
73
72
|
none: false
|
|
74
|
-
requirements:
|
|
73
|
+
requirements:
|
|
75
74
|
- - ~>
|
|
76
|
-
- !ruby/object:Gem::Version
|
|
77
|
-
version:
|
|
75
|
+
- !ruby/object:Gem::Version
|
|
76
|
+
version: '2.6'
|
|
78
77
|
type: :runtime
|
|
79
78
|
prerelease: false
|
|
80
|
-
version_requirements: *
|
|
81
|
-
- !ruby/object:Gem::Dependency
|
|
79
|
+
version_requirements: *70230257116200
|
|
80
|
+
- !ruby/object:Gem::Dependency
|
|
82
81
|
name: rspec
|
|
83
|
-
requirement: &
|
|
82
|
+
requirement: &70230257115420 !ruby/object:Gem::Requirement
|
|
84
83
|
none: false
|
|
85
|
-
requirements:
|
|
86
|
-
- -
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
84
|
+
requirements:
|
|
85
|
+
- - ! '>='
|
|
86
|
+
- !ruby/object:Gem::Version
|
|
88
87
|
version: 1.2.9
|
|
89
88
|
type: :development
|
|
90
89
|
prerelease: false
|
|
91
|
-
version_requirements: *
|
|
92
|
-
- !ruby/object:Gem::Dependency
|
|
90
|
+
version_requirements: *70230257115420
|
|
91
|
+
- !ruby/object:Gem::Dependency
|
|
93
92
|
name: activerecord
|
|
94
|
-
requirement: &
|
|
93
|
+
requirement: &70230257113880 !ruby/object:Gem::Requirement
|
|
95
94
|
none: false
|
|
96
|
-
requirements:
|
|
97
|
-
- -
|
|
98
|
-
- !ruby/object:Gem::Version
|
|
95
|
+
requirements:
|
|
96
|
+
- - ! '>='
|
|
97
|
+
- !ruby/object:Gem::Version
|
|
99
98
|
version: 3.0.0
|
|
100
99
|
type: :runtime
|
|
101
100
|
prerelease: false
|
|
102
|
-
version_requirements: *
|
|
103
|
-
description: Now you can use the super fast COPY for import/export data directly from
|
|
101
|
+
version_requirements: *70230257113880
|
|
102
|
+
description: Now you can use the super fast COPY for import/export data directly from
|
|
103
|
+
your AR models.
|
|
104
104
|
email: diogob@gmail.com
|
|
105
105
|
executables: []
|
|
106
|
-
|
|
107
106
|
extensions: []
|
|
108
|
-
|
|
109
|
-
extra_rdoc_files:
|
|
107
|
+
extra_rdoc_files:
|
|
110
108
|
- LICENSE
|
|
111
109
|
- README.md
|
|
112
|
-
files:
|
|
110
|
+
files:
|
|
113
111
|
- .document
|
|
112
|
+
- .rspec
|
|
114
113
|
- Gemfile
|
|
115
114
|
- Gemfile.lock
|
|
116
115
|
- LICENSE
|
|
@@ -122,6 +121,7 @@ files:
|
|
|
122
121
|
- lib/postgres-copy/csv_responder.rb
|
|
123
122
|
- lib/postgres-copy/zip_responder.rb
|
|
124
123
|
- postgres-copy.gemspec
|
|
124
|
+
- spec/fixtures/comma_with_header.csv
|
|
125
125
|
- spec/fixtures/extra_field.rb
|
|
126
126
|
- spec/fixtures/reserved_word_model.rb
|
|
127
127
|
- spec/fixtures/reserved_words.csv
|
|
@@ -140,30 +140,26 @@ files:
|
|
|
140
140
|
- spec/spec_helper.rb
|
|
141
141
|
homepage: http://github.com/diogob/postgres-copy
|
|
142
142
|
licenses: []
|
|
143
|
-
|
|
144
143
|
post_install_message:
|
|
145
144
|
rdoc_options: []
|
|
146
|
-
|
|
147
|
-
require_paths:
|
|
145
|
+
require_paths:
|
|
148
146
|
- lib
|
|
149
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
|
147
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
150
148
|
none: false
|
|
151
|
-
requirements:
|
|
152
|
-
- -
|
|
153
|
-
- !ruby/object:Gem::Version
|
|
154
|
-
version:
|
|
155
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
|
+
requirements:
|
|
150
|
+
- - ! '>='
|
|
151
|
+
- !ruby/object:Gem::Version
|
|
152
|
+
version: '0'
|
|
153
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
154
|
none: false
|
|
157
|
-
requirements:
|
|
158
|
-
- -
|
|
159
|
-
- !ruby/object:Gem::Version
|
|
160
|
-
version:
|
|
155
|
+
requirements:
|
|
156
|
+
- - ! '>='
|
|
157
|
+
- !ruby/object:Gem::Version
|
|
158
|
+
version: '0'
|
|
161
159
|
requirements: []
|
|
162
|
-
|
|
163
160
|
rubyforge_project:
|
|
164
|
-
rubygems_version: 1.
|
|
161
|
+
rubygems_version: 1.8.10
|
|
165
162
|
signing_key:
|
|
166
163
|
specification_version: 3
|
|
167
164
|
summary: Put COPY command functionality in ActiveRecord's model class
|
|
168
165
|
test_files: []
|
|
169
|
-
|