pgcp 0.0.0 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.pgcp.yml.sample +11 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +43 -0
- data/README.md +69 -0
- data/bin/pgcp +2 -7
- data/lib/pgcp.rb +5 -0
- data/lib/{postgres.rb → pgcp/postgres.rb} +0 -0
- data/lib/pgcp/qualified_name.rb +17 -0
- data/lib/{transport.rb → pgcp/transport.rb} +2 -3
- data/pgcp.gemspec +23 -0
- data/spec/postgres_spec.rb +17 -0
- data/spec/qualified_name_spec.rb +17 -0
- data/spec/spec_helper.rb +96 -0
- data/spec/transport_spec.rb +32 -0
- metadata +16 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1641448e3565b4a61b61ea4e4977a6ec734c0ba5
|
4
|
+
data.tar.gz: 100456b3ef3c8585abc5a443e749c9528559fb35
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e11be5310bf941f21045f5c052245d5d4e1dccea0c801cba546b7145ab3f70785d59da911aa4ac3e7a88b476d4a5cdbea0cf57fdf47464f621fa8b5cb12b2861
|
7
|
+
data.tar.gz: dda0a068b9cbe883be68eea6d12dd1e841f3fe97d8dda228f490d2f49ac46df79df728d34c237c5111e9744cbfe74e8a08b7c31d3650926a8dcfca0fcb601ced
|
data/.gitignore
ADDED
data/.pgcp.yml.sample
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
pgcp
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby-2.2.2
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
specs:
|
4
|
+
activesupport (4.2.4)
|
5
|
+
i18n (~> 0.7)
|
6
|
+
json (~> 1.7, >= 1.7.7)
|
7
|
+
minitest (~> 5.1)
|
8
|
+
thread_safe (~> 0.3, >= 0.3.4)
|
9
|
+
tzinfo (~> 1.1)
|
10
|
+
diff-lcs (1.2.5)
|
11
|
+
i18n (0.7.0)
|
12
|
+
json (1.8.1)
|
13
|
+
minitest (5.4.3)
|
14
|
+
pg (0.18.3)
|
15
|
+
rspec (3.3.0)
|
16
|
+
rspec-core (~> 3.3.0)
|
17
|
+
rspec-expectations (~> 3.3.0)
|
18
|
+
rspec-mocks (~> 3.3.0)
|
19
|
+
rspec-core (3.3.2)
|
20
|
+
rspec-support (~> 3.3.0)
|
21
|
+
rspec-expectations (3.3.1)
|
22
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
23
|
+
rspec-support (~> 3.3.0)
|
24
|
+
rspec-mocks (3.3.2)
|
25
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
26
|
+
rspec-support (~> 3.3.0)
|
27
|
+
rspec-support (3.3.0)
|
28
|
+
thor (0.19.1)
|
29
|
+
thread_safe (0.3.5)
|
30
|
+
tzinfo (1.2.2)
|
31
|
+
thread_safe (~> 0.1)
|
32
|
+
|
33
|
+
PLATFORMS
|
34
|
+
ruby
|
35
|
+
|
36
|
+
DEPENDENCIES
|
37
|
+
activesupport
|
38
|
+
pg
|
39
|
+
rspec
|
40
|
+
thor
|
41
|
+
|
42
|
+
BUNDLED WITH
|
43
|
+
1.10.6
|
data/README.md
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
# pgcp - Copy PostgreSQL tables between databases (for analytics purpose)
|
2
|
+
|
3
|
+
pgcp is a simple tool to copy tables from a source postgres DB to a destination postgres DB.
|
4
|
+
|
5
|
+
|
6
|
+
The main use case of the tool is to make a dump of data from production databases
|
7
|
+
to a centralized database for analytics purpose. Don't expect the copy to be an exact copy,
|
8
|
+
we do drop a lot of table's metadata and only retain what we think is important for analytics.
|
9
|
+
|
10
|
+
|
11
|
+
What we retain:
|
12
|
+
|
13
|
+
* Table's schema structure
|
14
|
+
* Table's indexes
|
15
|
+
* Table's data
|
16
|
+
|
17
|
+
What we don't clone over (non-exhaustive):
|
18
|
+
|
19
|
+
* Sequences (auto increment)
|
20
|
+
* Triggers
|
21
|
+
* ...
|
22
|
+
|
23
|
+
Requirements:
|
24
|
+
|
25
|
+
* Make sure you have `psql` installed on the machine running `pgcp`
|
26
|
+
|
27
|
+
## Installation
|
28
|
+
|
29
|
+
|
30
|
+
Install gem from RubyGems
|
31
|
+
|
32
|
+
gem install pgcp
|
33
|
+
|
34
|
+
|
35
|
+
Create file `~/.pgcp.yml` that contains the credentials of your interested databases:
|
36
|
+
|
37
|
+
databases:
|
38
|
+
production_db:
|
39
|
+
user: postgres
|
40
|
+
password:
|
41
|
+
dbname:
|
42
|
+
host: your_production.server.com
|
43
|
+
analytics_db:
|
44
|
+
user: postgres
|
45
|
+
dbname: analytics
|
46
|
+
password:
|
47
|
+
host: your_analytics.server.com
|
48
|
+
|
49
|
+
|
50
|
+
## Usage
|
51
|
+
|
52
|
+
For usage details, runs: `pgcp help`
|
53
|
+
|
54
|
+
Copy a single `bookings` table from production to analytics:
|
55
|
+
|
56
|
+
pgcp -s production_db -d analytics_db -t public.bookings
|
57
|
+
|
58
|
+
Copy multiple tables:
|
59
|
+
|
60
|
+
pgcp -s production_db -d analytics_db -t public.bookings public.registrations
|
61
|
+
|
62
|
+
|
63
|
+
Copy all tables in schema public to destination database
|
64
|
+
|
65
|
+
pgcp -s production_db -d analytics_db -t public.*
|
66
|
+
|
67
|
+
Copy all tables in schema public to destination database, but to a different schema
|
68
|
+
|
69
|
+
pgcp -s production_db -d analytics_db -t public.* --force-schema
|
data/bin/pgcp
CHANGED
data/lib/pgcp.rb
CHANGED
File without changes
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class QualifiedName
|
2
|
+
attr_accessor :schema_name, :table_name
|
3
|
+
|
4
|
+
def initialize(schema_name, table_name=nil)
|
5
|
+
if table_name.nil?
|
6
|
+
@table_name = schema_name.split('.')[1]
|
7
|
+
@schema_name = schema_name.split('.')[0]
|
8
|
+
else
|
9
|
+
@schema_name = schema_name
|
10
|
+
@table_name = table_name
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def full_name
|
15
|
+
"#{@schema_name}.#{@table_name}"
|
16
|
+
end
|
17
|
+
end
|
data/pgcp.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
$LOAD_PATH << File.expand_path('../lib', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'pgcp'
|
5
|
+
s.version = '0.0.1'
|
6
|
+
s.date = '2015-11-05'
|
7
|
+
s.summary = 'A simple command line tool to copy tables from one Postgres database to another'
|
8
|
+
s.description = 'A simple command line tool to copy tables from one Postgres database to another'
|
9
|
+
s.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
|
10
|
+
s.email = 'thanh@holistics.io'
|
11
|
+
|
12
|
+
s.homepage = 'http://rubygems.org/gems/pgcp'
|
13
|
+
s.license = 'GPL'
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.require_paths = ['lib']
|
17
|
+
s.executables << 'pgcp'
|
18
|
+
|
19
|
+
s.add_runtime_dependency "activesupport", "~> 4.2"
|
20
|
+
s.add_runtime_dependency "thor", "~> 0.19"
|
21
|
+
s.add_runtime_dependency "pg", "~> 0.18"
|
22
|
+
|
23
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'transport'
|
2
|
+
|
3
|
+
RSpec.describe Postgres do
|
4
|
+
it "should be able to get index names from a table" do
|
5
|
+
src = {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
|
6
|
+
|
7
|
+
p = Postgres.new(src)
|
8
|
+
p.index_names('public', 'users')
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should be able to get indexes from a table" do
|
12
|
+
src = {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
|
13
|
+
|
14
|
+
p = Postgres.new(src)
|
15
|
+
p.get_indexes('public', 'users')
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'qualified_name'
|
2
|
+
|
3
|
+
RSpec.describe QualifiedName do
|
4
|
+
it "should be able to be created with both arguments" do
|
5
|
+
q = QualifiedName.new('public', 'test')
|
6
|
+
|
7
|
+
expect(q.schema_name).to eq('public')
|
8
|
+
expect(q.table_name).to eq('test')
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should be able to be created with single argument" do
|
12
|
+
q = QualifiedName.new('public.test')
|
13
|
+
|
14
|
+
expect(q.schema_name).to eq('public')
|
15
|
+
expect(q.table_name).to eq('test')
|
16
|
+
end
|
17
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause
|
4
|
+
# this file to always be loaded, without a need to explicitly require it in any
|
5
|
+
# files.
|
6
|
+
#
|
7
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
8
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
9
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
10
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
11
|
+
# a separate helper file that requires the additional dependencies and performs
|
12
|
+
# the additional setup, and require it from the spec files that actually need
|
13
|
+
# it.
|
14
|
+
#
|
15
|
+
# The `.rspec` file also contains a few flags that are not defaults but that
|
16
|
+
# users commonly want.
|
17
|
+
#
|
18
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
19
|
+
RSpec.configure do |config|
|
20
|
+
# rspec-expectations config goes here. You can use an alternate
|
21
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
22
|
+
# assertions if you prefer.
|
23
|
+
config.expect_with :rspec do |expectations|
|
24
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
25
|
+
# and `failure_message` of custom matchers include text for helper methods
|
26
|
+
# defined using `chain`, e.g.:
|
27
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
28
|
+
# # => "be bigger than 2 and smaller than 4"
|
29
|
+
# ...rather than:
|
30
|
+
# # => "be bigger than 2"
|
31
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
32
|
+
end
|
33
|
+
|
34
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
35
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
36
|
+
config.mock_with :rspec do |mocks|
|
37
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
38
|
+
# a real object. This is generally recommended, and will default to
|
39
|
+
# `true` in RSpec 4.
|
40
|
+
mocks.verify_partial_doubles = true
|
41
|
+
end
|
42
|
+
|
43
|
+
# The settings below are suggested to provide a good initial experience
|
44
|
+
# with RSpec, but feel free to customize to your heart's content.
|
45
|
+
=begin
|
46
|
+
# These two settings work together to allow you to limit a spec run
|
47
|
+
# to individual examples or groups you care about by tagging them with
|
48
|
+
# `:focus` metadata. When nothing is tagged with `:focus`, all examples
|
49
|
+
# get run.
|
50
|
+
config.filter_run :focus
|
51
|
+
config.run_all_when_everything_filtered = true
|
52
|
+
|
53
|
+
# Allows RSpec to persist some state between runs in order to support
|
54
|
+
# the `--only-failures` and `--next-failure` CLI options. We recommend
|
55
|
+
# you configure your source control system to ignore this file.
|
56
|
+
config.example_status_persistence_file_path = "spec/examples.txt"
|
57
|
+
|
58
|
+
# Limits the available syntax to the non-monkey patched syntax that is
|
59
|
+
# recommended. For more details, see:
|
60
|
+
# - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
|
61
|
+
# - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
62
|
+
# - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
|
63
|
+
config.disable_monkey_patching!
|
64
|
+
|
65
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
66
|
+
# be too noisy due to issues in dependencies.
|
67
|
+
config.warnings = true
|
68
|
+
|
69
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
70
|
+
# file, and it's useful to allow more verbose output when running an
|
71
|
+
# individual spec file.
|
72
|
+
if config.files_to_run.one?
|
73
|
+
# Use the documentation formatter for detailed output,
|
74
|
+
# unless a formatter has already been configured
|
75
|
+
# (e.g. via a command-line flag).
|
76
|
+
config.default_formatter = 'doc'
|
77
|
+
end
|
78
|
+
|
79
|
+
# Print the 10 slowest examples and example groups at the
|
80
|
+
# end of the spec run, to help surface which specs are running
|
81
|
+
# particularly slow.
|
82
|
+
config.profile_examples = 10
|
83
|
+
|
84
|
+
# Run specs in random order to surface order dependencies. If you find an
|
85
|
+
# order dependency and want to debug it, you can fix the order by providing
|
86
|
+
# the seed, which is printed after each run.
|
87
|
+
# --seed 1234
|
88
|
+
config.order = :random
|
89
|
+
|
90
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
91
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
92
|
+
# test failures related to randomization by passing the same `--seed` value
|
93
|
+
# as the one that triggered the failure.
|
94
|
+
Kernel.srand config.seed
|
95
|
+
=end
|
96
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'transport'
|
2
|
+
|
3
|
+
RSpec.describe Transport do
|
4
|
+
context "Directly copy table from one database to another" do
|
5
|
+
let (:src) {
|
6
|
+
{host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
|
7
|
+
}
|
8
|
+
let (:dest) {
|
9
|
+
{host: 'localhost', dbname: 'dashboard_test', user: 'postgres'}
|
10
|
+
}
|
11
|
+
|
12
|
+
it "should be able to create the Transport class" do
|
13
|
+
tr = Transport.new(src, dest)
|
14
|
+
tr.copy_table('public.users')
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should be able to directly copy table" do
|
18
|
+
tr = Transport.new(src, dest)
|
19
|
+
tr.copy_table('public.users')
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should not create schema if specified" do
|
23
|
+
tr = Transport.new(src, dest)
|
24
|
+
tr.copy_table('public.users', create_schema: false)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should skip copy indexes if specified" do
|
28
|
+
tr = Transport.new(src, dest)
|
29
|
+
tr.copy_table('public.users', skip_indexes: true)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgcp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thanh Dinh Khac
|
@@ -61,10 +61,23 @@ executables:
|
|
61
61
|
extensions: []
|
62
62
|
extra_rdoc_files: []
|
63
63
|
files:
|
64
|
+
- ".gitignore"
|
65
|
+
- ".pgcp.yml.sample"
|
66
|
+
- ".ruby-gemset"
|
67
|
+
- ".ruby-version"
|
68
|
+
- Gemfile
|
69
|
+
- Gemfile.lock
|
70
|
+
- README.md
|
64
71
|
- bin/pgcp
|
65
72
|
- lib/pgcp.rb
|
66
|
-
- lib/postgres.rb
|
67
|
-
- lib/
|
73
|
+
- lib/pgcp/postgres.rb
|
74
|
+
- lib/pgcp/qualified_name.rb
|
75
|
+
- lib/pgcp/transport.rb
|
76
|
+
- pgcp.gemspec
|
77
|
+
- spec/postgres_spec.rb
|
78
|
+
- spec/qualified_name_spec.rb
|
79
|
+
- spec/spec_helper.rb
|
80
|
+
- spec/transport_spec.rb
|
68
81
|
homepage: http://rubygems.org/gems/pgcp
|
69
82
|
licenses:
|
70
83
|
- GPL
|