pgcp 0.0.0 → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e0bf53cb613c6859915d664646ef18c6ef216767
4
- data.tar.gz: b3e0ace6249b20249efba309f40e7385d1f4cd24
3
+ metadata.gz: 1641448e3565b4a61b61ea4e4977a6ec734c0ba5
4
+ data.tar.gz: 100456b3ef3c8585abc5a443e749c9528559fb35
5
5
  SHA512:
6
- metadata.gz: 41ae7a1c3b5c39e29b698a402f7c336d77de4a2f69c2cf92c2512624c788d6e37f61300a66b570cc5148b34100cd77e8e136332df81a64bea1395501de992a43
7
- data.tar.gz: 21534f4d98783c90a57d54bcbb978864711206dc24686f275c286de031a79acbd668e1ea01ce4799a5b884821f50f07c579fb1a99dc55f6c81d26bad5713bbc2
6
+ metadata.gz: e11be5310bf941f21045f5c052245d5d4e1dccea0c801cba546b7145ab3f70785d59da911aa4ac3e7a88b476d4a5cdbea0cf57fdf47464f621fa8b5cb12b2861
7
+ data.tar.gz: dda0a068b9cbe883be68eea6d12dd1e841f3fe97d8dda228f490d2f49ac46df79df728d34c237c5111e9744cbfe74e8a08b7c31d3650926a8dcfca0fcb601ced
@@ -0,0 +1,2 @@
1
+ .idea
2
+ .pgcp.yml
@@ -0,0 +1,11 @@
1
+ databases:
2
+ db1:
3
+ user: postgres
4
+ password:
5
+ dbname: reporting_test
6
+ host: localhost
7
+ db2:
8
+ user: postgres
9
+ dbname: reporting_test
10
+ password:
11
+ host: localhost
@@ -0,0 +1 @@
1
+ pgcp
@@ -0,0 +1 @@
1
+ ruby-2.2.2
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'pg'
4
+ gem 'thor'
5
+ gem 'rspec'
6
+ gem 'activesupport'
@@ -0,0 +1,43 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ activesupport (4.2.4)
5
+ i18n (~> 0.7)
6
+ json (~> 1.7, >= 1.7.7)
7
+ minitest (~> 5.1)
8
+ thread_safe (~> 0.3, >= 0.3.4)
9
+ tzinfo (~> 1.1)
10
+ diff-lcs (1.2.5)
11
+ i18n (0.7.0)
12
+ json (1.8.1)
13
+ minitest (5.4.3)
14
+ pg (0.18.3)
15
+ rspec (3.3.0)
16
+ rspec-core (~> 3.3.0)
17
+ rspec-expectations (~> 3.3.0)
18
+ rspec-mocks (~> 3.3.0)
19
+ rspec-core (3.3.2)
20
+ rspec-support (~> 3.3.0)
21
+ rspec-expectations (3.3.1)
22
+ diff-lcs (>= 1.2.0, < 2.0)
23
+ rspec-support (~> 3.3.0)
24
+ rspec-mocks (3.3.2)
25
+ diff-lcs (>= 1.2.0, < 2.0)
26
+ rspec-support (~> 3.3.0)
27
+ rspec-support (3.3.0)
28
+ thor (0.19.1)
29
+ thread_safe (0.3.5)
30
+ tzinfo (1.2.2)
31
+ thread_safe (~> 0.1)
32
+
33
+ PLATFORMS
34
+ ruby
35
+
36
+ DEPENDENCIES
37
+ activesupport
38
+ pg
39
+ rspec
40
+ thor
41
+
42
+ BUNDLED WITH
43
+ 1.10.6
@@ -0,0 +1,69 @@
1
+ # pgcp - Copy PostgreSQL tables between databases (for analytics purpose)
2
+
3
+ pgcp is a simple tool to copy tables from a source postgres DB to a destination postgres DB.
4
+
5
+
6
+ The main use case of the tool is to make a dump of data from production databases
7
+ to a centralized database for analytics purpose. Don't expect the copy to be an exact copy,
8
+ we do drop a lot of table's metadata and only retain what we think is important for analytics.
9
+
10
+
11
+ What we retain:
12
+
13
+ * Table's schema structure
14
+ * Table's indexes
15
+ * Table's data
16
+
17
+ What we don't clone over (non-exhaustive):
18
+
19
+ * Sequences (auto increment)
20
+ * Triggers
21
+ * ...
22
+
23
+ Requirements:
24
+
25
+ * Make sure you have `psql` installed on the machine running `pgcp`
26
+
27
+ ## Installation
28
+
29
+
30
+ Install gem from RubyGems
31
+
32
+ gem install pgcp
33
+
34
+
35
+ Create file `~/.pgcp.yml` that contains the credentials of your interested databases:
36
+
37
+ databases:
38
+ production_db:
39
+ user: postgres
40
+ password:
41
+ dbname:
42
+ host: your_production.server.com
43
+ analytics_db:
44
+ user: postgres
45
+ dbname: analytics
46
+ password:
47
+ host: your_analytics.server.com
48
+
49
+
50
+ ## Usage
51
+
52
+ For usage details, runs: `pgcp help`
53
+
54
+ Copy a single `bookings` table from production to analytics:
55
+
56
+ pgcp -s production_db -d analytics_db -t public.bookings
57
+
58
+ Copy multiple tables:
59
+
60
+ pgcp -s production_db -d analytics_db -t public.bookings public.registrations
61
+
62
+
63
+ Copy all tables in schema public to destination database
64
+
65
+ pgcp -s production_db -d analytics_db -t public.*
66
+
67
+ Copy all tables in schema public to destination database, but to a different schema
68
+
69
+ pgcp -s production_db -d analytics_db -t public.* --force-schema
data/bin/pgcp CHANGED
@@ -1,9 +1,4 @@
1
1
  #!/usr/bin/env ruby
2
- require 'yaml'
3
- require 'thor'
4
- require 'active_support'
5
- require 'active_support/core_ext'
6
- require './lib/transport'
7
- require './lib/pgcp'
8
-
2
+ require 'pgcp'
9
3
  PgcpRunner.start
4
+
@@ -1,4 +1,9 @@
1
1
  require 'logger'
2
+ require 'yaml'
3
+ require 'thor'
4
+ require 'active_support'
5
+ require 'active_support/core_ext'
6
+ require 'pgcp/transport'
2
7
 
3
8
  class PgcpRunner < Thor
4
9
  desc 'cp', 'Perform copies of tables between Postgres databases'
File without changes
@@ -0,0 +1,17 @@
1
+ class QualifiedName
2
+ attr_accessor :schema_name, :table_name
3
+
4
+ def initialize(schema_name, table_name=nil)
5
+ if table_name.nil?
6
+ @table_name = schema_name.split('.')[1]
7
+ @schema_name = schema_name.split('.')[0]
8
+ else
9
+ @schema_name = schema_name
10
+ @table_name = table_name
11
+ end
12
+ end
13
+
14
+ def full_name
15
+ "#{@schema_name}.#{@table_name}"
16
+ end
17
+ end
@@ -1,8 +1,7 @@
1
1
  require 'active_support'
2
2
  require 'active_support/core_ext'
3
- require './lib/postgres'
4
- require './lib/qualified_name'
5
- require './lib/pgcp'
3
+ require 'pgcp/postgres'
4
+ require 'pgcp/qualified_name'
6
5
  require 'securerandom'
7
6
 
8
7
  class Transport
@@ -0,0 +1,23 @@
1
+ $LOAD_PATH << File.expand_path('../lib', __FILE__)
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = 'pgcp'
5
+ s.version = '0.0.1'
6
+ s.date = '2015-11-05'
7
+ s.summary = 'A simple command line tool to copy tables from one Postgres database to another'
8
+ s.description = 'A simple command line tool to copy tables from one Postgres database to another'
9
+ s.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
10
+ s.email = 'thanh@holistics.io'
11
+
12
+ s.homepage = 'http://rubygems.org/gems/pgcp'
13
+ s.license = 'GPL'
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.require_paths = ['lib']
17
+ s.executables << 'pgcp'
18
+
19
+ s.add_runtime_dependency "activesupport", "~> 4.2"
20
+ s.add_runtime_dependency "thor", "~> 0.19"
21
+ s.add_runtime_dependency "pg", "~> 0.18"
22
+
23
+ end
@@ -0,0 +1,17 @@
1
+ require 'transport'
2
+
3
+ RSpec.describe Postgres do
4
+ it "should be able to get index names from a table" do
5
+ src = {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
6
+
7
+ p = Postgres.new(src)
8
+ p.index_names('public', 'users')
9
+ end
10
+
11
+ it "should be able to get indexes from a table" do
12
+ src = {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
13
+
14
+ p = Postgres.new(src)
15
+ p.get_indexes('public', 'users')
16
+ end
17
+ end
@@ -0,0 +1,17 @@
1
+ require 'qualified_name'
2
+
3
+ RSpec.describe QualifiedName do
4
+ it "should be able to be created with both arguments" do
5
+ q = QualifiedName.new('public', 'test')
6
+
7
+ expect(q.schema_name).to eq('public')
8
+ expect(q.table_name).to eq('test')
9
+ end
10
+
11
+ it "should be able to be created with single argument" do
12
+ q = QualifiedName.new('public.test')
13
+
14
+ expect(q.schema_name).to eq('public')
15
+ expect(q.table_name).to eq('test')
16
+ end
17
+ end
@@ -0,0 +1,96 @@
1
+ # This file was generated by the `rspec --init` command. Conventionally, all
2
+ # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
+ # The generated `.rspec` file contains `--require spec_helper` which will cause
4
+ # this file to always be loaded, without a need to explicitly require it in any
5
+ # files.
6
+ #
7
+ # Given that it is always loaded, you are encouraged to keep this file as
8
+ # light-weight as possible. Requiring heavyweight dependencies from this file
9
+ # will add to the boot time of your test suite on EVERY test run, even for an
10
+ # individual file that may not need all of that loaded. Instead, consider making
11
+ # a separate helper file that requires the additional dependencies and performs
12
+ # the additional setup, and require it from the spec files that actually need
13
+ # it.
14
+ #
15
+ # The `.rspec` file also contains a few flags that are not defaults but that
16
+ # users commonly want.
17
+ #
18
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
19
+ RSpec.configure do |config|
20
+ # rspec-expectations config goes here. You can use an alternate
21
+ # assertion/expectation library such as wrong or the stdlib/minitest
22
+ # assertions if you prefer.
23
+ config.expect_with :rspec do |expectations|
24
+ # This option will default to `true` in RSpec 4. It makes the `description`
25
+ # and `failure_message` of custom matchers include text for helper methods
26
+ # defined using `chain`, e.g.:
27
+ # be_bigger_than(2).and_smaller_than(4).description
28
+ # # => "be bigger than 2 and smaller than 4"
29
+ # ...rather than:
30
+ # # => "be bigger than 2"
31
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
32
+ end
33
+
34
+ # rspec-mocks config goes here. You can use an alternate test double
35
+ # library (such as bogus or mocha) by changing the `mock_with` option here.
36
+ config.mock_with :rspec do |mocks|
37
+ # Prevents you from mocking or stubbing a method that does not exist on
38
+ # a real object. This is generally recommended, and will default to
39
+ # `true` in RSpec 4.
40
+ mocks.verify_partial_doubles = true
41
+ end
42
+
43
+ # The settings below are suggested to provide a good initial experience
44
+ # with RSpec, but feel free to customize to your heart's content.
45
+ =begin
46
+ # These two settings work together to allow you to limit a spec run
47
+ # to individual examples or groups you care about by tagging them with
48
+ # `:focus` metadata. When nothing is tagged with `:focus`, all examples
49
+ # get run.
50
+ config.filter_run :focus
51
+ config.run_all_when_everything_filtered = true
52
+
53
+ # Allows RSpec to persist some state between runs in order to support
54
+ # the `--only-failures` and `--next-failure` CLI options. We recommend
55
+ # you configure your source control system to ignore this file.
56
+ config.example_status_persistence_file_path = "spec/examples.txt"
57
+
58
+ # Limits the available syntax to the non-monkey patched syntax that is
59
+ # recommended. For more details, see:
60
+ # - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
61
+ # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
62
+ # - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
63
+ config.disable_monkey_patching!
64
+
65
+ # This setting enables warnings. It's recommended, but in some cases may
66
+ # be too noisy due to issues in dependencies.
67
+ config.warnings = true
68
+
69
+ # Many RSpec users commonly either run the entire suite or an individual
70
+ # file, and it's useful to allow more verbose output when running an
71
+ # individual spec file.
72
+ if config.files_to_run.one?
73
+ # Use the documentation formatter for detailed output,
74
+ # unless a formatter has already been configured
75
+ # (e.g. via a command-line flag).
76
+ config.default_formatter = 'doc'
77
+ end
78
+
79
+ # Print the 10 slowest examples and example groups at the
80
+ # end of the spec run, to help surface which specs are running
81
+ # particularly slow.
82
+ config.profile_examples = 10
83
+
84
+ # Run specs in random order to surface order dependencies. If you find an
85
+ # order dependency and want to debug it, you can fix the order by providing
86
+ # the seed, which is printed after each run.
87
+ # --seed 1234
88
+ config.order = :random
89
+
90
+ # Seed global randomization in this process using the `--seed` CLI option.
91
+ # Setting this allows you to use `--seed` to deterministically reproduce
92
+ # test failures related to randomization by passing the same `--seed` value
93
+ # as the one that triggered the failure.
94
+ Kernel.srand config.seed
95
+ =end
96
+ end
@@ -0,0 +1,32 @@
1
+ require 'transport'
2
+
3
+ RSpec.describe Transport do
4
+ context "Directly copy table from one database to another" do
5
+ let (:src) {
6
+ {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
7
+ }
8
+ let (:dest) {
9
+ {host: 'localhost', dbname: 'dashboard_test', user: 'postgres'}
10
+ }
11
+
12
+ it "should be able to create the Transport class" do
13
+ tr = Transport.new(src, dest)
14
+ tr.copy_table('public.users')
15
+ end
16
+
17
+ it "should be able to directly copy table" do
18
+ tr = Transport.new(src, dest)
19
+ tr.copy_table('public.users')
20
+ end
21
+
22
+ it "should not create schema if specified" do
23
+ tr = Transport.new(src, dest)
24
+ tr.copy_table('public.users', create_schema: false)
25
+ end
26
+
27
+ it "should skip copy indexes if specified" do
28
+ tr = Transport.new(src, dest)
29
+ tr.copy_table('public.users', skip_indexes: true)
30
+ end
31
+ end
32
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgcp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thanh Dinh Khac
@@ -61,10 +61,23 @@ executables:
61
61
  extensions: []
62
62
  extra_rdoc_files: []
63
63
  files:
64
+ - ".gitignore"
65
+ - ".pgcp.yml.sample"
66
+ - ".ruby-gemset"
67
+ - ".ruby-version"
68
+ - Gemfile
69
+ - Gemfile.lock
70
+ - README.md
64
71
  - bin/pgcp
65
72
  - lib/pgcp.rb
66
- - lib/postgres.rb
67
- - lib/transport.rb
73
+ - lib/pgcp/postgres.rb
74
+ - lib/pgcp/qualified_name.rb
75
+ - lib/pgcp/transport.rb
76
+ - pgcp.gemspec
77
+ - spec/postgres_spec.rb
78
+ - spec/qualified_name_spec.rb
79
+ - spec/spec_helper.rb
80
+ - spec/transport_spec.rb
68
81
  homepage: http://rubygems.org/gems/pgcp
69
82
  licenses:
70
83
  - GPL