pgcp 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0bf53cb613c6859915d664646ef18c6ef216767
-  data.tar.gz: b3e0ace6249b20249efba309f40e7385d1f4cd24
+  metadata.gz: 1641448e3565b4a61b61ea4e4977a6ec734c0ba5
+  data.tar.gz: 100456b3ef3c8585abc5a443e749c9528559fb35
 SHA512:
-  metadata.gz: 41ae7a1c3b5c39e29b698a402f7c336d77de4a2f69c2cf92c2512624c788d6e37f61300a66b570cc5148b34100cd77e8e136332df81a64bea1395501de992a43
-  data.tar.gz: 21534f4d98783c90a57d54bcbb978864711206dc24686f275c286de031a79acbd668e1ea01ce4799a5b884821f50f07c579fb1a99dc55f6c81d26bad5713bbc2
+  metadata.gz: e11be5310bf941f21045f5c052245d5d4e1dccea0c801cba546b7145ab3f70785d59da911aa4ac3e7a88b476d4a5cdbea0cf57fdf47464f621fa8b5cb12b2861
+  data.tar.gz: dda0a068b9cbe883be68eea6d12dd1e841f3fe97d8dda228f490d2f49ac46df79df728d34c237c5111e9744cbfe74e8a08b7c31d3650926a8dcfca0fcb601ced

data/.gitignore

@@ -0,0 +1,2 @@
+.idea
+.pgcp.yml

data/.pgcp.yml.sample

@@ -0,0 +1,11 @@
+databases:
+  db1:
+    user: postgres
+    password:
+    dbname: reporting_test
+    host: localhost
+  db2:
+    user: postgres
+    dbname: reporting_test
+    password:
+    host: localhost

data/.ruby-gemset

@@ -0,0 +1 @@
+pgcp

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.2.2

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gem 'pg'
+gem 'thor'
+gem 'rspec'
+gem 'activesupport'

data/Gemfile.lock

@@ -0,0 +1,43 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.4)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    diff-lcs (1.2.5)
+    i18n (0.7.0)
+    json (1.8.1)
+    minitest (5.4.3)
+    pg (0.18.3)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  pg
+  rspec
+  thor
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -0,0 +1,69 @@
+# pgcp - Copy PostgreSQL tables between databases (for analytics purpose)
+
+pgcp is a simple tool to copy tables from a source postgres DB to a destination postgres DB.
+
+
+The main use case of the tool is to make a dump of data from production databases
+ to a centralized database for analytics purpose. Don't expect the copy to be an exact copy,
+ we do drop a lot of table's metadata and only retain what we think is important for analytics.
+
+
+What we retain:
+
+* Table's schema structure
+* Table's indexes
+* Table's data
+
+What we don't clone over (non-exhaustive):
+
+* Sequences (auto increment)
+* Triggers
+* ...
+
+Requirements:
+
+* Make sure you have `psql` installed on the machine running `pgcp`
+
+## Installation
+
+
+Install gem from RubyGems
+
+    gem install pgcp
+
+
+Create file `~/.pgcp.yml` that contains the credentials of your interested databases:
+
+    databases:
+      production_db:
+        user: postgres
+        password:
+        dbname: 
+        host: your_production.server.com
+      analytics_db:
+        user: postgres
+        dbname: analytics
+        password:
+        host: your_analytics.server.com
+
+
+## Usage
+
+For usage details, runs: `pgcp help`
+
+Copy a single `bookings` table from production to analytics:
+
+    pgcp -s production_db -d analytics_db -t public.bookings 
+
+Copy multiple tables:
+
+    pgcp -s production_db -d analytics_db -t public.bookings public.registrations
+
+
+Copy all tables in schema public to destination database
+
+    pgcp -s production_db -d analytics_db -t public.*
+
+Copy all tables in schema public to destination database, but to a different schema
+
+    pgcp -s production_db -d analytics_db -t public.* --force-schema

data/bin/pgcp

@@ -1,9 +1,4 @@
 #!/usr/bin/env ruby
-require 'yaml'
-require 'thor'
-require 'active_support'
-require 'active_support/core_ext'
-require './lib/transport'
-require './lib/pgcp'
-
+require 'pgcp'
 PgcpRunner.start
+

data/lib/pgcp.rb

@@ -1,4 +1,9 @@
 require 'logger'
+require 'yaml'
+require 'thor'
+require 'active_support'
+require 'active_support/core_ext'
+require 'pgcp/transport'
 
 class PgcpRunner < Thor
   desc 'cp', 'Perform copies of tables between Postgres databases'

data/lib/pgcp/qualified_name.rb

@@ -0,0 +1,17 @@
+class QualifiedName
+  attr_accessor :schema_name, :table_name
+
+  def initialize(schema_name, table_name=nil)
+    if table_name.nil?
+      @table_name = schema_name.split('.')[1]
+      @schema_name = schema_name.split('.')[0]
+    else
+      @schema_name = schema_name
+      @table_name = table_name
+    end
+  end
+
+  def full_name
+    "#{@schema_name}.#{@table_name}"
+  end
+end

data/lib/{transport.rb → pgcp/transport.rb}

@@ -1,8 +1,7 @@
 require 'active_support'
 require 'active_support/core_ext'
-require './lib/postgres'
-require './lib/qualified_name'
-require './lib/pgcp'
+require 'pgcp/postgres'
+require 'pgcp/qualified_name'
 require 'securerandom'
 
 class Transport

data/pgcp.gemspec

@@ -0,0 +1,23 @@
+$LOAD_PATH << File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name = 'pgcp'
+  s.version = '0.0.1'
+  s.date = '2015-11-05'
+  s.summary = 'A simple command line tool to copy tables from one Postgres database to another'
+  s.description = 'A simple command line tool to copy tables from one Postgres database to another'
+  s.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
+  s.email = 'thanh@holistics.io'
+
+  s.homepage = 'http://rubygems.org/gems/pgcp'
+  s.license = 'GPL'
+
+  s.files = `git ls-files`.split("\n")
+  s.require_paths = ['lib']
+  s.executables << 'pgcp'
+
+  s.add_runtime_dependency "activesupport", "~> 4.2"
+  s.add_runtime_dependency "thor", "~> 0.19"
+  s.add_runtime_dependency "pg", "~> 0.18"
+
+end

data/spec/postgres_spec.rb

@@ -0,0 +1,17 @@
+require 'transport'
+
+RSpec.describe Postgres do
+  it "should be able to get index names from a table" do
+    src = {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
+
+    p = Postgres.new(src)
+    p.index_names('public', 'users')
+  end
+
+  it "should be able to get indexes from a table" do
+    src = {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
+
+    p = Postgres.new(src)
+    p.get_indexes('public', 'users')
+  end
+end

data/spec/qualified_name_spec.rb

@@ -0,0 +1,17 @@
+require 'qualified_name'
+
+RSpec.describe QualifiedName do
+  it "should be able to be created with both arguments" do
+    q = QualifiedName.new('public', 'test')
+
+    expect(q.schema_name).to eq('public')
+    expect(q.table_name).to eq('test')
+  end
+
+  it "should be able to be created with single argument" do
+    q = QualifiedName.new('public.test')
+
+    expect(q.schema_name).to eq('public')
+    expect(q.table_name).to eq('test')
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,96 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

data/spec/transport_spec.rb

@@ -0,0 +1,32 @@
+require 'transport'
+
+RSpec.describe Transport do
+  context "Directly copy table from one database to another" do
+    let (:src) {
+      {host: 'localhost', dbname: 'dashboard_dev', user: 'postgres'}
+    }
+    let (:dest) {
+      {host: 'localhost', dbname: 'dashboard_test', user: 'postgres'}
+    }
+
+    it "should be able to create the Transport class" do
+      tr = Transport.new(src, dest)
+      tr.copy_table('public.users')
+    end
+
+    it "should be able to directly copy table" do
+      tr = Transport.new(src, dest)
+      tr.copy_table('public.users')
+    end
+
+    it "should not create schema if specified" do
+      tr = Transport.new(src, dest)
+      tr.copy_table('public.users', create_schema: false)
+    end
+
+    it "should skip copy indexes if specified" do
+      tr = Transport.new(src, dest)
+      tr.copy_table('public.users', skip_indexes: true)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgcp
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors:
 - Thanh Dinh Khac
@@ -61,10 +61,23 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".pgcp.yml.sample"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- README.md
 - bin/pgcp
 - lib/pgcp.rb
-- lib/postgres.rb
-- lib/transport.rb
+- lib/pgcp/postgres.rb
+- lib/pgcp/qualified_name.rb
+- lib/pgcp/transport.rb
+- pgcp.gemspec
+- spec/postgres_spec.rb
+- spec/qualified_name_spec.rb
+- spec/spec_helper.rb
+- spec/transport_spec.rb
 homepage: http://rubygems.org/gems/pgcp
 licenses:
 - GPL