pg_conduit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c3a6e9d910b4d945b126b1a769182899c3230db825b30a384dc13b4eef7b34e3
+  data.tar.gz: 8441234b1dd220fdb4b1fa8b8628dda8d5efe016f30bc196968e98421b3a9eb0
+SHA512:
+  metadata.gz: '08e0fc8a6219b8c2ee4844ec56a415ea064c62415a53a1d89f4cdd5b09f847dea19396a1e93fb4c1f35fe26af35583a55129ea1999675c842461d328d195df90'
+  data.tar.gz: b82ed7a7d8d2dd8537600cf049c32eee574b8dcaed951c015578aab125025bc0e2cae155bd9d779f1513b5b007a493e80b918876ad09d3f71148a176f9f50a67

data/.circleci/config.yml

@@ -0,0 +1,70 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/ruby:2.5.1-node-browsers
+        environment:
+          - TEST_DB_HOST=postgres://circleci@localhost/
+          - TEST_DB_SRC=postgres://circleci@localhost/pg_conduit_src_test
+          - TEST_DB_DEST=postgres://circleci@localhost/pg_conduit_dest_test
+
+      - image: circleci/postgres:9.6-alpine
+        environment:
+          POSTGRES_USER: circleci
+          POSTGRES_DB: circleci
+
+    working_directory: ~/src
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      - run:
+          name: Wait for postgres container
+          command: dockerize -wait tcp://127.0.0.1:5432 -timeout 120s
+
+      - run:
+          name: create databases
+          command: bundle exec rake db:create
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+
+            bundle exec rspec --format progress \
+                            --format RspecJunitFormatter \
+                            --out /tmp/test-results/rspec.xml \
+                            --format progress \
+                            $TEST_FILES
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+.env
+
+# simplecov results
+coverage

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.5.1@pg_conduit

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.2
+before_install: gem install bundler -v 1.16.0

data/Dockerfile

@@ -0,0 +1,24 @@
+FROM ruby:2.5.1-alpine
+
+RUN apk add --update \
+  build-base \
+  libxml2-dev \
+  libxslt-dev \
+  postgresql-dev \
+  postgresql-client \
+  tzdata \
+  git \
+  && rm -rf /var/cache/apk/*
+
+ENV APP_HOME /src
+
+RUN mkdir $APP_HOME
+WORKDIR $APP_HOME
+
+ENV BUNDLE_PATH=/bundle
+
+ADD . $APP_HOME
+
+RUN bundle install
+
+CMD ['ash']

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+git_source(:github) {|repo_name| 'https://github.com/#{repo_name}' }
+
+# Specify your gem's dependencies in pg_conduit.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: .
+  specs:
+    pg_conduit (0.1.0)
+      connection_pool (~> 2.2)
+      pg (~> 1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    connection_pool (2.2.2)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    ffi (1.9.25)
+    formatador (0.2.5)
+    guard (2.14.2)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    json (2.1.0)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    lumberjack (1.0.13)
+    method_source (0.9.0)
+    nenv (0.3.0)
+    notiffany (0.1.1)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    pg (1.0.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (10.5.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    ruby_dep (1.5.0)
+    shellany (0.0.1)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    thor (0.20.0)
+    yard (0.9.14)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  guard (~> 2.14)
+  guard-rspec (~> 4.7)
+  pg_conduit!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec_junit_formatter (~> 0.4.1)
+  simplecov (~> 0.16.1)
+  yard (~> 0.9)
+
+BUNDLED WITH
+   1.16.2

data/Guardfile

@@ -0,0 +1,21 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+guard :rspec, cmd: 'bundle exec rspec', all_on_start: true do
+  watch(%r{^spec/(.+)\.rb$}) { 'spec' }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2018 James Brennan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+# PgConduit
+
+[![CircleCI](https://img.shields.io/circleci/project/github/jamesBrennan/pg_conduit.svg?style=svg)](https://circleci.com/gh/jamesBrennan/pg_conduit)
+
+Stream data between two postgres databases. This is mostly an excuse for me to
+play around with concurrency in Ruby. 
+
+This gem is in early development. As such I would advise against using it in any 
+environment where data integrity is important. I will release version 1.0 when 
+I feel confident that the code is sufficiently robust.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_conduit'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pg_conduit
+
+## Quick Start
+
+### `PgConduit.db_to_db(source, destination)`
+
+Returns an instance of `PgConduit::Pipe` that will execute queries passed to
+`read` and `write` against the `source` and `destination` databases, 
+respectively.
+
+The `source` and `destination` arguments are passed to 
+[`PG::Connection`](https://www.rubydoc.info/gems/pg/PG/Connection), so any 
+arguments that it accepts can be used.
+
+#### Write one row at a time
+    
+```ruby
+source      = 'postgres://user:pass@source/db'
+destination = { dbname: 'my_local_db' }
+
+pipe = PgConduit.db_to_db(source, destination)
+
+pipe.read('SELECT id, full_name, email FROM users')
+    .write do |user|
+      <<-SQL
+        INSERT INTO customers(user_id, name, email)
+        VALUES ('#{user['id']}', '#{user['full_name']}', '#{user['email']}')
+      SQL
+    end 
+```
+
+#### Write in batches
+
+```ruby
+source      = 'postgres://user:pass@source/db'
+destination = { dbname: 'my_local_db' }
+
+pipe = PgConduit.db_to_db(source, destination)
+
+pipe.read('SELECT id, full_name, email FROM users')
+    .transform do |user| 
+      <<-SQL
+        ('#{user['id']}', '#{user['full_name']}', '#{user['email']}')
+      SQL
+    end
+    .write_batched(size: 100) do |values|
+      <<-SQL
+        INSERT INTO customers(user_id, name, email)
+        VALUES #{values.join(',')}
+      SQL
+    end 
+```
+
+### `PgConduit.db_to_file(source, destination)`
+
+Write output from source database to file.
+
+```ruby
+source      = 'postgres://user:pass@source/db'
+destination = '/some/system/path/user_count.txt'
+
+pipe = PgConduit.db_to_file(source, destination)
+
+pipe.read('SELECT count(*) FROM users')
+    .write { |res| "Number of users: #{res['count']}" }
+```
+
+### `PgConduit.db_to_stdout(source)`
+
+Write output from source database to stdout.
+
+```ruby
+pipe = PgConduit.db_to_stdout('postgres://user:pass@source/db')
+
+query = <<-SQL
+  SELECT posts.user_id, users.email, count(posts.*) FROM users
+  JOIN posts ON posts.user_id = users.id
+  GROUP BY posts.user_id, users.email
+SQL
+
+pipe.read(query)
+    .write do |post_count| 
+      "#{post_count['user_id']} | #{post_count['email']} - #{post_count['count']}"
+    end
+```
+        
+### `PgConduit.db_to_null(source)`
+
+Swallow output from source database. Mostly useful for testing. `exec` is an
+alias of `write`.
+
+```ruby
+pipe = PgConduit.db_to_null('postgres://user:pass@source/db')
+pipe.read('SELECT count(*) FROM users')
+    .exec { |res| raise 'fail' unless res['count'] == 10 }
+```
+
+## Development
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pg_conduit.

data/Rakefile

@@ -0,0 +1,23 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'pg'
+
+def create_db(conn, name)
+  conn.exec "CREATE DATABASE #{name}"
+rescue PG::DuplicateDatabase
+  puts "Create database skipped: '#{name}' already exists."
+end
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+namespace :db do
+  task :create do
+    conn = PG::Connection.open(ENV['TEST_DB_HOST'])
+    create_db conn, 'pg_conduit_src_test'
+    create_db conn, 'pg_conduit_dest_test'
+  ensure
+    conn.close
+  end
+end

data/bin/checksum.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+require 'digest/sha2'
+require 'pg_conduit/version'
+
+gem_name = "pg_conduit-#{PgConduit::VERSION}.gem"
+checksum = Digest::SHA512.new.hexdigest File.read("pkg/#{gem_name}")
+File.open("checksum/#{gem_name}.sha512", 'w' ) do |f|
+  f.write checksum
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'pg_conduit'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/certs/jamesbrennan.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDgDCCAmigAwIBAgIBATANBgkqhkiG9w0BAQUFADBDMRUwEwYDVQQDDAxicmVu
+bmFubXVzaWMxFTATBgoJkiaJk/IsZAEZFgVnbWFpbDETMBEGCgmSJomT8ixkARkW
+A2NvbTAeFw0xODA3MDYwMDA2MzRaFw0xOTA3MDYwMDA2MzRaMEMxFTATBgNVBAMM
+DGJyZW5uYW5tdXNpYzEVMBMGCgmSJomT8ixkARkWBWdtYWlsMRMwEQYKCZImiZPy
+LGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyxykNKqp
+jD5gnQd2YHuRDNsLN5limGMDjSb38lMZAfJJugoWbSSeQM3+iVl26YWE5JeJpqDu
+Rr/GOQ54bsev4IxCfPl8yIkhAao0i96+k1EeEb/i3pgZOfjXgGSTM71baCZzbqBH
+A5ivcb0wqoHWyqrxNtc4XhSs5RtvtNl3IEe/bs5qqixt6PwsCorD6kAoUUNHNM9W
+MLexivfiJooHOvVRmf1DQuJ2fDccKNHf7ZR4PuiguT4Z73bT2njCBm2o86o/clvR
+WV0jasBOKvsgcg65U10klngBEG56nWSdIE9PrXGPqwBwA6jXttAwFLUNvuam5nGQ
+V3MbCFDeHUUrtwIDAQABo38wfTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNV
+HQ4EFgQUPJGTSMevbRtDlY9WBrs21ae/+5YwIQYDVR0RBBowGIEWYnJlbm5hbm11
+c2ljQGdtYWlsLmNvbTAhBgNVHRIEGjAYgRZicmVubmFubXVzaWNAZ21haWwuY29t
+MA0GCSqGSIb3DQEBBQUAA4IBAQBr5Fm1ULBbLUYvX85iXI1Q7L7BZp53p9Q4TXrd
+0n0dE0qHJELwJQkErwZwMhOzaLqcctCwdzLkc9/VZSpLLuzv5bfEWP4EyTET5a+i
+4dn/Wko+6aNPVonmlHDhNYOPl3edxgxoD0WW08U7NJ4tGxJJURVv4yrCayT3xLCA
+yzUUQaHKkLaXHSfH0yhha+pFpUlTsLeg9hrZ0jqSk4FzUrKbiieLY/f/p5Xr5QDg
+fXe/xr/Sc+2wCjHPVE2J+auN5hk3KCp1I4s2fKqyLIwyhTEF3shuYfCpC8rt/YdN
+cy9/lg5LCI3OvakzxL4Xt1Sq4h/xJZ06ydTVJ1wxfk6BXHrg
+-----END CERTIFICATE-----

data/checksum/pg_conduit-0.1.0.gem.sha512

@@ -0,0 +1 @@
+2401cb5efe647705eee22f6ae64a183012e5552bb2417dcc1c78cf52c5c44901925b849443685e63402da82e97a8d8c88c1fe3c1c52d86d184862e900258af58

data/docker-compose.yml

@@ -0,0 +1,34 @@
+version: "3"
+services:
+  yard-docs:
+    build: .
+    command: bundle exec yard server --reload
+    ports:
+      - 9999:8808
+    volumes:
+      - .:/src
+      - bundle:/bundle
+
+  test:
+    build: .
+    command: bundle exec guard --no-bundler-warning --no-interactions
+    volumes:
+      - .:/src
+      - bundle:/bundle
+    environment:
+      - TEST_DB_SRC=postgres://postgres@db/pg_conduit_src_test
+      - TEST_DB_DEST=postgres://postgres@db/pg_conduit_dest_test
+      - TEST_DB_HOST=postgres://postgres@db/
+    links:
+      - db
+
+  db:
+    image: postgres:9.6.6
+    ports:
+      - 5499:5432
+    volumes:
+      - database:/var/lib/postgresql/data
+
+volumes:
+  bundle:
+  database:

data/lib/pg_conduit/connections.rb

@@ -0,0 +1,24 @@
+require 'connection_pool'
+
+module PgConduit
+  class Connections
+    attr_reader :src_pool, :dest_pool
+
+    def self.init_pool(params)
+      ConnectionPool.new { PG::Connection.open(params) }
+    end
+
+    def initialize(source, destination)
+      @src_pool   = self.class.init_pool source
+      @dest_pool  = self.class.init_pool destination
+    end
+
+    def with_source
+      @src_pool.with { |conn| yield conn }
+    end
+
+    def with_destination
+      @dest_pool.with { |conn| yield conn }
+    end
+  end
+end

data/lib/pg_conduit/db_writer.rb

@@ -0,0 +1,11 @@
+module PgConduit
+  class DBWriter
+    def initialize(pool)
+      @pool = pool
+    end
+
+    def write
+      @pool.with { |conn| conn.exec yield }
+    end
+  end
+end

data/lib/pg_conduit/file_writer.rb

@@ -0,0 +1,11 @@
+module PgConduit
+  class FileWriter
+    def initialize(path)
+      @path = path
+    end
+
+    def write
+      open(@path, 'a') { |f| f.puts yield }
+    end
+  end
+end

data/lib/pg_conduit/null_writer.rb

@@ -0,0 +1,7 @@
+module PgConduit
+  class NullWriter
+    def write
+      nil.tap { yield }
+    end
+  end
+end

data/lib/pg_conduit/parallel_stream_reader.rb

@@ -0,0 +1,60 @@
+module PgConduit
+  # A multi threaded stream reader
+  class ParallelStreamReader
+    # @param query_stream [PgConduit::QueryStream]
+    # @param threads [Integer] The number of threads to use for workers
+    # @param queue_max_size [Integer] How many rows should be stored in memory
+    #   in the work queue.
+    def initialize(query_stream, threads: 5, queue_max_size: 1000)
+      @queue = SizedQueue.new(queue_max_size)
+      @workers = threads
+      @stream = query_stream
+    end
+
+    # Read A QueryStream and yield it's rows
+    #
+    # @yield [Hash] A single row from the QueryStream. Every row from the stream
+    #   will be yielded but order is not guaranteed.
+    def read(&callback)
+      reader  = read_stream(@stream)
+      workers = dispatch_workers(&callback)
+      reader.join
+      workers.each { |t| t.join }
+      :ok
+    end
+
+    private
+
+    def read_stream(query_stream)
+      Thread.new do
+        query_stream.each_row { |row| @queue << row }
+        @queue.close
+      end
+    end
+
+    def dispatch_workers(&callback)
+      (1..@workers).to_a.map { dispatch_worker(&callback) }
+    end
+
+    def dispatch_worker(&callback)
+      Thread.new do
+        loop do
+          continue = process_next(&callback)
+          break if @queue.closed? && !continue
+        end
+      end
+    end
+
+    def process_next(&callback)
+      continue = false
+      Thread.new do
+        row = @queue.deq
+        if row
+          callback.call row
+          continue = true
+        end
+      end.join
+      continue
+    end
+  end
+end

data/lib/pg_conduit/pipe.rb

@@ -0,0 +1,62 @@
+module PgConduit
+  class Pipe
+    # @example
+    #   Pipe
+    #     .new(from: query_stream, to: db_writer)
+    #     .send('SELECT name FROM users')
+    #     .as do |user|
+    #       %(INSERT INTO friends (name) VALUES ('#{user["full_name"]}'))
+    #     end
+    def initialize(from:, to:)
+      @stream = from
+      @writer = to
+      @reader = ParallelStreamReader.new(@stream)
+      @transformers = []
+    end
+
+    def read(query)
+      self.tap { @stream.query(query) }
+    end
+
+    def transform(&transformer)
+      self.tap { @transformers << transformer }
+    end
+
+    def write
+      exec_read { |row| exec_write { exec_transform(row) } }
+    end
+
+    def peak
+      self.tap { @transformers << ->(row) { row.tap { yield row } } }
+    end
+
+    def write_batched(size: 1000)
+      collector = RowCollector.new(chunk_size: size)
+
+      # Set callback to yield collected rows
+      collector.on_chunk { |rows| exec_write { yield rows } }
+
+      # Process each row
+      exec_read { |row| collector << exec_transform(row) }
+
+      # Yield any remaining rows
+      collector.finish
+    end
+
+    alias_method :exec, :write
+
+    private
+
+    def exec_read(&b)
+      @reader.read(&b)
+    end
+
+    def exec_write(&b)
+      @writer.write(&b)
+    end
+
+    def exec_transform(row)
+      @transformers.reduce(row) { |data, transform| transform.call data }
+    end
+  end
+end

data/lib/pg_conduit/query_stream.rb

@@ -0,0 +1,41 @@
+module PgConduit
+  # Execute a SQL query and provide the results as a stream
+  # @example Print username and email for all users
+  #
+  #   conn    = PG::Connection.open
+  #   stream  = PgConduit::QueryStream.new(conn)
+  #
+  #   stream.query('SELECT * FROM users').each_row do |row|
+  #     puts "#{row['username']}, #{row['email']}"
+  #   end
+  #
+  class QueryStream
+    attr_reader :sql
+
+    # @param pool [ConnectionPool] A pool of PG::Connections
+    def initialize(pool)
+      @pool = pool
+    end
+
+    # @param sql [String] The SQL query to execute
+    # @return [self]
+    def query(sql)
+      self.tap { @sql = sql }
+    end
+
+    # Execute query and yield each row
+    # @yield [Hash] A hash representing a single row from the result set
+    def each_row
+      @pool.with do |conn|
+        conn.send_query @sql
+        conn.set_single_row_mode
+        loop do
+          res = conn.get_result
+          break unless res
+          res.check
+          res.stream_each { |row| yield row }
+        end
+      end
+    end
+  end
+end

data/lib/pg_conduit/row_collector.rb

@@ -0,0 +1,68 @@
+module PgConduit
+  # A thread safe accumulator, used to chunk an input stream
+  class RowCollector
+    # @param chunk_size [Integer] How many rows should be collected before
+    #   yielding
+    def initialize(chunk_size: 100)
+      @chunk_size = chunk_size
+      @rows = []
+      @finished = false
+      @mutex = Mutex.new
+    end
+
+    # Provide a block to be called with each accumulated chunk
+    #
+    # @yield [Array] collected rows
+    # @return [self]
+    #
+    # @example Print once every ten rows
+    #
+    #   collector = RowCollector.new(chunk_size: 10)
+    #   collector.on_chunk { |rows| puts rows }
+    #
+    #   100.times { |n| collector << n }
+    #
+    #   #> [0,1,2,3,4,5,6,7,8,9]
+    #   #> [10,11,12,13,14,15,16,17,18,19]
+    #   #> ...etc
+    #
+    def on_chunk(&callback)
+      self.tap do
+        @mutex.synchronize { @callback = callback }
+      end
+    end
+
+    # @param row [Object] Row to add to the buffer
+    def <<(row)
+      @mutex.synchronize do
+        if @finished
+          raise 'Data may not be added to a row collector that has been marked as finished'
+        end
+
+        @rows << row
+        if @rows.length % @chunk_size == 0
+          flush(&@callback)
+        end
+      end
+    end
+
+    # Flushes any collected rows, yielding them to the callback and marks the
+    # collector as finished. Any subsequent calls to :<< will raise an error.
+    def finish
+      @mutex.synchronize do
+        flush(&@callback)
+        @finished = true
+      end
+    end
+
+    private
+
+    # Yields the collected rows and resets the row collector
+    # @yield [Array<Hash>] The collected rows
+    def flush
+      yield @rows if @rows.length > 0
+      @rows = []
+      true
+    end
+  end
+end

data/lib/pg_conduit/stdout_writer.rb

@@ -0,0 +1,7 @@
+module PgConduit
+  class STDOUTWriter
+    def write
+      puts yield
+    end
+  end
+end

data/lib/pg_conduit/version.rb

@@ -0,0 +1,3 @@
+module PgConduit
+  VERSION = '0.1.0'
+end

data/lib/pg_conduit.rb

@@ -0,0 +1,67 @@
+require 'pg'
+require 'connection_pool'
+require 'pg_conduit/version'
+
+module PgConduit
+  autoload :Connections,          'pg_conduit/connections'
+  autoload :DBWriter,             'pg_conduit/db_writer'
+  autoload :FileWriter,           'pg_conduit/file_writer'
+  autoload :NullWriter,           'pg_conduit/null_writer'
+  autoload :ParallelStreamReader, 'pg_conduit/parallel_stream_reader'
+  autoload :Pipe,                 'pg_conduit/pipe'
+  autoload :QueryStream,          'pg_conduit/query_stream'
+  autoload :RowCollector,         'pg_conduit/row_collector'
+  autoload :STDOUTWriter,         'pg_conduit/stdout_writer'
+
+  class << self
+    # Create a new DB -> DB Pipe
+    #
+    # @param src [String,Hash] Source database connection params
+    # @param dest [String,Hash] Destination database connection params
+    # @return [PgConduit::Pipe]
+    def db_to_db(src, dest)
+      connections   = Connections.new src, dest
+      query_stream  = QueryStream.new connections.src_pool
+      db_writer     = DBWriter.new connections.dest_pool
+
+      Pipe.new from: query_stream, to: db_writer
+    end
+
+    # Create a new DB -> File Pipe
+    #
+    # @param src [String,Hash] Source database connection params
+    # @param dest [Sting,Pathname] Path to destination file
+    # @return [PgConduit::Pipe]
+    def db_to_file(src, dest)
+      pool          = Connections.init_pool src
+      query_stream  = QueryStream.new pool
+      file_writer   = FileWriter.new dest
+
+      Pipe.new from: query_stream, to: file_writer
+    end
+
+    # Create a new DB -> STDOUT Pipe
+    #
+    # @param src [String,Hash] Source database connection params
+    # @return [PgConduit::Pipe]
+    def db_to_stdout(src)
+      pool          = Connections.init_pool src
+      query_stream  = QueryStream.new pool
+      stdout_writer = STDOUTWriter.new
+
+      Pipe.new from: query_stream, to: stdout_writer
+    end
+
+    # Create a new DB -> Null pipe
+    #
+    # @param src [String,Hash] Source database connection params
+    # @return [PgConduit::Pipe]
+    def db_to_null(src)
+      pool          = Connections.init_pool src
+      query_stream  = QueryStream.new pool
+      stdout_writer = NullWriter.new
+
+      Pipe.new from: query_stream, to: stdout_writer
+    end
+  end
+end

data/pg_conduit.gemspec

@@ -0,0 +1,37 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pg_conduit/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pg_conduit'
+  spec.version       = PgConduit::VERSION
+  spec.authors       = ['James Brennan']
+  spec.email         = ['brennanmusic@gmail.com']
+
+  spec.summary       = 'Stream data from one postgres database to another'
+  spec.homepage      = 'https://github.com/jamesBrennan/pg_conduit'
+
+  spec.license       = 'MIT'
+
+  spec.cert_chain  = ['certs/jamesbrennan.pem']
+  spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem') if $0 =~ /gem\z/
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^spec/})
+  end
+
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'pg', '~> 1.0'
+  spec.add_dependency 'connection_pool', '~> 2.2'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4.1'
+  spec.add_development_dependency 'simplecov', '~> 0.16.1'
+  spec.add_development_dependency 'yard', '~> 0.9'
+  spec.add_development_dependency 'guard', '~> 2.14'
+  spec.add_development_dependency 'guard-rspec', '~> 4.7'
+end

metadata

@@ -0,0 +1,236 @@
+--- !ruby/object:Gem::Specification
+name: pg_conduit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- James Brennan
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDgDCCAmigAwIBAgIBATANBgkqhkiG9w0BAQUFADBDMRUwEwYDVQQDDAxicmVu
+  bmFubXVzaWMxFTATBgoJkiaJk/IsZAEZFgVnbWFpbDETMBEGCgmSJomT8ixkARkW
+  A2NvbTAeFw0xODA3MDYwMDA2MzRaFw0xOTA3MDYwMDA2MzRaMEMxFTATBgNVBAMM
+  DGJyZW5uYW5tdXNpYzEVMBMGCgmSJomT8ixkARkWBWdtYWlsMRMwEQYKCZImiZPy
+  LGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyxykNKqp
+  jD5gnQd2YHuRDNsLN5limGMDjSb38lMZAfJJugoWbSSeQM3+iVl26YWE5JeJpqDu
+  Rr/GOQ54bsev4IxCfPl8yIkhAao0i96+k1EeEb/i3pgZOfjXgGSTM71baCZzbqBH
+  A5ivcb0wqoHWyqrxNtc4XhSs5RtvtNl3IEe/bs5qqixt6PwsCorD6kAoUUNHNM9W
+  MLexivfiJooHOvVRmf1DQuJ2fDccKNHf7ZR4PuiguT4Z73bT2njCBm2o86o/clvR
+  WV0jasBOKvsgcg65U10klngBEG56nWSdIE9PrXGPqwBwA6jXttAwFLUNvuam5nGQ
+  V3MbCFDeHUUrtwIDAQABo38wfTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNV
+  HQ4EFgQUPJGTSMevbRtDlY9WBrs21ae/+5YwIQYDVR0RBBowGIEWYnJlbm5hbm11
+  c2ljQGdtYWlsLmNvbTAhBgNVHRIEGjAYgRZicmVubmFubXVzaWNAZ21haWwuY29t
+  MA0GCSqGSIb3DQEBBQUAA4IBAQBr5Fm1ULBbLUYvX85iXI1Q7L7BZp53p9Q4TXrd
+  0n0dE0qHJELwJQkErwZwMhOzaLqcctCwdzLkc9/VZSpLLuzv5bfEWP4EyTET5a+i
+  4dn/Wko+6aNPVonmlHDhNYOPl3edxgxoD0WW08U7NJ4tGxJJURVv4yrCayT3xLCA
+  yzUUQaHKkLaXHSfH0yhha+pFpUlTsLeg9hrZ0jqSk4FzUrKbiieLY/f/p5Xr5QDg
+  fXe/xr/Sc+2wCjHPVE2J+auN5hk3KCp1I4s2fKqyLIwyhTEF3shuYfCpC8rt/YdN
+  cy9/lg5LCI3OvakzxL4Xt1Sq4h/xJZ06ydTVJ1wxfk6BXHrg
+  -----END CERTIFICATE-----
+date: 2018-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+description: 
+email:
+- brennanmusic@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- bin/checksum.rb
+- bin/console
+- bin/setup
+- certs/jamesbrennan.pem
+- checksum/pg_conduit-0.1.0.gem.sha512
+- docker-compose.yml
+- lib/pg_conduit.rb
+- lib/pg_conduit/connections.rb
+- lib/pg_conduit/db_writer.rb
+- lib/pg_conduit/file_writer.rb
+- lib/pg_conduit/null_writer.rb
+- lib/pg_conduit/parallel_stream_reader.rb
+- lib/pg_conduit/pipe.rb
+- lib/pg_conduit/query_stream.rb
+- lib/pg_conduit/row_collector.rb
+- lib/pg_conduit/stdout_writer.rb
+- lib/pg_conduit/version.rb
+- pg_conduit.gemspec
+homepage: https://github.com/jamesBrennan/pg_conduit
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Stream data from one postgres database to another
+test_files: []