pg_stream 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e17576eec6868271a775e28d9faa54b9f9c4a355
+  data.tar.gz: 39d171f0127d2e661257603ea0ad84eec7d53bb1
+SHA512:
+  metadata.gz: aa07dd59490ed44f222f63087f7fbfdb22f8cf2d3e378087a2530096346c1fc8f40d39a4ac64681c423ea615125292331486bd6a7bb6913516836afd7de60b29
+  data.tar.gz: 7cb30df117098486ed0dc1b7635da9aa8b75b31c28d7505b06cdf2a20f7402f682407c82251a8b02f8da487c04eef7b59886d9072ea1ad20a54bec7e83026887

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_stream.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    pg_stream (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    rake (10.4.2)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.4)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.4)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.9)
+  pg_stream!
+  rake (~> 10.0)
+  rspec (~> 3.0.0)

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Lumos Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,76 @@
+# PgStream
+
+PgStream allows you to stream data from Postgres or Redshift, and provides a framework to help you process it. Normally, Postgres and Redshift collect the entire result set in memory and then return it to you at once. PgStream allows you to process the result row by row, which may be the only way to handle very large results.
+
+## Requirements
+
+You must be using PostgreSQL 9.2beta3 or later client libraries. You must be using pg 0.18.2 or later.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_stream'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pg_stream
+
+## Usage
+
+To get started, just pass it a PG::Connection and a query body string:
+
+```ruby
+require 'pg'
+
+body = 'SELECT * FROM huge_table;'
+
+conn = PG::Connection.open(:dbname => 'test')
+query_stream = PgStream::Stream.new(conn, body)
+
+# You can get the headers from the Stream object.
+headers = query_stream.headers
+```
+
+You use the Processor to consume the data from the db. The processor will register `before_execute`, `during_execute` and `after_execute` callbacks. The processor yields the `row` and `row_count` to the `during_execute` callback, and the `row_count` to the `after_execute` callback.
+
+```ruby
+stream_processor = PgStream::Processor.new(query_stream)
+
+def setup_csv
+  lambda do
+    @csv = CSV.open('some_filepath', 'w')
+    @csv << headers
+  end
+end
+
+stream_processor.register(
+  {
+    before_execute: setup_csv,
+    during_execute: ->(row, _row_count) { @csv << row },
+    after_execute: ->(_row_count) { @csv.close }
+  }
+)
+```
+
+You may call register multiple times on a single Processor.
+
+To make the Processor run and yield to your callbacks, just call `execute` on it. It will return the row_count when it is done. Keep in mind this may take seconds or minutes depending on the size of your result, so you will probably want to do this in a background job.
+
+```
+row_count = stream_processor.execute
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/pg_stream/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pg_stream"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/pg_stream/processor.rb

@@ -0,0 +1,34 @@
+module PgStream
+  class Processor
+    CALLBACK_TYPES = [:before_execute, :during_execute, :after_execute]
+
+    def initialize(stream)
+      @stream = stream
+      @callbacks = CALLBACK_TYPES.map do |type|
+        [type, []]
+      end.to_h
+      @row_count = 0
+    end
+
+    def register(args)
+      args.each do |type, function|
+        if CALLBACK_TYPES.include?(type)
+          @callbacks[type] << function
+        else
+          raise "#{type} is not an acceptable callback type. Types include #{CALLBACK_TYPES.join(', ')}"
+        end
+      end
+      @callbacks
+    end
+
+    def execute
+      @callbacks[:before_execute].each(&:call)
+      @stream.each_row do |row|
+        @row_count += 1
+        @callbacks[:during_execute].each { |y| y.call(row, @row_count) }
+      end
+      @callbacks[:after_execute].each { |y| y.call(@row_count) }
+      @row_count
+    end
+  end
+end

data/lib/pg_stream/stream.rb

@@ -0,0 +1,40 @@
+module PgStream
+  class Stream
+    def initialize(db_conn, body)
+      @db_conn = db_conn
+      @db_conn.send_query(body)
+      @db_conn.set_single_row_mode
+    end
+
+    def headers
+      @headers ||= get_headers
+    end
+
+    def each_row(&block)
+      headers # ensure that headers has been called so we have the first row
+      return unless @first_row # dont yield any rows if the query returns no results
+      yield(@first_row)
+
+      loop do
+        res = @db_conn.get_result || break
+        res.check
+        res.each_row do |row|
+          yield(row)
+        end
+      end
+    end
+
+    private
+
+    def get_headers
+      first_row_result = @db_conn.get_result
+      first_row_result.check
+
+      # values is method on the result object, which can have many rows,
+      # but which will only return one row in single_row_mode
+      @first_row = first_row_result.values[0]
+
+      first_row_result.fields
+    end
+  end
+end

data/lib/pg_stream/version.rb

@@ -0,0 +1,3 @@
+module PgStream
+  VERSION = '0.1.0'
+end

data/lib/pg_stream.rb

@@ -0,0 +1,5 @@
+require 'pg_stream/version'
+require 'pg_stream/processor'
+require 'pg_stream/stream'
+
+module PgStream; end

data/pg_stream.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pg_stream/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pg_stream'
+  spec.version       = PgStream::VERSION
+  spec.authors       = ['Rob Froetscher', 'Andrew Xue']
+  spec.email         = ['rfroetscher@lumoslabs.com', 'andrew@lumoslabs.com']
+
+  spec.summary       = %q{Stream data from postgres or Redshift}
+  spec.homepage      = 'https://github.com/lumoslabs/pg_stream'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.9'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0.0'
+
+  spec.add_dependency 'pg', '>= 0.18.2'
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: pg_stream
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Rob Froetscher
+- Andrew Xue
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.2
+description: 
+email:
+- rfroetscher@lumoslabs.com
+- andrew@lumoslabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/pg_stream.rb
+- lib/pg_stream/processor.rb
+- lib/pg_stream/stream.rb
+- lib/pg_stream/version.rb
+- pg_stream.gemspec
+homepage: https://github.com/lumoslabs/pg_stream
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.3
+signing_key: 
+specification_version: 4
+summary: Stream data from postgres or Redshift
+test_files: []
+has_rdoc: