qreport 0.0.2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--color -f d

data/ChangeLog

@@ -0,0 +1,11 @@
+2013-06-26  Kurt A. Stephens  <ks.github@kurtstephens.com>
+
+	* v0.0.2: New Version: New Functionality.
+	* Qreport::Connection#conn= for use with existing PGConn objects.
+	* Fix Boolean and Float escape/unescape.
+	* Test Coverage.
+	* Depends on pg gem.
+
+2013-01-15  Kurt A. Stephens  <ks.github@kurtstephens.com>
+
+	* v0.0.1: New Version: Initial version.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in qreport.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Kurt Stephens
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# Qreport
+
+Executes a SQL query into a report table.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'qreport'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install qreport
+
+## Usage
+
+Qreport rewrites a plain SQL query so that its result set can populate a report table.
+It automatically creates the report table based on a signature of the column names and types of the query result.
+It can also add additional columns to the report table for other uses, for example: batch processing.
+New queries, rollups and reports can be built from previous reports.
+
+Currently supports PostgreSQL.
+
+## Example
+
+We have users that write articles.
+Generate a report named "users_with_articles" of all users that have written an article in N days:
+
+    SELECT u.id AS "user_id"
+    FROM   users u
+    WHERE 
+      EXISTS(SELECT * FROM articles a 
+             WHERE a.user_id = u.id AND a.created_on >= NOW() - INTERVAL '30 days')
+
+Create a Qreport::ReportRun:
+
+    conn = Qreport::Connection.new(...)
+    report_run = Qreport::ReportRun.new(:name => :users_with_articles)
+    report_run.sql = <<"END"
+      SELECT u.id AS "user_id"
+      FROM   users u
+      WHERE
+        EXISTS(SELECT * FROM articles a
+               WHERE a.user_id = u.id AND a.created_on >= NOW() - INTERVAL '30 days')
+    END
+    report_run.run! conn
+
+Qreport translates the query above into:
+
+    SELECT 0 AS "qr_run_id"
+         , nextval('qr_row_seq') AS "qr_row_id"
+         , u.id AS "user_id"
+    FROM users u
+    WHERE 
+      EXISTS(SELECT * FROM articles a
+             WHERE a.user_id = u.id AND a.created_on >= NOW() - INTERVAL '30 days')
+
+Then analyzes the columns names and types of this query to produce a result signature.
+The result signature is hashed, e.g.: "x2yu78i".
+The result signature hash is used to create a unique report table name: e.g. "users_with_articles_x2yu78i".
+The qr_report_runs table keeps track of each report run.
+A record is inserted into the qr_report_runs table with a unique id.
+Qreport then executes:
+
+    CREATE TABLE users_with_articles_x2yu78i AS
+    SELECT 123 AS "qr_run_id"
+         , nextval('qr_row_seq') AS "qr_row_id"
+         , u.id AS "user_id"
+    FROM users u
+    WHERE 
+      EXISTS(SELECT * FROM articles a
+             WHERE a.user_id = u.id AND a.created_on >= NOW() - INTERVAL '30 days')
+
+The ReportRun object state is updated:
+
+    report_run.id # => Integer
+    report_run.nrows # => Integer
+    report_run.started_at # => Time
+    report_run.finished_at # => Time
+
+Subsequent queries with the same column signature will use "INSERT INTO users_with_articles_x2yu78i".
+
+## Parameterizing Reports
+
+## Batch Processing
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+gem 'rspec'
+require 'rspec/core/rake_task'
+
+desc "Default => :test"
+task :default => :test
+
+desc "Run all tests"
+task :test => [ :spec ]
+
+desc "Run specs"
+RSpec::Core::RakeTask.new(:spec)
+

data/lib/qreport.rb

@@ -0,0 +1,8 @@
+require "qreport/version"
+
+module Qreport
+  EMPTY_Hash = {}.freeze
+  EMPTY_Array = [].freeze
+  EMPTY_String = ''.freeze
+  class Error < ::Exception; end
+end

data/lib/qreport/connection.rb

@@ -0,0 +1,378 @@
+require 'qreport'
+require 'time' # iso8601
+
+module Qreport
+  class Connection
+    attr_accessor :arguments, :verbose, :verbose_result, :env
+    attr_accessor :conn, :conn_owned
+
+    class << self
+      attr_accessor :current
+    end
+
+    def initialize args = nil
+      @arguments = args
+      initialize_copy nil
+      if conn = @arguments && @arguments.delete(:conn)
+        self.conn = conn
+      end
+    end
+
+    def initialize_copy src
+      @conn = @conn_owned = nil
+      @abort_transaction = @invalid = nil
+      @transaction_nesting = 0
+    end
+
+    def env
+      @env || ENV
+    end
+
+    def arguments
+      @arguments || {
+        :host => env['PGHOST'] || 'test',
+        :port => env['PGPORT'],
+        :user => env['PGUSER'] || 'test',
+        :password => env['PGPASSWORD'] || 'test',
+        :dbname => env['PGDATABASE'] || 'test',
+      }
+    end
+
+    # Returns the PG connection object.
+    # Create a new connection from #arguments.
+    # New connection will be closed by #close.
+    def conn
+      @conn ||=
+        begin
+          if @@require_pg
+            require 'pg'
+            @@require_pg = false
+          end
+          initialize_copy nil
+          c = PG.connect(arguments)
+          @conn_owned = true if c
+          c
+        end
+    end
+    @@require_pg = true
+
+    # Sets the PG connection object.
+    # Connection will not closed by #close.
+    def conn= x
+      @conn_owned = false
+      @conn = x
+    end
+
+    def fd
+      @conn && @conn.socket
+    end
+
+    def close
+      raise Error, "close during transaction" if in_transaction?
+      if @conn
+        conn = @conn
+        @conn = nil
+        conn.close if @conn_owned
+      end
+    ensure
+      @invalid = false
+      @transaction_nesting = 0
+      @conn_owned = false
+    end
+
+    def in_transaction?; @transaction_nesting > 0; end
+
+    def transaction
+      raise Error, "no block" unless block_given?
+      abort = false
+      begin
+        transaction_begin
+        yield
+      rescue ::Exception => exc
+        abort = @abort_transaction = exc
+        raise exc
+      ensure
+        transaction_end abort
+      end
+    end
+
+    def transaction_begin
+      if @transaction_nesting == 0
+        _transaction_begin
+      end
+      @transaction_nesting += 1
+      self
+    end
+
+    def transaction_end abort = nil
+      if (@transaction_nesting -= 1) == 0
+        begin
+          if abort
+            _transaction_abort
+          else
+            _transaction_commit
+          end
+        ensure
+          if @invalid
+            close
+          end
+        end
+      end
+      self
+    end
+
+    def _transaction_begin
+      run "BEGIN"; self
+    end
+
+    def _transaction_commit
+      run "COMMIT"; self
+    end
+
+    def _transaction_abort
+      run "ABORT"; self
+    end
+
+    def table_exists? name, schemaname = 'public'
+      #result = run "SELECT * FROM pg_catalog.pg_tables",
+      #:arguments => { :tablename => name }
+      #pp result.rows
+
+      result =
+      run "SELECT EXISTS(SELECT * FROM pg_catalog.pg_tables WHERE tablename = :tablename AND schemaname = :schemaname) as \"exists\"",
+      :arguments => { :tablename => name, :schemaname => schemaname }
+      # result.rows; pp result
+      result.rows[0]["exists"]
+    end
+
+    # options:
+    #   :arguments => { :key => value, ... }
+    #   :limit => size
+    #   :limit => [ size, offset ]
+    def run sql, options = nil
+      options ||= { }
+      conn = options[:connection] || self.conn
+      result = Query.new
+      result.sql = sql
+      result.options = options
+      result.conn = self
+      result.run!
+      dump_result result if @verbose_result || options[:verbose_result]
+      result
+    end
+
+    # Represents raw SQL.
+    class SQL
+      def self.new sql; self === sql ? sql : super; end
+      def initialize sql; @sql = sql.freeze; end
+      def to_s; @sql; end
+    end
+
+    def safe_sql x
+      SQL.new(x)
+    end
+
+    def escape_identifier name
+      conn.escape_identifier name.to_s
+    end
+
+    def escape_value val
+      case val
+      when SQL
+        val.to_s
+      when nil
+        NULL
+      when true
+        T_
+      when false
+        F_
+      when Integer, Float
+        val
+      when String, Symbol
+        "'" << conn.escape_string(val.to_s) << QUOTE
+      when Time
+        escape_value(val.iso8601(6)) << "::timestamp"
+      when Hash, Array
+        escape_value(val.to_json)
+      else
+        raise TypeError
+      end.to_s
+    end
+    NULL = 'NULL'.freeze
+    QUOTE = "'".freeze
+    T_ = "'t'::boolean".freeze
+    F_ = "'f'::boolean".freeze
+    T = 't'.freeze
+
+    def unescape_value val, type
+      case val
+      when String
+        return nil if val == NULL
+        case type
+        when "boolean"
+          val = val == T
+        when /int/
+          val = val.to_i
+        when "numeric"
+          val = val.to_f
+        when /timestamp/
+          val = Time.parse(val)
+        else
+          val
+        end
+      else
+        val
+      end
+    end
+
+    def dump_result result
+      pp result if result
+      result
+    end
+
+    def type_name type, mod
+      @type_names ||= { }
+      @type_names[[type, mod]] ||=
+        conn.exec("SELECT format_type($1,$2)", [type, mod]).
+        getvalue(0, 0).to_s.dup.freeze
+    end
+
+    def with_limit sql, limit = nil
+      sql = sql.dup
+      case limit
+      when Integer
+        limit = "LIMIT #{limit}"
+      when Array
+        limit = "OFFSET #{limit[1].to_i}\nLIMIT #{limit[0].to_i}"
+      end
+      unless sql.sub!(/:LIMIT\b|\bLIMIT\s+\S+\s*|\Z/im, "\n#{limit}")
+        sql << "\n" << limit
+      end
+      sql
+    end
+
+    def run_query! sql, query, options = nil
+      options ||= EMPTY_Hash
+      result = nil
+      begin
+        result = conn.async_exec(sql)
+      rescue ::PG::Error => exc
+        # $stderr.puts "  ERROR: #{exc.inspect}\n  #{exc.backtrace * "\n  "}"
+        query.error = exc.inspect
+        raise exc unless options[:capture_error]
+      rescue ::Exception => exc
+        @invalid = true
+        query.error = exc.inspect
+        raise exc unless options[:capture_error]
+      end
+      result
+    end
+
+    class Query
+      attr_accessor :conn, :sql, :options
+      attr_accessor :error, :cmd_status_raw, :cmd_status, :cmd_tuples
+      attr_accessor :nfields, :fields, :ftypes, :fmods
+      attr_accessor :type_names
+      attr_accessor :columns, :rows
+
+      def run!
+        @error = nil
+        sql = @sql_prepared = prepare_sql self.sql
+        if conn.verbose || options[:verbose]
+          $stderr.puts "\n-- =================================================================== --"
+          $stderr.puts sql
+          $stderr.puts "-- ==== --"
+        end
+        return self if options[:dry_run]
+        if result = conn.run_query!(sql, self, options)
+          extract_results! result
+        end
+        self
+      end
+
+      def prepare_sql sql
+        sql = sql.sub(/[\s\n]*;[\s\n]*\Z/, '')
+        if options.key?(:limit)
+          sql = conn.with_limit sql, options[:limit]
+        end
+        if arguments = options[:arguments]
+          if values = arguments[:names_and_values]
+            n = conn.safe_sql(values.keys * ', ')
+            v = conn.safe_sql(values.keys.map{|k| ":#{k}"} * ', ')
+            sql = sql_replace_arguments(sql,
+                              :NAMES  => n,
+                              :VALUES => v,
+                              :NAMES_AND_VALUES => conn.safe_sql("( #{n} ) VALUES ( #{v} )"),
+                              :SET_VALUES => conn.safe_sql(values.keys.map{|k| "#{conn.escape_identifier(k)} = :#{k}"} * ', '))
+            arguments = arguments.merge(values)
+          end
+          sql = sql_replace_arguments(sql, arguments)
+        end
+        sql
+      end
+
+      def sql_replace_arguments sql, arguments
+        sql = sql.gsub(/(:(\w+)\b([?]?))/) do | m |
+          name = $2.to_sym
+          optional = ! $3.empty?
+          if arguments.key?(name) || optional
+            val = arguments[name]
+            unless optional && val.nil?
+              val = conn.escape_value(val)
+            end
+            $stderr.puts "  #{name} => #{val}" if options[:verbose_arguments]
+            val
+          else
+            $1
+          end
+        end
+        sql
+      end
+
+      def extract_results! result
+        error = result.error_message
+        error &&= ! error.empty? && error
+        @error = error
+        @cmd_status_raw = result.cmd_status
+        @cmd_tuples = result.cmd_tuples
+        @nfields = result.nfields
+        @ntuples = result.ntuples
+        @fields = result.fields
+        @ftypes = (0 ... nfields).map{|i| result.ftype(i) }
+        @fmods  = (0 ... nfields).map{|i| result.fmod(i) }
+        @rows = result.to_a
+        result.clear
+        self
+      end
+
+      def columns
+        @columns ||= @fields.zip(type_names)
+      end
+
+      def type_names
+        @type_names ||= (0 ... nfields).map{|i| conn.type_name(@ftypes[i], @fmods[i])}
+      end
+
+      def cmd_status
+        @cmd_status ||=
+          begin
+            x = @cmd_status_raw.split(/\s+/)
+            [ x[0] ] + x[1 .. -1].map(&:to_i)
+          end.freeze
+      end
+
+      def rows
+        return @rows if @rows_unescaped
+        @rows.each do | r |
+          columns.each do | c, t |
+            r[c] = conn.unescape_value(r[c], t)
+          end
+        end
+        @rows_unescaped = true
+        @rows
+      end
+
+    end
+  end
+end
+