pg_exec_array_params 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 487eaa0af43d806c4d5ead7fe7ba640a23f0f382611773ed79565c583740fb5c
+  data.tar.gz: e6b1b0c0347546658a9786642a19f77acd173d1413c9174165e4ec12f312e105
+SHA512:
+  metadata.gz: c7f70ef196f4353edf9195224598e83b82be4e5811ae2610b631bf416ccdceac89cdedf133aa96ad1c706e1e6994dbf074d171584fec9bbff91e61775b3c613a
+  data.tar.gz: 4410043f791594bc5367ffc7c94d75ab56bfd5cfeaf84076b2745ae27854d56645aec7d6eea5a7bf1984f6c16d373b1da1ec5e25c13cf360d33ffd9c4bbe26f2

data/.github/workflows/branch.yml

@@ -0,0 +1,50 @@
+name: branch
+on:
+  push:
+    branches-ignore:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: pg_exec_array_params_test
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      matrix:
+        ruby: [ '2.7' ]
+        pg: [ '>= 1' ]
+    name: Ruby ${{ matrix.ruby }}, pg ${{ matrix.pg }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ matrix.pg }}-${{ hashFiles('**/Gemfile') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ matrix.pg }}
+      - name: Test
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: pg_exec_array_params_test
+          PG: ${{ matrix.pg }}
+          PG_GEM_VERSION: ${{ matrix.pg }}
+          CI: true
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: |
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3 --without benchmark
+          bundle exec rake ci

data/.github/workflows/ci.yml

@@ -0,0 +1,55 @@
+name: ci
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  push:
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: pg_exec_array_params_test
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    strategy:
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7' ]
+        pg: [ '< 1', '>= 1' ]
+    name: Ruby ${{ matrix.ruby }}, pg ${{ matrix.pg }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ matrix.pg }}-${{ hashFiles('**/Gemfile') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ matrix.pg }}
+      - name: Test
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: pg_exec_array_params_test
+          PG: ${{ matrix.pg }}
+          PG_GEM_VERSION: ${{ matrix.pg }}
+          CI: true
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: |
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3 --without benchmark
+          bundle exec rake ci

data/.gitignore

@@ -0,0 +1,6 @@
+Gemfile.lock
+.rspec_status
+.bundle
+.rspec
+bin
+coverage/

data/.rubocop.yml

@@ -0,0 +1,11 @@
+Metrics/AbcSize:
+  Max: 30
+
+Metrics/MethodLength:
+  Max: 30
+
+Metrics/BlockLength:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_exec_array_params.gemspec
+gemspec
+
+gem 'codecov', require: false
+gem 'rubocop', '~> 1.0'
+gem 'rubocop-rspec', '~> 2.0.0.pre'
+gem 'rspec-github', '~> 2.3'
+gem 'pry-byebug', '~> 3.9'
+gem 'simplecov', '~> 0.19'
+
+group :benchmark do
+  gem 'activerecord', require: false
+  gem 'benchmark-ips', require: false
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2020 Vlad Bokov
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,115 @@
+# PgExecArrayParams
+
+![](https://github.com/lunatic-cat/pg_exec_array_params/workflows/ci/badge.svg)
+
+Use same parametized query and put `Array<T>` instead of any `T`
+
+## Example
+
+```ruby
+query = 'select * from t1 where a1 = $1 and a2 = $2 and a3 = $3 and a4 = $4'
+params = [1, [2, 3], 'foo', ['bar', 'baz']]
+
+# PG::Connection.exec_params called with:
+# 'SELECT * FROM "t1" WHERE "a1" = $1 AND "a2" IN ($2, $3) AND "a3" = $4 AND "a4" IN ($5, $6)'
+# [1, 2, 3, "foo", "bar", "baz"]
+PgExecArrayParams.exec_array_params(conn, query, params)
+```
+
+## Problem
+
+```ruby
+conn.exec_params('select * from users where id IN ($1)', [1,2])
+=> PG::IndeterminateDatatype: ERROR:  could not determine data type of parameter $2
+
+conn.exec_params('select * from users where id IN ($1)', [[1,2]])
+=> PG::InvalidTextRepresentation: ERROR:  invalid input syntax for integer: "[1, 2]"
+```
+
+Currently you would generate `$n` parts and flatten params.
+Or you can inline and embed arrays into query. *Don't forget to escape them*
+
+## Solution
+
+This library encapsulates the first approach in a clean way:
+
+```ruby
+# rewrite query under the hood to
+# select * from users where id IN ($1, $2)
+PgExecArrayParams.exec_array_params(conn, 'select * from users where id = $1', [[1,2]])
+=> [{"id" => 1}, {"id" => 2}]
+```
+
+## Integration with 'pg' gem
+
+```ruby
+PG::Connection.include(PgExecArrayParams) # once in initializer
+
+conn.exec_array_params('select * from users where id = $1', [[1,2]])
+=> [{"id" => 1}, {"id" => 2}]
+```
+
+## Rails note
+
+`ActiveRecord` uses the second path (inline + escape).
+
+```ruby
+User.where(age: ["1'; drop table users;", "2"]).to_sql
+=> SELECT "users".* FROM "users" WHERE "users"."age" IN ('1''; drop table users;', '2')
+```
+
+It's solid and bulletproof, but
+
+- it must support multiple databases, but non-trivial queries require raw sql chunks anyway
+- it's clever, but not so fast as raw `pg`
+- if you're using `AR::Relation#to_sql` just to handle arrays, consider using this
+
+## Benchmark
+
+```sh
+BENCH_PG_URL='postgres://...' bundle exec ruby benchmark.rb
+```
+
+<details>
+<summary>Benchmarking SQL generation</summary>
+
+```
+Warming up --------------------------------------
+        activerecord     1.070k i/100ms
+   exec_array_params   213.704k i/100ms
+Calculating -------------------------------------
+        activerecord     11.359k (± 3.9%) i/s -     56.710k in   5.000406s
+   exec_array_params      2.151M (± 3.0%) i/s -     10.899M in   5.072579s
+```
+</details>
+
+```
+Comparison:
+   exec_array_params:  2150601.0 i/s
+        activerecord:    11359.0 i/s - 189.33x  (± 0.00) slower
+```
+
+<details>
+<summary>Benchmarking query</summary>
+
+```
+Warming up --------------------------------------
+   activerecord#to_a     1.000  i/100ms
+  activerecord#pluck     1.000  i/100ms
+   exec_array_params     2.000  i/100ms
+                  pg     2.000  i/100ms
+Calculating -------------------------------------
+   activerecord#to_a      4.429  (± 0.0%) i/s -     23.000  in   5.203405s
+  activerecord#pluck     18.889  (± 5.3%) i/s -     95.000  in   5.044102s
+   exec_array_params     25.093  (± 4.0%) i/s -    126.000  in   5.039405s
+                  pg     23.632  (± 8.5%) i/s -    118.000  in   5.033961s
+```
+</details>
+
+```
+Comparison:
+   exec_array_params:       25.1 i/s
+                  pg:       23.6 i/s - same-ish: difference falls within error
+  activerecord#pluck:       18.9 i/s - 1.33x  (± 0.00) slower
+   activerecord#to_a:        4.4 i/s - 5.67x  (± 0.00) slower
+```

data/Rakefile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+RSpec::Core::RakeTask.new(:spec_github) do |t|
+  t.rspec_opts = '--format RSpec::Github::Formatter -f progress'
+end
+
+task ci: %i[rubocop spec_github]
+task default: %i[rubocop spec]

data/benchmark.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'benchmark/ips'
+require 'active_record'
+require 'pg'
+
+$LOAD_PATH.unshift(File.expand_path('lib', __dir__))
+require 'pg_exec_array_params'
+
+class User < ActiveRecord::Base
+end
+
+def connect(url)
+  # ActiveRecord::Base.legacy_connection_handling = false
+  ActiveRecord::Base.logger = Logger.new(IO::NULL)
+  ActiveRecord::Base.configurations = { default_env: { adapter: 'postgresql', url: url, pool: 1 } }
+  ActiveRecord::Base.establish_connection
+  ActiveRecord::Migration.verbose = false
+  ActiveRecord::Schema.define(version: 1) do
+    create_table :users, if_not_exists: true do |t|
+      t.integer :age
+    end
+    add_index(:users, :age) unless index_exists?(:users, :age)
+  end
+  ActiveRecord::Base.connection.raw_connection
+end
+
+def sql
+  puts 'Benchmarking SQL generation'
+
+  connect(ENV.fetch('BENCH_PG_URL'))
+  params = ["1'; drop table users;", '2']
+  query = 'select * from users where age = $1'
+
+  Benchmark.ips do |x|
+    x.report('activerecord') { User.where(age: params).to_sql }
+    x.report('exec_array_params') { PgExecArrayParams::Query.new(query, params).sql }
+    x.compare!
+  end
+end
+
+def query
+  puts 'Benchmarking query'
+
+  conn = connect(ENV.fetch('BENCH_PG_URL'))
+  PG::Connection.include(PgExecArrayParams)
+  if conn.exec('select count(*) from users').first['count'].to_i < 1_000_000
+    puts "Seed #{conn.exec('INSERT INTO users (age) SELECT generate_series(1,1500000) % 90;')}"
+  end
+
+  query = 'select * from users where age IN ($1, $2, $3)'
+  params = [10, 20, 30]
+  query2 = 'select * from users where age = $1'
+  params2 = [[10, 20, 30]]
+
+  Benchmark.ips do |x|
+    x.report('activerecord#to_a') { User.where(age: params).to_a }
+    x.report('activerecord#pluck') { User.where(age: params).pluck(:id, :age) }
+    x.report('exec_array_params')  { conn.exec_array_params(query2, params2).to_a }
+    x.report('pg') { conn.exec_params(query, params).to_a }
+    x.compare!
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  if (meth = ARGV.first)
+    send meth
+  else
+    sql && query
+  end
+end

data/lib/pg_exec_array_params.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'pg_exec_array_params/query'
+require 'pg_exec_array_params/version'
+
+module PgExecArrayParams
+  class Error < StandardError; end
+  module_function
+
+  def exec_array_params(conn, sql, params, *args)
+    Query.new(sql, params).exec_params(conn, *args)
+  end
+
+  def self.included(base)
+    return unless base.name == 'PG::Connection'
+
+    base.define_method :exec_array_params do |sql, params, *args|
+      Query.new(sql, params).exec_params(self, *args)
+    end
+  end
+end

data/lib/pg_exec_array_params/query.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'pg_query'
+
+module PgExecArrayParams
+  class Query
+    PARAM_REF = 'ParamRef'
+    REXPR = 'rexpr'
+    A_EXPR = 'A_Expr'
+    KIND = 'kind'
+    LOCATION = 'location'
+    NUMBER = 'number'
+
+    EQ_KIND = 0
+    IN_KIND = 7
+
+    attr_reader :query, :args
+
+    def initialize(query, args = [])
+      @query = query
+      @args = args
+    end
+
+    def exec_params(conn, *args)
+      conn.exec_params(sql, binds, *args)
+    end
+
+    def sql
+      return query unless should_rebuild?
+
+      @sql || (rebuild_query! && @sql)
+    end
+
+    def binds
+      return args unless should_rebuild?
+
+      @binds || (rebuild_query! && @binds)
+    end
+
+    private
+
+    def should_rebuild?
+      args.any? { |param| param.is_a?(Array) }
+    end
+
+    def rebuild_query!
+      @param_idx = 0
+      @ref_idx = 1
+      @binds = []
+      each_param_ref do |value|
+        # puts({value_before: value}.inspect)
+
+        if args[@param_idx].is_a? Array
+          value[KIND] = IN_KIND
+          value[REXPR] = []
+          args[@param_idx].each do |param|
+            raise Error, "Param: #{param.inspect} not primitive" if param.respond_to?(:each)
+
+            value[REXPR] << { PARAM_REF => { NUMBER => @ref_idx } }
+            @binds << param
+            @ref_idx += 1
+          end
+        else
+          value[REXPR][PARAM_REF][NUMBER] = @ref_idx
+          @ref_idx += 1
+
+          # nested_refs == 1 unwraps, wrap it back
+          value[REXPR] = [value[REXPR]] if value[KIND] == IN_KIND
+
+          @binds << args[@param_idx]
+        end
+
+        @param_idx += 1
+        # puts({value_after_: value}.inspect)
+      end
+      @sql = tree.deparse
+      # puts({sql: @sql, binds: @binds}.inspect)
+      true
+    end
+
+    def tree
+      @tree ||= PgQuery.parse(query)
+    end
+
+    def each_param_ref
+      tree.send :treewalker!, tree.tree do |_expr, key, value, _location|
+        if key == A_EXPR
+          if assign_param_via_eq?(value)
+            yield value
+          elsif (nested_refs = assign_param_via_in?(value))
+            if nested_refs == 1
+              value[REXPR] = value[REXPR].first
+              yield value
+            else
+              message = [
+                'Cannot splice multiple references, leave the only one:',
+                query,
+                refs_underline(value)
+              ].join("\n")
+              raise Error, message
+            end
+          end
+        end
+      end
+    end
+
+    def refs_underline(value)
+      from, size = refs_at(value)
+      "#{'^'.rjust(from, ' ')}#{'-'.rjust(size, '-')}^"
+    end
+
+    def refs_at(value)
+      first_ref = value[REXPR].find { |vexpr| vexpr.key?(PARAM_REF) } [PARAM_REF]
+      last_ref = value[REXPR].reverse.find { |vexpr| vexpr.key?(PARAM_REF) } [PARAM_REF]
+      started = first_ref[LOCATION] + 1
+      ended = last_ref[LOCATION] + last_ref[NUMBER].to_s.size
+      [started, ended - started]
+    end
+
+    # = $1
+    # {"kind"=>0, "name"=>[{"String"=>{"str"=>"="}}],
+    #  "lexpr"=>{"ColumnRef"=>{"fields"=>[{"String"=>{"str"=>"companies"}}, {"String"=>{"str"=>"id"}}],
+    #                          "location"=>1242}},
+    #  "rexpr"=>{"ParamRef"=>{"number"=>4, "location"=>1261}}, "location"=>1259}
+    def assign_param_via_eq?(value)
+      (value[KIND] == EQ_KIND) && value[REXPR].is_a?(Hash) && value[REXPR].key?(PARAM_REF)
+    end
+
+    # IN ($1), returns number of nested REFs
+    def assign_param_via_in?(value)
+      (value[KIND] == IN_KIND) && value[REXPR].is_a?(Array) && value[REXPR].count { |vexpr| vexpr.key?(PARAM_REF) }
+    end
+  end
+end

data/lib/pg_exec_array_params/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PgExecArrayParams
+  VERSION = '0.1.0'
+end

data/pg_exec_array_params.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'lib/pg_exec_array_params/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pg_exec_array_params'
+  spec.version       = PgExecArrayParams::VERSION
+  spec.authors       = ['Vlad Bokov']
+  spec.email         = ['vlad@lunatic.cat']
+  spec.license       = 'MIT'
+
+  spec.summary       = 'PG::Connection#exec_params with arrays'
+  spec.description   = 'Escape each array element inside PG::Connection#exec_params properly'
+  spec.homepage      = 'https://github.com/lunatic-cat/pg_exec_array_params'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.4.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/lunatic-cat/pg_exec_array_params'
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_dependency('pg_query', '~> 1.2')
+
+  spec.add_development_dependency('pg', ENV.fetch('PG_GEM_VERSION', '~> 0'))
+  spec.add_development_dependency('rspec', '~> 3.0')
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: pg_exec_array_params
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vlad Bokov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-10-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Escape each array element inside PG::Connection#exec_params properly
+email:
+- vlad@lunatic.cat
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/branch.yml"
+- ".github/workflows/ci.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- benchmark.rb
+- lib/pg_exec_array_params.rb
+- lib/pg_exec_array_params/query.rb
+- lib/pg_exec_array_params/version.rb
+- pg_exec_array_params.gemspec
+homepage: https://github.com/lunatic-cat/pg_exec_array_params
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/lunatic-cat/pg_exec_array_params
+  source_code_uri: https://github.com/lunatic-cat/pg_exec_array_params
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: PG::Connection#exec_params with arrays
+test_files: []