sequel_pg_extended_columns 1.6.19

data/lib/sequel/extensions/pg_streaming.rb

@@ -0,0 +1,149 @@
+unless Sequel::Postgres.respond_to?(:supports_streaming?)
+  raise LoadError, "either sequel_pg not loaded, or an old version of sequel_pg loaded"
+end
+unless Sequel::Postgres.supports_streaming?
+  raise LoadError, "streaming is not supported by the version of libpq in use"
+end
+
+# Database methods necessary to support streaming.  You should load this extension
+# into your database object:
+#
+#   DB.extension(:pg_streaming)
+#
+# Then you can call #stream on your datasets to use the streaming support:
+#
+#   DB[:table].stream.each{|row| ...}
+#
+# Or change a set so that all dataset calls use streaming:
+#
+#   DB.stream_all_queries = true
+module Sequel::Postgres::Streaming
+  attr_accessor :stream_all_queries
+
+  # Also extend the database's datasets to support streaming.
+  # This extension requires modifying connections, so disconnect
+  # so that new connections will get the methods.
+  def self.extended(db)
+    db.extend_datasets(DatasetMethods)
+    db.stream_all_queries = false
+    db.disconnect
+  end
+
+  # Make sure all new connections have the appropriate methods added.
+  def connect(server)
+    conn = super
+    conn.extend(AdapterMethods)
+    conn
+  end
+
+  private
+
+  # If streaming is requested, and a prepared statement is not
+  # used, tell the connection to use single row mode for the query.
+  def _execute(conn, sql, opts={}, &block)
+    if opts[:stream] && !sql.is_a?(Symbol) 
+      conn.single_row_mode = true
+    end
+    super
+  end
+
+  # If streaming is requested, send the prepared statement instead
+  # of executing it and blocking.
+  def _execute_prepared_statement(conn, ps_name, args, opts)
+    if opts[:stream]
+      conn.send_prepared_statement(ps_name, args)
+    else
+      super
+    end
+  end
+
+  module AdapterMethods
+    # Whether the next query on this connection should use
+    # single_row_mode.
+    attr_accessor :single_row_mode
+
+    # Send the prepared statement on this connection using
+    # single row mode.
+    def send_prepared_statement(ps_name, args)
+      send_query_prepared(ps_name, args)
+      set_single_row_mode
+      block
+      self
+    end
+
+    private
+
+    if Sequel::Database.instance_methods.map(&:to_s).include?('log_connection_yield')
+      # If using single row mode, send the query instead of executing it.
+      def execute_query(sql, args)
+        if @single_row_mode
+          @single_row_mode = false
+          @db.log_connection_yield(sql, self, args){args ? send_query(sql, args) : send_query(sql)}
+          set_single_row_mode
+          block
+          self
+        else
+          super
+        end
+      end
+    else
+      def execute_query(sql, args)
+        if @single_row_mode
+          @single_row_mode = false
+          @db.log_yield(sql, args){args ? send_query(sql, args) : send_query(sql)}
+          set_single_row_mode
+          block
+          self
+        else
+          super
+        end
+      end
+    end
+  end
+
+  # Dataset methods used to implement streaming.
+  module DatasetMethods
+    # If streaming has been requested and the current dataset
+    # can be streamed, request the database use streaming when
+    # executing this query, and use yield_each_row to process
+    # the separate PGresult for each row in the connection.
+    def fetch_rows(sql)
+      if stream_results?
+        execute(sql, :stream=>true) do |conn|
+          yield_each_row(conn){|h| yield h}
+        end
+      else
+        super
+      end
+    end
+
+    # Use streaming to implement paging.
+    def paged_each(opts=Sequel::OPTS, &block)
+      stream.each(&block)
+    end
+
+    # Return a clone of the dataset that will use streaming to load
+    # rows.
+    def stream
+      clone(:stream=>true)
+    end
+
+    private
+
+    # Only stream results if streaming has been specifically requested
+    # and the query is streamable.
+    def stream_results?
+      (@opts[:stream] || db.stream_all_queries) && streamable?
+    end
+
+    # Queries using cursors are not streamable, and queries that use
+    # the map/select_map/to_hash/to_hash_groups optimizations are not
+    # streamable, but other queries are streamable.
+    def streamable?
+      spgt = (o = @opts)[:_sequel_pg_type]
+      (spgt.nil? || spgt == :model) && !o[:cursor]
+    end
+  end
+end
+
+Sequel::Database.register_extension(:pg_streaming, Sequel::Postgres::Streaming)

data/lib/sequel_pg/sequel_pg.rb

@@ -0,0 +1,103 @@
+# Add speedup for model class creation from dataset
+class Sequel::Postgres::Database
+  # Whether to optimize loads for all model datasets created from this dataset.
+  # Has certain limitations, see the README for details.
+  attr_accessor :optimize_model_load
+end
+
+# Add faster versions of Dataset#map, #to_hash, #select_map, #select_order_map, and #select_hash
+class Sequel::Postgres::Dataset
+  # Set whether to enable optimized model loading for this dataset.
+  attr_writer :optimize_model_load
+
+  # In the case where an argument is given, use an optimized version.
+  def map(sym=nil)
+    if sym
+      if block_given?
+        super
+      else
+        rows = []
+        clone(:_sequel_pg_type=>:map, :_sequel_pg_value=>sym).fetch_rows(sql){|s| rows << s}
+        rows
+      end
+    else
+      super
+    end
+  end
+
+  # If this dataset has turned model loading on or off, use the default value from
+  # the Database object.
+  def optimize_model_load
+    defined?(@optimize_model_load) ? @optimize_model_load : db.optimize_model_load
+  end
+
+  # In the case where both arguments given, use an optimized version.
+  def to_hash(key_column, value_column = nil, opts = Sequel::OPTS)
+    if value_column && !opts[:hash]
+      clone(:_sequel_pg_type=>:hash, :_sequel_pg_value=>[key_column, value_column]).fetch_rows(sql){|s| return s}
+    elsif opts.empty?
+      super(key_column, value_column)
+    else
+      super
+    end
+  end
+
+  # In the case where both arguments given, use an optimized version.
+  def to_hash_groups(key_column, value_column = nil, opts = Sequel::OPTS)
+    if value_column && !opts[:hash]
+      clone(:_sequel_pg_type=>:hash_groups, :_sequel_pg_value=>[key_column, value_column]).fetch_rows(sql){|s| return s}
+    elsif opts.empty?
+      super(key_column, value_column)
+    else
+      super
+    end
+  end
+
+  # If model loads are being optimized and this is a model load, use the optimized
+  # version.
+  def each
+    if (rp = row_proc) && optimize_model_load?
+      clone(:_sequel_pg_type=>:model, :_sequel_pg_value=>rp).fetch_rows(sql, &Proc.new)
+    else
+      super
+    end
+  end
+    
+  protected
+
+  # Always use optimized version
+  def _select_map_multiple(ret_cols)
+    rows = []
+    clone(:_sequel_pg_type=>:array).fetch_rows(sql){|s| rows << s}
+    rows
+  end
+
+  # Always use optimized version
+  def _select_map_single
+    rows = []
+    clone(:_sequel_pg_type=>:first).fetch_rows(sql){|s| rows << s}
+    rows
+  end
+
+  private
+
+  # The model load can only be optimized if it's for a model and it's not a graphed dataset
+  # or using a cursor.
+  def optimize_model_load?
+    (rp = row_proc).is_a?(Class) && (rp < Sequel::Model) && optimize_model_load && !opts[:use_cursor] && !opts[:graph]
+  end
+end
+
+if defined?(Sequel::Postgres::PGArray)
+  # pg_array extension previously loaded
+
+  class Sequel::Postgres::PGArray::Creator
+    # Override Creator to use sequel_pg's C-based parser instead of the pure ruby parser.
+    def call(string)
+      Sequel::Postgres::PGArray.new(Sequel::Postgres.parse_pg_array(string, @converter), @type)
+    end
+  end
+
+  # Remove the pure-ruby parser, no longer needed.
+  Sequel::Postgres::PGArray.send(:remove_const, :Parser)
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: sequel_pg_extended_columns
+version: !ruby/object:Gem::Version
+  version: 1.6.19
+platform: ruby
+authors:
+- Mansoor Ali Khan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.38.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.38.0
+description: |
+  sequel_pg overwrites the inner loop of the Sequel postgres
+  adapter row fetching code with a C version.  The C version
+  is significantly faster than the pure ruby version
+  that Sequel uses by default.
+
+  sequel_pg also offers optimized versions of some dataset
+  methods, as well as adds support for using PostgreSQL
+  streaming.
+email: mansoorkhan108@gmail.com
+executables: []
+extensions:
+- ext/sequel_pg/extconf.rb
+extra_rdoc_files:
+- README.rdoc
+- CHANGELOG
+- MIT-LICENSE
+files:
+- CHANGELOG
+- MIT-LICENSE
+- README.rdoc
+- Rakefile
+- ext/sequel_pg/extconf.rb
+- ext/sequel_pg/sequel_pg.c
+- lib/sequel/extensions/pg_streaming.rb
+- lib/sequel_pg/sequel_pg.rb
+homepage: https://github.com/mansoorkhan108/sequel_pg
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/jeremyevans/sequel_pg/issues
+  changelog_uri: https://github.com/jeremyevans/sequel_pg/blob/master/CHANGELOG
+  documentation_uri: https://github.com/jeremyevans/sequel_pg/blob/master/README.rdoc
+  mailing_list_uri: https://groups.google.com/forum/#!forum/sequel-talk
+  source_code_uri: https://github.com/jeremyevans/sequel_pg
+post_install_message: 
+rdoc_options:
+- "--quiet"
+- "--line-numbers"
+- "--inline-source"
+- "--title"
+- 'sequel_pg: Faster SELECTs when using Sequel with pg'
+- "--main"
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: Faster SELECTs when using Sequel with pg
+test_files: []