sequel_pg 1.12.5 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 609ef32bd9eb0fe13dad508682adc477415124359e04f1aebbf5e54cdaaf4630
-  data.tar.gz: 31d2bebfa20c348daf4a0e03bee7df3f727b0306fa3f0987bbcfdaa050f2a234
+  metadata.gz: 41aa757cb6f4ca6f56ca26044517a47a68731dd7d51335451f6da2e5b5b34d6f
+  data.tar.gz: ec43d49fd7b4b8821ecbb813f6244aaf46f69866e84b94df96a3c52ef3a6fd41
 SHA512:
-  metadata.gz: 9a1ba95aa91fa7e521b3b6991b716fec37722faca0d2a658a7743e4fdd5e631b789f3c6da667b1b6f84159ab0ce2091d72b753b1c1d7fb08a357ddf645e9b81d
-  data.tar.gz: 97bacb9be53e6781fcdfb60abbd934b8aa269554495559a64c2d223a0dde3f78a8ca64a32dc04898bec69c753656ffcad1005913fe0b5fc34597e4ab01f3267c
+  metadata.gz: 2a2578bec8fed224dcd3c1263cb86a4729cf11318f80d6cdb03366ae7dae6e68995447db06a1105d1ecda50222d2ff6258079fb7395d3f262094e1e05502fd48
+  data.tar.gz: 26a470d0e7d8290cade76b07cecd972be840c838492607a5649c0fd803de2ca7ea2e34b0c77fddcaff734a2a875b1f26ce8d5b03ec0fe0ab887ae3045ffa10e0

data/CHANGELOG

@@ -1,3 +1,23 @@
+=== 1.15.0 (2022-03-16)
+
+* Avoid deprecation warning in the pg_streaming extension on pg 1.3+ when streaming a query with bound parameters (jeremyevans)
+
+* Use pgresult_stream_any when using pg 1.3.4+ for faster streaming (jeremyevans)
+
+* Do not use streaming by default for Dataset#paged_each in the pg_streaming extension (jeremyevans)
+
+* Avoid verbose warning if loading sequel_pg after Sequel pg_array extension (jeremyevans)
+
+=== 1.14.0 (2020-09-22)
+
+* Reduce stack memory usage for result sets with 64 or fewer columns (jeremyevans)
+
+* Support result sets with more than 256 columns by default (jeremyevans) (#39)
+
+=== 1.13.0 (2020-04-13)
+
+* Allow overriding of inet/cidr type conversion using conversion procs (beanieboi, jeremyevans) (#36, #37)
+
 === 1.12.5 (2020-03-23)
 
 * Fix offset calculation for timestamptz types when datetime_class is DateTime and using local application timezone (jeremyevans)

data/README.rdoc

@@ -70,12 +70,6 @@ can do the following:
 
   gem install sequel_pg
 
-Note that by default sequel_pg only supports result sets with up to
-256 columns. If you will have a result set with more than 256 columns,
-you should modify the maximum supported number of columns via:
-
-   gem install sequel_pg -- --with-cflags=\"-DSPG_MAX_FIELDS=512\"
-
 Make sure the pg_config binary is in your PATH so the installation
 can find the PostgreSQL shared library and header files.  Alternatively,
 you can use the POSTGRES_LIB and POSTGRES_INCLUDE environment
@@ -118,19 +112,6 @@ requirements:
 
   rake build
 
-== Platforms Supported
-
-sequel_pg has been tested on the following:
-
-* ruby 1.9.3
-* ruby 2.0
-* ruby 2.1
-* ruby 2.2
-* ruby 2.3
-* ruby 2.4
-* ruby 2.5
-* ruby 2.6
-
 == Known Issues
 
 * You must be using the ISO PostgreSQL date format (which is the

data/ext/sequel_pg/sequel_pg.c

@@ -1,4 +1,4 @@
-#define SEQUEL_PG_VERSION_INTEGER 11205
+#define SEQUEL_PG_VERSION_INTEGER 11500
 
 #include <string.h>
 #include <stdio.h>
@@ -15,9 +15,6 @@
 #include <ruby/version.h>
 #include <ruby/encoding.h>
 
-#ifndef SPG_MAX_FIELDS
-#define SPG_MAX_FIELDS 256
-#endif
 #define SPG_MINUTES_PER_DAY 1440.0
 #define SPG_SECONDS_PER_DAY 86400.0
 
@@ -73,9 +70,11 @@
 PGconn* pg_get_pgconn(VALUE);
 PGresult* pgresult_get(VALUE);
 int pg_get_result_enc_idx(VALUE);
+VALUE pgresult_stream_any(VALUE self, void (*yielder)(VALUE, int, int, void*), void* data);
 
 static int spg_use_ipaddr_alloc;
 static int spg_use_pg_get_result_enc_idx;
+static int spg_use_pg_stream_any;
 
 static VALUE spg_Sequel;
 static VALUE spg_PGArray;
@@ -535,7 +534,7 @@ static VALUE spg_timestamp(const char *s, VALUE self, size_t length, int tz) {
   }
 
   if (remaining < 19) {
-    return spg_timestamp_error(s, self, "unexpected timetamp format, too short");
+    return spg_timestamp_error(s, self, "unexpected timestamp format, too short");
   }
 
   year = parse_year(&p, &remaining);
@@ -931,8 +930,10 @@ static VALUE spg__array_col_value(char *v, size_t length, VALUE converter, int e
       break;
     case 869: /* inet */
     case 650: /* cidr */
-      rv = spg_inet(v, length);
-      break;
+      if (!RTEST(converter)) {
+        rv = spg_inet(v, length);
+        break;
+      }
     default:
       rv = rb_tainted_str_new(v, length);
       PG_ENCODING_SET_NOCHECK(rv, enc_index);
@@ -1079,10 +1080,6 @@ static VALUE spg__col_value(VALUE self, PGresult *res, long i, long j, VALUE* co
         rv = rb_tainted_str_new(v, PQgetlength(res, i, j));
         PG_ENCODING_SET_NOCHECK(rv, enc_index);
         break;
-      case 869: /* inet */
-      case 650: /* cidr */
-        rv = spg_inet(v, PQgetlength(res, i, j));
-        break;
       /* array types */
       case 1009:
       case 1014:
@@ -1220,17 +1217,30 @@ static VALUE spg__col_value(VALUE self, PGresult *res, long i, long j, VALUE* co
             scalar_oid = 22;
             break;
           case 1041:
+            if (RTEST(colconvert[j])) {
+              goto default_cond;
+            }
             array_type = spg_sym_inet;
             scalar_oid = 869;
             break;
           case 651:
+            if (RTEST(colconvert[j])) {
+              goto default_cond;
+            }
             array_type = spg_sym_cidr;
             scalar_oid = 650;
             break;
         }
         rv = spg_array_value(v, PQgetlength(res, i, j), colconvert[j], enc_index, scalar_oid, self, array_type);
         break;
+      case 869: /* inet */
+      case 650: /* cidr */
+        if (colconvert[j] == Qnil) {
+          rv = spg_inet(v, PQgetlength(res, i, j));
+          break;
+        }
       default:
+      default_cond:
         rv = rb_tainted_str_new(v, PQgetlength(res, i, j));
         PG_ENCODING_SET_NOCHECK(rv, enc_index);
         if (colconvert[j] != Qnil) {
@@ -1369,10 +1379,7 @@ static void spg_set_column_info(VALUE self, PGresult *res, VALUE *colsyms, VALUE
   rb_funcall(self, spg_id_columns_equal, 1, rb_ary_new4(nfields, colsyms));
 }
 
-static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
-  PGresult *res;
-  VALUE colsyms[SPG_MAX_FIELDS];
-  VALUE colconvert[SPG_MAX_FIELDS];
+static VALUE spg_yield_hash_rows_internal(VALUE self, PGresult *res, int enc_index, VALUE* colsyms, VALUE* colconvert) {
   long ntuples;
   long nfields;
   long i;
@@ -1382,21 +1389,9 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
   VALUE pg_type;
   VALUE pg_value;
   char type = SPG_YIELD_NORMAL;
-  int enc_index;
-
-  if (!RTEST(rres)) {
-    return self;
-  }
-  res = pgresult_get(rres);
-
-  enc_index = spg_use_pg_get_result_enc_idx ? pg_get_result_enc_idx(rres) : enc_get_index(rres);
 
   ntuples = PQntuples(res);
   nfields = PQnfields(res);
-  if (nfields > SPG_MAX_FIELDS) {
-    rb_raise(rb_eRangeError, "more than %d columns in query (%ld columns detected)", SPG_MAX_FIELDS, nfields);
-  }
-
   spg_set_column_info(self, res, colsyms, colconvert, enc_index);
 
   opts = rb_funcall(self, spg_id_opts, 0);
@@ -1613,6 +1608,40 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
   return self;
 }
 
+#define def_spg_yield_hash_rows(max_fields) static VALUE spg_yield_hash_rows_ ## max_fields(VALUE self, PGresult *res, int enc_index) { \
+  VALUE colsyms[max_fields]; \
+  VALUE colconvert[max_fields]; \
+  return spg_yield_hash_rows_internal(self, res, enc_index, colsyms, colconvert); \
+}
+
+def_spg_yield_hash_rows(16)
+def_spg_yield_hash_rows(64)
+def_spg_yield_hash_rows(256)
+def_spg_yield_hash_rows(1664)
+
+static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
+  PGresult *res;
+  long nfields;
+  int enc_index;
+
+  if (!RTEST(rres)) {
+    return self;
+  }
+  res = pgresult_get(rres);
+
+  enc_index = spg_use_pg_get_result_enc_idx ? pg_get_result_enc_idx(rres) : enc_get_index(rres);
+
+  nfields = PQnfields(res);
+  if (nfields <= 16) return spg_yield_hash_rows_16(self, res, enc_index);
+  else if (nfields <= 64) return spg_yield_hash_rows_64(self, res, enc_index);
+  else if (nfields <= 256) return spg_yield_hash_rows_256(self, res, enc_index);
+  else if (nfields <= 1664) return spg_yield_hash_rows_1664(self, res, enc_index);
+  else rb_raise(rb_eRangeError, "more than 1664 columns in query (%ld columns detected)", nfields);
+
+  /* UNREACHABLE */
+  return self;
+}
+
 static VALUE spg_supports_streaming_p(VALUE self) {
   return
 #if HAVE_PQSETSINGLEROWMODE
@@ -1632,12 +1661,39 @@ static VALUE spg_set_single_row_mode(VALUE self) {
   return Qnil;
 }
 
-static VALUE spg__yield_each_row(VALUE self) {
-  PGresult *res;
-  VALUE rres;
-  VALUE rconn;
-  VALUE colsyms[SPG_MAX_FIELDS];
-  VALUE colconvert[SPG_MAX_FIELDS];
+struct spg__yield_each_row_stream_data {
+  VALUE self;
+  VALUE *colsyms;
+  VALUE *colconvert;
+  VALUE pg_value;
+  int enc_index;
+  char type;
+};
+
+static void spg__yield_each_row_stream(VALUE rres, int ntuples, int nfields, void *rdata) {
+  struct spg__yield_each_row_stream_data* data = (struct spg__yield_each_row_stream_data *)rdata;
+  VALUE h = rb_hash_new();
+  VALUE self = data->self;
+  VALUE *colsyms = data->colsyms;
+  VALUE *colconvert= data->colconvert;
+  PGresult *res = pgresult_get(rres);
+  int enc_index = data->enc_index;
+  long j;
+
+  for(j=0; j<nfields; j++) {
+    rb_hash_aset(h, colsyms[j], spg__col_value(self, res, 0, j, colconvert , enc_index));
+  }
+
+  if(data->type == SPG_YIELD_MODEL) {
+    VALUE model = rb_obj_alloc(data->pg_value);
+    rb_ivar_set(model, spg_id_values, h);
+    rb_yield(model);
+  } else {
+    rb_yield(h);
+  }
+}
+
+static VALUE spg__yield_each_row_internal(VALUE self, VALUE rconn, VALUE rres, PGresult *res, int enc_index, VALUE *colsyms, VALUE *colconvert) {
   long nfields;
   long j;
   VALUE h;
@@ -1645,19 +1701,9 @@ static VALUE spg__yield_each_row(VALUE self) {
   VALUE pg_type;
   VALUE pg_value = Qnil;
   char type = SPG_YIELD_NORMAL;
-  int enc_index;
-
-  rconn = rb_ary_entry(self, 1);
-  self = rb_ary_entry(self, 0);
+  struct spg__yield_each_row_stream_data data;
 
-  rres = rb_funcall(rconn, spg_id_get_result, 0);
-  if (rres == Qnil) {
-    goto end_yield_each_row;
-  }
-  rb_funcall(rres, spg_id_check, 0);
-  res = pgresult_get(rres);
-
-  enc_index = spg_use_pg_get_result_enc_idx ? pg_get_result_enc_idx(rres) : enc_get_index(rres);
+  nfields = PQnfields(res);
 
   /* Only handle regular and model types.  All other types require compiling all
    * of the results at once, which is not a use case for streaming.  The streaming
@@ -1671,14 +1717,20 @@ static VALUE spg__yield_each_row(VALUE self) {
     }
   }
 
-  nfields = PQnfields(res);
-  if (nfields > SPG_MAX_FIELDS) {
-    rb_funcall(rres, spg_id_clear, 0);
-    rb_raise(rb_eRangeError, "more than %d columns in query", SPG_MAX_FIELDS);
-  }
-
   spg_set_column_info(self, res, colsyms, colconvert, enc_index);
 
+  if (spg_use_pg_stream_any) {
+    data.self = self;
+    data.colsyms = colsyms;
+    data.colconvert = colconvert;
+    data.pg_value = pg_value;
+    data.enc_index = enc_index;
+    data.type = type;
+    
+    pgresult_stream_any(rres, spg__yield_each_row_stream, &data);
+    return self;
+  }
+
   while (PQntuples(res) != 0) {
     h = rb_hash_new();
     for(j=0; j<nfields; j++) {
@@ -1698,14 +1750,57 @@ static VALUE spg__yield_each_row(VALUE self) {
 
     rres = rb_funcall(rconn, spg_id_get_result, 0);
     if (rres == Qnil) {
-      goto end_yield_each_row;
+      return self;
     }
     rb_funcall(rres, spg_id_check, 0);
     res = pgresult_get(rres);
   }
   rb_funcall(rres, spg_id_clear, 0);
 
-end_yield_each_row:
+  return self;
+}
+
+#define def_spg__yield_each_row(max_fields) static VALUE spg__yield_each_row_ ## max_fields(VALUE self, VALUE rconn, VALUE rres, PGresult *res, int enc_index) { \
+  VALUE colsyms[max_fields]; \
+  VALUE colconvert[max_fields]; \
+  return spg__yield_each_row_internal(self, rconn, rres, res, enc_index, colsyms, colconvert); \
+}
+
+def_spg__yield_each_row(16)
+def_spg__yield_each_row(64)
+def_spg__yield_each_row(256)
+def_spg__yield_each_row(1664)
+
+static VALUE spg__yield_each_row(VALUE self) {
+  PGresult *res;
+  VALUE rres;
+  VALUE rconn;
+  int enc_index;
+  long nfields;
+
+  rconn = rb_ary_entry(self, 1);
+  self = rb_ary_entry(self, 0);
+
+  rres = rb_funcall(rconn, spg_id_get_result, 0);
+  if (rres == Qnil) {
+    return self;
+  }
+  rb_funcall(rres, spg_id_check, 0);
+  res = pgresult_get(rres);
+
+  enc_index = spg_use_pg_get_result_enc_idx ? pg_get_result_enc_idx(rres) : enc_get_index(rres);
+
+  nfields = PQnfields(res);
+  if (nfields <= 16) return spg__yield_each_row_16(self, rconn, rres, res, enc_index);
+  else if (nfields <= 64) return spg__yield_each_row_64(self, rconn, rres, res, enc_index);
+  else if (nfields <= 256) return spg__yield_each_row_256(self, rconn, rres, res, enc_index);
+  else if (nfields <= 1664) return spg__yield_each_row_1664(self, rconn, rres, res, enc_index);
+  else {
+    rb_funcall(rres, spg_id_clear, 0);
+    rb_raise(rb_eRangeError, "more than 1664 columns in query (%ld columns detected)", nfields);
+  }
+
+  /* UNREACHABLE */
   return self;
 }
 
@@ -1761,10 +1856,21 @@ void Init_sequel_pg(void) {
     }
   }
 
-  if (RTEST(rb_eval_string("defined?(PG::VERSION) && PG::VERSION.to_f >= 1.2"))) {
-    spg_use_pg_get_result_enc_idx = 1;
+  c = rb_eval_string("defined?(PG::VERSION) && PG::VERSION.split('.').map(&:to_i)");
+  if (RB_TYPE_P(c, T_ARRAY) && RARRAY_LEN(c) >= 3) {
+    if (FIX2INT(RARRAY_AREF(c, 0)) > 1) {
+      spg_use_pg_get_result_enc_idx = 1;
+      spg_use_pg_stream_any = 1;
+    } else if (FIX2INT(RARRAY_AREF(c, 0)) == 1) {
+      if (FIX2INT(RARRAY_AREF(c, 1)) >= 2) {
+        spg_use_pg_get_result_enc_idx = 1;
+      }
+      if (FIX2INT(RARRAY_AREF(c, 1)) > 3 || (FIX2INT(RARRAY_AREF(c, 1)) == 3 && FIX2INT(RARRAY_AREF(c, 2)) >= 4)) {
+        spg_use_pg_stream_any = 1;
+      }
+    }
   }
-  
+
   rb_const_set(spg_Postgres, rb_intern("SEQUEL_PG_VERSION_INTEGER"), INT2FIX(SEQUEL_PG_VERSION_INTEGER));
 
   spg_id_BigDecimal = rb_intern("BigDecimal");

data/lib/sequel/extensions/pg_streaming.rb

@@ -73,12 +73,18 @@ module Sequel::Postgres::Streaming
 
     private
 
+    unless Sequel::Postgres::Adapter.method_defined?(:send_query_params)
+      def send_query_params(*args)
+        send_query(*args)
+      end
+    end
+
     if Sequel::Database.instance_methods.map(&:to_s).include?('log_connection_yield')
       # If using single row mode, send the query instead of executing it.
       def execute_query(sql, args)
         if @single_row_mode
           @single_row_mode = false
-          @db.log_connection_yield(sql, self, args){args ? send_query(sql, args) : send_query(sql)}
+          @db.log_connection_yield(sql, self, args){args ? send_query_params(sql, args) : send_query(sql)}
           set_single_row_mode
           block
           self
@@ -122,7 +128,12 @@ module Sequel::Postgres::Streaming
       unless block_given?
         return enum_for(:paged_each, opts)
       end
-      stream.each(&block)
+
+      if stream_results?
+        each(&block)
+      else
+        super
+      end
     end
 
     # Return a clone of the dataset that will use streaming to load

data/lib/sequel_pg/sequel_pg.rb

@@ -120,6 +120,9 @@ if defined?(Sequel::Postgres::PGArray)
   # pg_array extension previously loaded
 
   class Sequel::Postgres::PGArray::Creator
+    # Avoid method redefined verbose warning
+    alias call call if method_defined?(:call)
+
     # Override Creator to use sequel_pg's C-based parser instead of the pure ruby parser.
     def call(string)
       Sequel::Postgres::PGArray.new(Sequel::Postgres.parse_pg_array(string, @converter), @type)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel_pg
 version: !ruby/object:Gem::Version
-  version: 1.12.5
+  version: 1.15.0
 platform: ruby
 authors:
 - Jeremy Evans
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2022-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -101,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Faster SELECTs when using Sequel with pg